From e633e10bf78ee38f62a08792730b8a05c17749e8 Mon Sep 17 00:00:00 2001 From: Asmo Bot Date: Tue, 21 Jul 2026 18:33:36 -0700 Subject: [PATCH 1/4] Rebuild Reachy voice and motion pipeline --- .env.example | 62 +- .gitignore | 1 + EMOTIONS.md | 87 +++ README.md | 44 +- index.html | 10 +- pyproject.toml | 20 +- services/voice_stack/README.md | 14 + services/voice_stack/app.py | 158 +++++ services/voice_stack/pyproject.toml | 28 + services/voice_stack/tests/test_contract.py | 33 + .../audio/conversation.py | 320 ++++++++++ src/reachy_mini_openclaw/audio/playback.py | 12 + src/reachy_mini_openclaw/audio/providers.py | 204 +++++++ src/reachy_mini_openclaw/camera_worker.py | 111 +++- src/reachy_mini_openclaw/config.py | 84 ++- src/reachy_mini_openclaw/gradio_app.py | 24 +- src/reachy_mini_openclaw/main.py | 145 ++++- src/reachy_mini_openclaw/motion_catalog.py | 379 ++++++++++++ src/reachy_mini_openclaw/moves.py | 43 +- src/reachy_mini_openclaw/openai_realtime.py | 563 ------------------ src/reachy_mini_openclaw/openclaw_bridge.py | 319 ++++++++-- src/reachy_mini_openclaw/prompts.py | 14 +- src/reachy_mini_openclaw/tools/__init__.py | 2 +- src/reachy_mini_openclaw/tools/core_tools.py | 120 ++-- tests/test_conversation.py | 163 +++++ tests/test_expressive_motion.py | 48 ++ tests/test_motion_catalog.py | 121 ++++ tests/test_playback.py | 16 + tests/test_speech_providers.py | 115 ++++ 29 files changed, 2490 insertions(+), 770 deletions(-) create mode 100644 EMOTIONS.md create mode 100644 services/voice_stack/README.md create mode 100644 services/voice_stack/app.py create mode 100644 services/voice_stack/pyproject.toml create mode 100644 services/voice_stack/tests/test_contract.py create mode 100644 src/reachy_mini_openclaw/audio/conversation.py create mode 100644 src/reachy_mini_openclaw/audio/playback.py create mode 100644 src/reachy_mini_openclaw/audio/providers.py create mode 100644 src/reachy_mini_openclaw/motion_catalog.py delete mode 100644 src/reachy_mini_openclaw/openai_realtime.py create mode 100644 tests/test_conversation.py create mode 100644 tests/test_expressive_motion.py create mode 100644 tests/test_motion_catalog.py create mode 100644 tests/test_playback.py create mode 100644 tests/test_speech_providers.py diff --git a/.env.example b/.env.example index f65de21..da9bf5d 100644 --- a/.env.example +++ b/.env.example @@ -2,18 +2,26 @@ # Give your OpenClaw AI agent a physical robot body! # ============================================================================== -# REQUIRED: OpenAI API Key +# OPTIONAL: OpenAI API Key (legacy cloud vision fallback only) # ============================================================================== # Get your key at: https://platform.openai.com/api-keys -# Requires Realtime API access OPENAI_API_KEY=sk-your-openai-key +# ============================================================================== +# REQUIRED: Always-hot local speech-to-text +# ============================================================================== +# OpenAI-compatible /v1/audio/transcriptions service on xeon-serv. +STT_BASE_URL=http://speech-host.local:8890/v1 +STT_MODEL=distil-large-v3 +STT_LANGUAGE=en +# STT_API_KEY=optional-local-bearer-token + # ============================================================================== # REQUIRED: OpenClaw Gateway # ============================================================================== # The URL where your OpenClaw gateway is running # If running on the same machine as the robot, use the host machine's IP -OPENCLAW_GATEWAY_URL=http://192.168.1.100:18789 +OPENCLAW_GATEWAY_URL=http://openclaw-host.local:18789 # Your OpenClaw gateway authentication token # Find this in ~/.openclaw/openclaw.json under gateway.token @@ -23,18 +31,42 @@ OPENCLAW_TOKEN=your-gateway-token OPENCLAW_AGENT_ID=main # Session key for conversation context - IMPORTANT! -# Use "main" (default) to share context with WhatsApp and other DM channels -# This allows the robot to be aware of all your conversations -OPENCLAW_SESSION_KEY=main +# Keep the robot in a dedicated session while using the same OpenClaw agent. +OPENCLAW_SESSION_KEY=reachy + +# Reachy daemon. These settings let ClawBody run on xeon-serv while using +# Reachy's remote WebRTC media and motion connection. +ROBOT_HOST=reachy-mini.local +ROBOT_PORT=8000 +ROBOT_CONNECTION_MODE=network # ============================================================================== -# OPTIONAL: Voice Settings +# REQUIRED: Voice Settings # ============================================================================== -# OpenAI Realtime voice (alloy, echo, fable, onyx, nova, shimmer, cedar) -OPENAI_VOICE=cedar +# Chatterbox is the default local provider. Its service uses a named voice +# profile installed on xeon-serv rather than receiving a secret/reference file +# on every request. +TTS_PROVIDER=chatterbox +CHATTERBOX_URL=http://speech-host.local:8890/v1/audio/speech +CHATTERBOX_VOICE=default +# Software playback gain before Reachy's hardware/app volume. Start at 1.0; +# values above 1.0 boost quiet voices with clipping protection. +AUDIO_OUTPUT_GAIN=1.0 + +# If both values are present, ElevenLabs is the automatic Chatterbox fallback. +# Set TTS_PROVIDER=elevenlabs to make it primary. +# ELEVENLABS_API_KEY=... +# ELEVENLABS_VOICE_ID=... +# ELEVENLABS_MODEL_ID=eleven_flash_v2_5 -# OpenAI model for Realtime API -OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17 +# Local VAD tuning +VAD_RMS_THRESHOLD=0.022 +VAD_ACTIVATION_MS=160 +VAD_PREFIX_MS=250 +VAD_SILENCE_MS=650 +VAD_MIN_SPEECH_MS=300 +VAD_MAX_SPEECH_SECONDS=20 +VAD_REQUIRE_HARDWARE_SPEECH=true # ============================================================================== # OPTIONAL: Features @@ -42,3 +74,11 @@ OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17 # Enable/disable features (true/false) ENABLE_CAMERA=true ENABLE_OPENCLAW_TOOLS=true +ENABLE_FACE_TRACKING=true +HEAD_TRACKER_TYPE=daemon +FACE_TRACKING_WEIGHT=0.7 +ENABLE_SOUND_TRACKING=true + +# Dedicated body-session latency. "off" is the instant conversational mode; +# it does not change reasoning settings for Discord, cron, or other sessions. +OPENCLAW_THINKING=off diff --git a/.gitignore b/.gitignore index 49ad75e..ee28adb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # Environment and secrets .env +.private/ *.env.local # Python diff --git a/EMOTIONS.md b/EMOTIONS.md new file mode 100644 index 0000000..c4d9603 --- /dev/null +++ b/EMOTIONS.md @@ -0,0 +1,87 @@ +# Reachy emotion map + +ClawBody exposes the 81 expressive recordings from Pollen's Reachy Mini +emotions dataset. Exact numbered names always select that recording; a family +name or conversational alias selects the first (restrained default) variant. + +The four additional dataset recordings are utilities, not emotions: +`mini-deep-sleep`, `toc-toc-toc`, `waiting`, and `wake-mini-up`. + +## Positive and social + +- amazed: `amazed1` (astonished, impressed, wow) +- attentive: `attentive1`, `attentive2` (alert, focused, listening) +- calming: `calming1` (soothing, reassuring) +- cheerful: `cheerful1` (happy, upbeat) +- come: `come1` (come here, beckon, invite) +- curious: `curious1` +- enthusiastic: `enthusiastic1`, `enthusiastic2` (excited, eager) +- grateful: `grateful1` (thankful) +- helpful: `helpful1`, `helpful2` (assist) +- laughing: `laughing1`, `laughing2` (laugh, amused) +- loving: `loving1` (love, affectionate) +- proud: `proud1`, `proud2`, `proud3` +- relief: `relief1`, `relief2` (relieved) +- serenity: `serenity1` (peaceful) +- success: `success1`, `success2` (victory, celebrating) +- understanding: `understanding1`, `understanding2` (got it, comprehend) +- welcoming: `welcoming1`, `welcoming2` (welcome, greet) +- yes: `yes1`, `yes_sad1` (agree, affirmative; “sad yes” selects `yes_sad1`) + +## Thinking and uncertainty + +- anxiety: `anxiety1` (anxious, nervous, worried) +- confused: `confused1` (puzzled) +- incomprehensible: `incomprehensible2` (baffled, don't understand) +- inquiring: `inquiring1`, `inquiring2`, `inquiring3` (questioning) +- lost: `lost1` +- oops: `oops1`, `oops2` (mistake) +- surprised: `surprised1`, `surprised2` (startled) +- thoughtful: `thoughtful1`, `thoughtful2` (thinking, contemplative) +- uncertain: `uncertain1` (unsure, doubtful) + +## Low energy and vulnerable + +- boredom: `boredom1`, `boredom2` (bored) +- downcast: `downcast1` (dejected) +- exhausted: `exhausted1` (wiped out) +- lonely: `lonely1` +- resigned: `resigned1` (give up, accept defeat) +- sad: `sad1`, `sad2` (unhappy) +- shy: `shy1` (bashful) +- sleep: `sleep1` (sleepy) +- tired: `tired1` +- uncomfortable: `uncomfortable1` (awkward, uneasy) + +## Negative and defensive + +- contempt: `contempt1` (disdain, dismissive) +- disgusted: `disgusted1` (grossed out) +- displeased: `displeased1`, `displeased2` (dissatisfied) +- fear: `fear1` (afraid) +- frustrated: `frustrated1` +- furious: `furious1` (livid) +- go away: `go_away1` (shoo, leave) +- impatient: `impatient1`, `impatient2` +- indifferent: `indifferent1` (meh, apathetic) +- irritated: `irritated1`, `irritated2` (annoyed) +- no: `no1`, `no_excited1`, `no_sad1` (decline, refuse; “excited no” and “sad no” select their exact variants) +- rage: `rage1` (enraged) +- reprimand: `reprimand1`, `reprimand2`, `reprimand3` (scold, admonish) +- scared: `scared1` (frightened) + +## Theatrical and movement reactions + +- dance: `dance1`, `dance2`, `dance3` (legacy emotion-dataset motions; explicit dance requests use the separate official dance library) +- dying: `dying1` (dramatic death) +- electric: `electric1` (energized) + +## Selection rules + +- Exact names win: “express proud three” selects `proud3`. +- Family names choose the first installed variant: “act proud” selects `proud1`. +- Common aliases resolve to a family: “show victory” selects `success1`. +- Authored emotion moves get exclusive body control; face tracking, sound tracking, + and speech wobble pause until the move completes. +- New upstream recordings appear as unmapped until intentionally assigned to a + family, preventing silent or nonsensical gesture selection. diff --git a/README.md b/README.md index 9a8d5a3..8e293c7 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ tags: - ai-assistant - voice-assistant - robotics - - openai-realtime + - local-speech - conversational-ai - physical-ai - robot-body @@ -33,9 +33,12 @@ tags: # 🦞🤖 ClawBody +See [EMOTIONS.md](EMOTIONS.md) for the complete 81-move Reachy emotion map, +semantic families, aliases, and selection behavior. + **Give your OpenClaw AI agent a physical robot body!** -ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body, using OpenAI's Realtime API for ultra-responsive voice conversation. Your OpenClaw assistant (Clawson) can now see, hear, speak, and move in the physical world. +ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body. Local STT transcribes the microphone, OpenClaw produces the actual response, and Chatterbox or ElevenLabs gives it a voice. ![Reachy Mini Dance](https://huggingface.co/spaces/pollen-robotics/reachy_mini_conversation_app/resolve/main/docs/assets/reachy_mini_dance.gif) @@ -48,21 +51,18 @@ ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot **The robot looks at you when you speak!** -ClawBody now includes real-time face tracking that makes conversations feel natural and engaging: +ClawBody uses Reachy Mini 1.9's daemon-side face tracking and microphone-array +direction-of-arrival support, avoiding a duplicate detector on the OpenClaw host: -- **Automatic Face Detection**: Uses MediaPipe or YOLO to detect faces at 25Hz +- **Automatic Face Detection**: Runs in Reachy's daemon and blends with expressive movement - **Smooth Head Tracking**: Robot smoothly follows your face as you move -- **Natural Eye Contact**: Maintains engagement during conversation -- **Graceful Fallback**: Smoothly returns to neutral position when you leave +- **Sound Tracking**: Turns conservatively toward speech when no face is visible +- **Visual Questions**: Attaches a current camera frame to OpenClaw when asked what it sees ```bash # Face tracking is enabled by default clawbody -# Choose your tracker (MediaPipe is lighter, YOLO is more accurate) -clawbody --head-tracker mediapipe -clawbody --head-tracker yolo - # Disable if needed clawbody --no-face-tracking ``` @@ -93,7 +93,7 @@ clawbody --gradio ## ✨ Features - **👁️ Face Tracking**: Robot tracks your face and maintains eye contact during conversation -- **🎤 Real-time Voice Conversation**: OpenAI Realtime API for sub-second response latency +- **🎤 Local Voice Pipeline**: Always-hot STT and pluggable Chatterbox/ElevenLabs speech - **🧠 OpenClaw Intelligence**: Your responses come from OpenClaw with full tool access - **👀 Vision**: See through the robot's camera and describe the environment - **💃 Expressive Movements**: Natural head movements, emotions, dances, and audio-driven wobble @@ -120,9 +120,9 @@ clawbody --gradio ┌─────────────────────────────────────────────────┼───────────────┐ │ ClawBody │ │ │ ┌─────────────────────────────────────────────┼────────────┐ │ -│ │ OpenAI Realtime API Handler │ │ │ -│ │ • Speech recognition (Whisper) │ │ │ -│ │ • Text-to-speech (voices) ─┘ │ │ +│ │ Provider-based Voice Pipeline │ │ │ +│ │ • Local STT + local VAD │ │ │ +│ │ • Chatterbox / ElevenLabs TTS ─┘ │ │ │ │ • Audio analysis → head wobble │ │ │ └─────────────────────────────────────────────────────────┘ │ │ │ │ @@ -156,7 +156,8 @@ clawbody --gradio - Python 3.11+ - [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini) installed - [OpenClaw](https://github.com/openclaw/openclaw) gateway running -- OpenAI API key with Realtime API access +- An OpenAI-compatible local transcription endpoint +- A local Chatterbox endpoint or ElevenLabs API credentials ## 🚀 Installation @@ -214,16 +215,19 @@ cp .env.example .env 2. Edit `.env` with your configuration: ```bash -# Required -OPENAI_API_KEY=sk-...your-key... +# Required local STT +STT_BASE_URL=http://speech-host.local:8890/v1 +STT_MODEL=distil-large-v3 # OpenClaw Gateway (required for AI responses) OPENCLAW_GATEWAY_URL=http://localhost:18789 # or your host IP OPENCLAW_TOKEN=your-gateway-token OPENCLAW_AGENT_ID=main -# Optional - Customize voice -OPENAI_VOICE=cedar +# Local Chatterbox voice profile +TTS_PROVIDER=chatterbox +CHATTERBOX_URL=http://speech-host.local:8890/v1/audio/speech +CHATTERBOX_VOICE=default # Optional - Face tracking (enabled by default) ENABLE_FACE_TRACKING=true @@ -308,7 +312,7 @@ ClawBody builds on: - [Pollen Robotics](https://www.pollen-robotics.com/) - Reachy Mini robot, SDK, and simulator - [OpenClaw](https://github.com/openclaw/openclaw) - AI assistant framework (Clawson!) -- [OpenAI](https://openai.com/) - Realtime API for voice I/O +- [Resemble AI Chatterbox](https://github.com/resemble-ai/chatterbox) - local speech synthesis - [MuJoCo](https://mujoco.org/) - Physics simulation engine - [pollen-robotics/reachy_mini_conversation_app](https://huggingface.co/spaces/pollen-robotics/reachy_mini_conversation_app) - Movement and audio systems diff --git a/index.html b/index.html index 79d1c84..5069592 100644 --- a/index.html +++ b/index.html @@ -26,15 +26,15 @@

Give OpenClaw a physical body.

Connect your OpenClaw AI assistant (Clawson) to a Reachy Mini robot. - Ultra-responsive voice conversation through OpenAI Realtime API, - intelligent responses from OpenClaw, and expressive robot movements. + Local speech recognition and expressive synthesis, intelligent + responses from OpenClaw, and expressive robot movements.

🖥️ Try with Simulator See features
- 🎙️ OpenAI Realtime + 🎙️ Local STT 🦞 OpenClaw Gateway 💃 Expressive movement 🖥️ No robot required! @@ -128,9 +128,9 @@

How it works

  1. 🎤 Robot (or simulator) captures your voice
  2. 🔇 Voice Activity Detection identifies when you stop speaking
    3. -
  3. 📝 OpenAI Realtime transcribes your speech instantly
    4. +
  4. 📝 Always-hot local STT transcribes your speech
  5. 🦞 OpenClaw processes your message with full AI capabilities
    6. -
  6. 🔊 OpenAI Realtime speaks the response naturally
    7. +
  7. 🔊 Chatterbox or ElevenLabs speaks the response naturally
  8. 🤖 Robot moves expressively while speaking
diff --git a/pyproject.toml b/pyproject.toml index 5ee2495..6c13857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "clawbody" version = "0.1.0" -description = "ClawBody - Give your OpenClaw AI agent a physical robot body with Reachy Mini. Voice conversation powered by OpenAI Realtime API with expressive movements." +description = "Give an OpenClaw agent a Reachy Mini body with local STT, pluggable TTS, and expressive movement." readme = "README.md" license = {text = "Apache-2.0"} requires-python = ">=3.11" @@ -19,7 +19,7 @@ keywords = [ "clawson", "robotics", "ai-assistant", - "openai-realtime", + "local-speech", "voice-conversation", "expressive-robot", "embodied-ai" @@ -35,13 +35,17 @@ classifiers = [ "Topic :: Scientific/Engineering :: Human Machine Interfaces", ] dependencies = [ - # OpenAI Realtime API + # Optional OpenAI client retained for the cloud vision fallback "openai>=1.50.0", + + # Local speech provider clients + "httpx>=0.27.0", - # Audio streaming - "fastrtc>=0.0.17", + # Audio processing "numpy", "scipy", + "pillow>=10", + "reachy-mini-dances-library>=0.2.1", # OpenClaw gateway client (WebSocket protocol) "websockets>=12.0", @@ -58,6 +62,9 @@ dependencies = [ # Or on the robot, it's pre-installed. [project.optional-dependencies] +robot = [ + "reachy-mini==1.9.0", +] wireless = [ "pygobject", ] @@ -114,6 +121,9 @@ target-version = "py311" select = ["E", "F", "I", "N", "W", "UP"] ignore = ["E501"] +[tool.pytest.ini_options] +testpaths = ["tests"] + [tool.mypy] python_version = "3.11" warn_return_any = true diff --git a/services/voice_stack/README.md b/services/voice_stack/README.md new file mode 100644 index 0000000..289c01d --- /dev/null +++ b/services/voice_stack/README.md @@ -0,0 +1,14 @@ +# Xeon voice stack + +Always-hot English speech services for ClawBody: + +- `POST /v1/audio/transcriptions`: faster-whisper `distil-large-v3` +- `POST /v1/audio/speech`: Chatterbox Turbo cloned from a configured voice reference +- `GET /health`: loaded-model readiness + +The service intentionally exposes an OpenAI-shaped API contract so ClawBody can switch providers without changing its conversation loop. It binds to loopback by default; use the Reachy LAN address only after firewalling it to the robot subnet. + +```bash +uv sync +ASMO_VOICE_REFERENCE=/path/to/asmo.wav uv run uvicorn app:app --host 127.0.0.1 --port 8890 +``` diff --git a/services/voice_stack/app.py b/services/voice_stack/app.py new file mode 100644 index 0000000..fd0fe59 --- /dev/null +++ b/services/voice_stack/app.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import io +import os +import tempfile +import threading +import time +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Annotated + +import soundfile as sf +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.responses import Response +from pydantic import BaseModel, Field + + +class SpeechRequest(BaseModel): + input: str = Field(min_length=1, max_length=1200) + voice: str = "asmo" + response_format: str = "wav" + speed: float = Field(default=1.0, ge=0.75, le=1.25) + temperature: float = Field(default=0.8, ge=0.05, le=2.0) + repetition_penalty: float = Field(default=1.2, ge=1.0, le=2.0) + + +class ModelRegistry: + def __init__(self) -> None: + self.device = os.getenv("VOICE_DEVICE", "cuda") + self.stt_model_name = os.getenv("STT_MODEL", "distil-large-v3") + self.stt_compute_type = os.getenv("STT_COMPUTE_TYPE", "int8_float16") + self.voice_reference = Path(os.environ["ASMO_VOICE_REFERENCE"]) + self.stt = None + self.tts = None + self.stt_lock = threading.Lock() + self.tts_lock = threading.Lock() + self.loaded_at: float | None = None + + def load(self) -> None: + if not self.voice_reference.is_file(): + raise RuntimeError(f"Voice reference does not exist: {self.voice_reference}") + + from faster_whisper import WhisperModel + from chatterbox.tts_turbo import ChatterboxTurboTTS + + self.stt = WhisperModel( + self.stt_model_name, + device=self.device, + compute_type=self.stt_compute_type, + ) + self.tts = ChatterboxTurboTTS.from_pretrained(self.device) + self.loaded_at = time.time() + + @property + def ready(self) -> bool: + return self.stt is not None and self.tts is not None + + +models: ModelRegistry | None = None + + +@asynccontextmanager +async def lifespan(_: FastAPI): + global models + models = ModelRegistry() + models.load() + yield + + +app = FastAPI(title="ClawBody Voice Stack", version="0.1.0", lifespan=lifespan) + + +def registry() -> ModelRegistry: + if models is None or not models.ready: + raise HTTPException(status_code=503, detail="Models are not ready") + return models + + +@app.get("/health") +def health() -> dict[str, object]: + current = registry() + return { + "ok": True, + "device": current.device, + "stt_model": current.stt_model_name, + "tts_model": "ResembleAI/chatterbox-turbo", + "loaded_at": current.loaded_at, + } + + +@app.post("/v1/audio/transcriptions") +def transcribe( + file: Annotated[UploadFile, File()], + language: Annotated[str, Form()] = "en", + response_format: Annotated[str, Form()] = "json", +) -> dict[str, object]: + current = registry() + suffix = Path(file.filename or "utterance.wav").suffix or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix) as tmp: + while chunk := file.file.read(1024 * 1024): + tmp.write(chunk) + tmp.flush() + started = time.perf_counter() + with current.stt_lock: + segments, info = current.stt.transcribe( + tmp.name, + language=language, + beam_size=1, + vad_filter=True, + condition_on_previous_text=False, + ) + segment_list = list(segments) + + text = " ".join(segment.text.strip() for segment in segment_list).strip() + elapsed = time.perf_counter() - started + result = { + "text": text, + "language": info.language, + "duration": info.duration, + "elapsed_seconds": elapsed, + } + if response_format == "verbose_json": + result["segments"] = [ + {"start": segment.start, "end": segment.end, "text": segment.text.strip()} + for segment in segment_list + ] + return result + + +@app.post("/v1/audio/speech") +def synthesize(request: SpeechRequest) -> Response: + if request.response_format != "wav": + raise HTTPException(status_code=400, detail="Only wav output is supported locally") + if request.voice != "asmo": + raise HTTPException(status_code=400, detail="Unknown local voice") + + current = registry() + started = time.perf_counter() + with current.tts_lock: + waveform = current.tts.generate( + request.input, + audio_prompt_path=str(current.voice_reference), + temperature=request.temperature, + repetition_penalty=request.repetition_penalty, + norm_loudness=True, + ) + + audio = waveform.squeeze(0).detach().cpu().numpy() + output = io.BytesIO() + sf.write(output, audio, current.tts.sr, format="WAV", subtype="PCM_16") + return Response( + content=output.getvalue(), + media_type="audio/wav", + headers={ + "X-Generation-Seconds": f"{time.perf_counter() - started:.3f}", + "X-Voice-Model": "chatterbox-turbo", + }, + ) diff --git a/services/voice_stack/pyproject.toml b/services/voice_stack/pyproject.toml new file mode 100644 index 0000000..825a13a --- /dev/null +++ b/services/voice_stack/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "asmo-voice-stack" +version = "0.1.0" +description = "Always-hot local STT and voice-cloned TTS for ClawBody" +requires-python = ">=3.11,<3.12" +dependencies = [ + "chatterbox-tts>=0.1.6", + "fastapi>=0.116.0", + "faster-whisper>=1.2.0", + "numpy<2", + "python-multipart>=0.0.20", + "setuptools<81", + "soundfile>=0.13.1", + "uvicorn[standard]>=0.35.0", +] + +[dependency-groups] +dev = [ + "httpx>=0.28.1", + "pytest>=8.4.0", +] + +[tool.uv] +package = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/services/voice_stack/tests/test_contract.py b/services/voice_stack/tests/test_contract.py new file mode 100644 index 0000000..def1218 --- /dev/null +++ b/services/voice_stack/tests/test_contract.py @@ -0,0 +1,33 @@ +import os + +os.environ.setdefault("ASMO_VOICE_REFERENCE", __file__) + +from fastapi.testclient import TestClient + +import app as voice_app + + +class FakeRegistry: + ready = True + device = "cuda" + stt_model_name = "distil-large-v3" + loaded_at = 1.0 + + +def test_health_contract(monkeypatch): + monkeypatch.setattr(voice_app, "models", FakeRegistry()) + client = TestClient(voice_app.app) + response = client.get("/health") + assert response.status_code == 200 + assert response.json()["ok"] is True + assert response.json()["stt_model"] == "distil-large-v3" + + +def test_speech_rejects_unknown_voice(monkeypatch): + monkeypatch.setattr(voice_app, "models", FakeRegistry()) + client = TestClient(voice_app.app) + response = client.post( + "/v1/audio/speech", + json={"input": "hello", "voice": "somebody-else"}, + ) + assert response.status_code == 400 diff --git a/src/reachy_mini_openclaw/audio/conversation.py b/src/reachy_mini_openclaw/audio/conversation.py new file mode 100644 index 0000000..ac9069f --- /dev/null +++ b/src/reachy_mini_openclaw/audio/conversation.py @@ -0,0 +1,320 @@ +"""Provider-based voice pipeline with OpenClaw as the only conversational brain.""" + +from __future__ import annotations + +import asyncio +import base64 +import logging +import re +import time +from collections import deque +from dataclasses import dataclass +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini_openclaw.audio.providers import SpeechToText, TextToSpeech + +logger = logging.getLogger(__name__) + +_VISION_REQUESTS = ( + re.compile(r"\b(?:can|do) you see\b", re.I), + re.compile(r"\bwhat (?:do|can) you see\b", re.I), + re.compile(r"\blook at\b", re.I), + re.compile(r"\bwhat(?:'s| is) (?:this|that|in front of you|on my|on the)\b", re.I), + re.compile(r"\bwho (?:is|are) (?:here|there|in front of you)\b", re.I), +) + + +def needs_vision(text: str) -> bool: + return any(pattern.search(text) for pattern in _VISION_REQUESTS) + + +@dataclass(frozen=True) +class Utterance: + sample_rate: int + samples: NDArray[np.float32] + + +class EnergyVAD: + """Small deterministic utterance segmenter for the always-hot local STT path.""" + + def __init__( + self, + *, + rms_threshold: float = 0.022, + activation_ms: int = 160, + prefix_ms: int = 250, + silence_ms: int = 650, + min_speech_ms: int = 300, + max_speech_seconds: float = 20.0, + ) -> None: + self.rms_threshold = rms_threshold + self.activation_ms = activation_ms + self.prefix_ms = prefix_ms + self.silence_ms = silence_ms + self.min_speech_ms = min_speech_ms + self.max_speech_seconds = max_speech_seconds + self.speaking = False + self._sample_rate = 0 + self._prefix: deque[NDArray[np.float32]] = deque() + self._prefix_samples = 0 + self._frames: list[NDArray[np.float32]] = [] + self._activation_samples = 0 + self._voiced_samples = 0 + self._silence_samples = 0 + + @staticmethod + def normalize(audio: NDArray[Any]) -> NDArray[np.float32]: + values = np.asarray(audio) + if values.ndim == 2: + channel_axis = 0 if values.shape[0] <= values.shape[1] else 1 + values = values.mean(axis=channel_axis) + values = values.reshape(-1) + if values.dtype == np.int16: + return values.astype(np.float32) / 32768.0 + return values.astype(np.float32) + + def feed( + self, + sample_rate: int, + audio: NDArray[Any], + *, + allow_start: bool = True, + ) -> Utterance | None: + frame = self.normalize(audio) + if frame.size == 0: + return None + if self._sample_rate and sample_rate != self._sample_rate: + self.reset() + self._sample_rate = sample_rate + rms = float(np.sqrt(np.mean(np.square(frame, dtype=np.float32)))) + voiced = rms >= self.rms_threshold and (self.speaking or allow_start) + + if not self.speaking: + self._prefix.append(frame.copy()) + self._prefix_samples += frame.size + prefix_limit = max(1, int(sample_rate * self.prefix_ms / 1000)) + while self._prefix and self._prefix_samples - self._prefix[0].size >= prefix_limit: + self._prefix_samples -= self._prefix.popleft().size + if not voiced: + self._activation_samples = 0 + return None + self._activation_samples += frame.size + activation_limit = max(1, int(sample_rate * self.activation_ms / 1000)) + if self._activation_samples < activation_limit: + return None + self.speaking = True + self._frames = list(self._prefix) + self._voiced_samples = self._activation_samples + self._silence_samples = 0 + self._prefix.clear() + self._prefix_samples = 0 + return None + + self._frames.append(frame.copy()) + if voiced: + self._voiced_samples += frame.size + self._silence_samples = 0 + else: + self._silence_samples += frame.size + + silence_limit = int(sample_rate * self.silence_ms / 1000) + max_limit = int(sample_rate * self.max_speech_seconds) + if self._silence_samples < silence_limit and sum(chunk.size for chunk in self._frames) < max_limit: + return None + + min_voiced = int(sample_rate * self.min_speech_ms / 1000) + samples = np.concatenate(self._frames) if self._voiced_samples >= min_voiced else None + self.reset() + return Utterance(sample_rate, samples) if samples is not None else None + + def reset(self) -> None: + self.speaking = False + self._sample_rate = 0 + self._prefix.clear() + self._prefix_samples = 0 + self._frames = [] + self._activation_samples = 0 + self._voiced_samples = 0 + self._silence_samples = 0 + + +class ConversationHandler: + """Turn Reachy microphone frames into spoken responses from OpenClaw.""" + + def __init__( + self, + *, + stt: SpeechToText, + tts: TextToSpeech, + openclaw_bridge: Any, + deps: Any, + vad: EnergyVAD, + output_chunk_samples: int = 2048, + attention_provider: Any = None, + require_hardware_speech: bool = True, + ) -> None: + self.stt = stt + self.tts = tts + self.openclaw_bridge = openclaw_bridge + self.deps = deps + self.vad = vad + self.output_chunk_samples = output_chunk_samples + self.attention_provider = attention_provider + self.require_hardware_speech = require_hardware_speech + self.output_queue: asyncio.Queue[tuple[int, NDArray[np.int16]] | dict[str, str]] = asyncio.Queue() + self._utterances: asyncio.Queue[Utterance] = asyncio.Queue(maxsize=2) + self._shutdown_requested = False + self._ignore_input_until = 0.0 + + async def start_up(self) -> None: + while not self._shutdown_requested: + utterance = await self._utterances.get() + try: + await self._process_utterance(utterance) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.error("Voice turn failed: %s", exc, exc_info=True) + self.deps.movement_manager.set_processing(False) + + async def receive(self, frame: tuple[int, NDArray[Any]]) -> None: + sample_rate, audio = frame + if time.monotonic() < self._ignore_input_until: + if self.vad.speaking: + self.vad.reset() + self.deps.movement_manager.set_listening(False) + return + was_speaking = self.vad.speaking + allow_start = True + attention_state = None + if not was_speaking and self.require_hardware_speech and self.attention_provider is not None: + attention_state = self.attention_provider.get_attention_state() + # Fail open if DoA is unavailable. When it is available, require + # the microphone array's own speech classifier as a second vote. + if attention_state.get("doa_angle") is not None: + allow_start = bool(attention_state.get("speech_detected")) + utterance = self.vad.feed(sample_rate, audio, allow_start=allow_start) + if self.vad.speaking and not was_speaking: + logger.info("Listening started (attention=%s)", attention_state) + self.deps.movement_manager.set_listening(True) + self.deps.movement_manager.set_processing(False) + self._clear_audio_output() + if self.deps.head_wobbler is not None: + self.deps.head_wobbler.reset() + if was_speaking and not self.vad.speaking: + self.deps.movement_manager.set_listening(False) + if utterance is not None: + if self._utterances.full(): + logger.warning("Dropping utterance because the conversational pipeline is busy") + return + await self._utterances.put(utterance) + + async def _process_utterance(self, utterance: Utterance) -> None: + turn_started = time.monotonic() + self.deps.movement_manager.set_processing(True) + transcript = await self.stt.transcribe(utterance.samples, utterance.sample_rate) + stt_done = time.monotonic() + if not transcript: + self.deps.movement_manager.set_processing(False) + return + logger.info("User: %s", transcript) + await self.output_queue.put({"role": "user", "content": transcript}) + + if not self.openclaw_bridge.is_connected: + if not await self.openclaw_bridge.connect(): + raise RuntimeError("OpenClaw gateway is unavailable") + motion_catalog = getattr(self.deps, "motion_catalog", None) + motion_hint = f" {motion_catalog.response_hint()}" if motion_catalog is not None else "" + image_b64 = None + if needs_vision(transcript) and self.attention_provider is not None: + jpeg = self.attention_provider.get_latest_jpeg() + if jpeg: + image_b64 = base64.b64encode(jpeg).decode("ascii") + logger.info("Attached current Reachy camera frame to visual question") + response = await self.openclaw_bridge.chat( + transcript, + image_b64=image_b64, + system_context=( + "The user is speaking to you through your Reachy Mini body. " + "Answer naturally and concisely for spoken delivery. " + "When a camera image is attached, it is your current first-person view." + + motion_hint + ), + ) + openclaw_done = time.monotonic() + if response.error: + raise RuntimeError(f"OpenClaw response failed: {response.error}") + if motion_catalog is not None: + text, directive = motion_catalog.parse_response(response.content) + directive = ( + directive + or motion_catalog.infer_request(transcript) + or motion_catalog.infer(text) + ) + else: + text, directive = response.content.strip(), None + if not text: + raise RuntimeError("OpenClaw returned an empty response") + + logger.info("Assistant: %s", text) + audio = await self.tts.synthesize(text) + tts_done = time.monotonic() + # Reachy's microphone hears its own speaker over the remote WebRTC + # stream. Suppress VAD for the synthesized duration plus a short room- + # echo tail so the robot does not recursively answer itself. + self._ignore_input_until = max( + self._ignore_input_until, + time.monotonic() + (audio.samples.size / audio.sample_rate) + 0.8, + ) + self.vad.reset() + self.deps.movement_manager.set_listening(False) + self.deps.movement_manager.set_processing(False) + # Start primary motion only after synthesis is ready. This synchronizes + # the gesture with playback and prevents thinking offsets from being + # blended into the opening of recorded choreography. + if directive is not None: + logger.info("Motion: %s:%s", directive.kind, directive.name) + self.deps.movement_manager.queue_move( + motion_catalog.get(directive.kind, directive.name) + ) + await self.output_queue.put({"role": "assistant", "content": text}) + for offset in range(0, audio.samples.size, self.output_chunk_samples): + chunk = audio.samples[offset : offset + self.output_chunk_samples] + await self.output_queue.put((audio.sample_rate, chunk.reshape(1, -1))) + logger.info( + "Turn latency: stt=%.2fs openclaw=%.2fs tts=%.2fs ready=%.2fs", + stt_done - turn_started, + openclaw_done - stt_done, + tts_done - openclaw_done, + tts_done - turn_started, + ) + + def _clear_audio_output(self) -> None: + retained: list[dict[str, str]] = [] + while not self.output_queue.empty(): + try: + item = self.output_queue.get_nowait() + if isinstance(item, dict): + retained.append(item) + except asyncio.QueueEmpty: + break + for item in retained: + self.output_queue.put_nowait(item) + + async def emit(self) -> tuple[int, NDArray[np.int16]] | dict[str, str] | None: + try: + return await asyncio.wait_for(self.output_queue.get(), timeout=0.1) + except TimeoutError: + return None + + async def shutdown(self) -> None: + self._shutdown_requested = True + self.vad.reset() + self._clear_audio_output() + self.deps.movement_manager.set_listening(False) + self.deps.movement_manager.set_processing(False) + if self.deps.head_wobbler is not None: + self.deps.head_wobbler.reset() diff --git a/src/reachy_mini_openclaw/audio/playback.py b/src/reachy_mini_openclaw/audio/playback.py new file mode 100644 index 0000000..62738b3 --- /dev/null +++ b/src/reachy_mini_openclaw/audio/playback.py @@ -0,0 +1,12 @@ +"""Audio preparation helpers for Reachy speaker playback.""" + +from __future__ import annotations + +import numpy as np +from numpy.typing import NDArray + + +def apply_output_gain(samples: NDArray[np.floating], gain: float) -> NDArray[np.float32]: + """Apply software gain and safely constrain samples to Reachy's float range.""" + audio = np.asarray(samples, dtype=np.float32) + return np.clip(audio * gain, -1.0, 1.0).astype(np.float32, copy=False) diff --git a/src/reachy_mini_openclaw/audio/providers.py b/src/reachy_mini_openclaw/audio/providers.py new file mode 100644 index 0000000..81dbe08 --- /dev/null +++ b/src/reachy_mini_openclaw/audio/providers.py @@ -0,0 +1,204 @@ +"""Provider adapters for ClawBody speech recognition and synthesis.""" + +from __future__ import annotations + +import io +import logging +import wave +from dataclasses import dataclass +from typing import Protocol + +import httpx +import numpy as np +from numpy.typing import NDArray + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SynthesizedAudio: + """Mono signed 16-bit PCM returned by a TTS provider.""" + + sample_rate: int + samples: NDArray[np.int16] + + +class SpeechToText(Protocol): + async def transcribe(self, samples: NDArray[np.float32], sample_rate: int) -> str: ... + + +class TextToSpeech(Protocol): + async def synthesize(self, text: str) -> SynthesizedAudio: ... + + +def encode_wav(samples: NDArray[np.float32], sample_rate: int) -> bytes: + """Encode normalized mono float audio as a PCM16 WAV.""" + mono = np.asarray(samples, dtype=np.float32).reshape(-1) + pcm = (np.clip(mono, -1.0, 1.0) * 32767.0).astype(" SynthesizedAudio: + """Decode a mono/stereo PCM WAV response to mono int16 samples.""" + with wave.open(io.BytesIO(payload), "rb") as wav: + channels = wav.getnchannels() + width = wav.getsampwidth() + sample_rate = wav.getframerate() + frames = wav.readframes(wav.getnframes()) + if width != 2: + raise ValueError(f"TTS WAV must be PCM16, got {width * 8}-bit audio") + samples = np.frombuffer(frames, dtype=" 1: + samples = samples.reshape(-1, channels).astype(np.int32).mean(axis=1).astype(np.int16) + return SynthesizedAudio(sample_rate=sample_rate, samples=samples.copy()) + + +class OpenAICompatibleSTT: + """Client for a local/OpenAI-compatible audio transcription endpoint.""" + + def __init__( + self, + base_url: str, + model: str, + *, + api_key: str | None = None, + language: str = "en", + timeout: float = 30.0, + client: httpx.AsyncClient | None = None, + ) -> None: + base = base_url.rstrip("/") + self.url = base if base.endswith("/audio/transcriptions") else f"{base}/audio/transcriptions" + self.model = model + self.api_key = api_key + self.language = language + self.timeout = timeout + self._client = client + + async def transcribe(self, samples: NDArray[np.float32], sample_rate: int) -> str: + headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} + data = {"model": self.model, "response_format": "json"} + if self.language: + data["language"] = self.language + files = {"file": ("utterance.wav", encode_wav(samples, sample_rate), "audio/wav")} + client = self._client or httpx.AsyncClient(timeout=self.timeout) + try: + response = await client.post(self.url, headers=headers, data=data, files=files) + response.raise_for_status() + body = response.json() + text = body.get("text", "") if isinstance(body, dict) else "" + return str(text).strip() + finally: + if self._client is None: + await client.aclose() + + +class ChatterboxTTS: + """Client for the local Chatterbox OpenAI-compatible speech endpoint.""" + + def __init__( + self, + url: str, + *, + voice: str = "asmo", + api_key: str | None = None, + sample_rate: int = 24000, + timeout: float = 45.0, + client: httpx.AsyncClient | None = None, + ) -> None: + self.url = url + self.voice = voice + self.api_key = api_key + self.sample_rate = sample_rate + self.timeout = timeout + self._client = client + + async def synthesize(self, text: str) -> SynthesizedAudio: + headers = {"Accept": "audio/wav"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + payload = { + "input": text, + "voice": self.voice, + "model": "chatterbox-turbo", + "response_format": "wav", + } + client = self._client or httpx.AsyncClient(timeout=self.timeout) + try: + response = await client.post(self.url, headers=headers, json=payload) + response.raise_for_status() + content_type = response.headers.get("content-type", "").lower() + if "wav" in content_type or response.content[:4] == b"RIFF": + return decode_wav(response.content) + return SynthesizedAudio( + sample_rate=self.sample_rate, + samples=np.frombuffer(response.content, dtype=" None: + self.url = f"{base_url.rstrip('/')}/v1/text-to-speech/{voice_id}/stream" + self.api_key = api_key + self.model_id = model_id + self.sample_rate = sample_rate + self.timeout = timeout + self._client = client + + async def synthesize(self, text: str) -> SynthesizedAudio: + client = self._client or httpx.AsyncClient(timeout=self.timeout) + try: + response = await client.post( + self.url, + params={"output_format": f"pcm_{self.sample_rate}"}, + headers={"xi-api-key": self.api_key, "Accept": "application/octet-stream"}, + json={"text": text, "model_id": self.model_id}, + ) + response.raise_for_status() + return SynthesizedAudio( + sample_rate=self.sample_rate, + samples=np.frombuffer(response.content, dtype=" None: + if not providers: + raise ValueError("at least one TTS provider is required") + self.providers = providers + + async def synthesize(self, text: str) -> SynthesizedAudio: + last_error: Exception | None = None + for provider in self.providers: + try: + return await provider.synthesize(text) + except Exception as exc: + last_error = exc + logger.warning("TTS provider %s failed: %s", type(provider).__name__, exc) + assert last_error is not None + raise last_error diff --git a/src/reachy_mini_openclaw/camera_worker.py b/src/reachy_mini_openclaw/camera_worker.py index 453b50d..36570be 100644 --- a/src/reachy_mini_openclaw/camera_worker.py +++ b/src/reachy_mini_openclaw/camera_worker.py @@ -13,9 +13,13 @@ import time import logging import threading +import math +from io import BytesIO from typing import Any, List, Tuple, Optional +import httpx import numpy as np +from PIL import Image from numpy.typing import NDArray from scipy.spatial.transform import Rotation as R @@ -36,7 +40,16 @@ class CameraWorker: RETURNING -- interpolating back to neutral before scanning again """ - def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None: + def __init__( + self, + reachy_mini: ReachyMini, + head_tracker: Any = None, + *, + daemon_tracking: bool = False, + daemon_url: str | None = None, + tracking_weight: float = 0.7, + sound_tracking: bool = True, + ) -> None: """Initialize camera worker. Args: @@ -45,6 +58,11 @@ def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None: """ self.reachy_mini = reachy_mini self.head_tracker = head_tracker + self.daemon_tracking = daemon_tracking + self.daemon_url = daemon_url.rstrip("/") if daemon_url else None + self.tracking_weight = min(1.0, max(0.0, tracking_weight)) + self.sound_tracking = sound_tracking + self._http = httpx.Client(timeout=0.5) if self.daemon_url else None # Thread-safe frame storage self.latest_frame: Optional[NDArray[np.uint8]] = None @@ -58,6 +76,12 @@ def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None: 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] # x, y, z, roll, pitch, yaw self.face_tracking_lock = threading.Lock() + self.face_detected = False + self.speech_detected = False + self.doa_angle: Optional[float] = None + self._last_attention_poll = 0.0 + self._sound_yaw = 0.0 + self._ever_received_frame = False # Face tracking timing (for smooth interpolation back to neutral) self.last_face_detected_time: Optional[float] = None @@ -104,6 +128,15 @@ def get_latest_frame(self) -> Optional[NDArray[np.uint8]]: return None return self.latest_frame.copy() + def get_latest_jpeg(self, quality: int = 80) -> bytes | None: + """Encode the latest BGR camera frame for an OpenClaw image turn.""" + frame = self.get_latest_frame() + if frame is None: + return None + output = BytesIO() + Image.fromarray(frame[:, :, ::-1]).save(output, format="JPEG", quality=quality) + return output.getvalue() + def get_face_tracking_offsets( self, ) -> Tuple[float, float, float, float, float, float]: @@ -128,6 +161,11 @@ def set_head_tracking_enabled(self, enabled: bool) -> None: # Start scanning immediately when re-enabled self._start_scanning() self.is_head_tracking_enabled = enabled + if self.daemon_tracking: + if enabled: + self.reachy_mini.start_head_tracking(weight=self.tracking_weight) + else: + self.reachy_mini.stop_head_tracking() logger.info("Head tracking %s", "enabled" if enabled else "disabled") def start(self) -> None: @@ -142,8 +180,67 @@ def stop(self) -> None: self._stop_event.set() if self._thread is not None: self._thread.join(timeout=2.0) + if self.daemon_tracking: + try: + self.reachy_mini.stop_head_tracking() + except Exception: + logger.debug("Could not stop daemon head tracking", exc_info=True) + if self._http is not None: + self._http.close() logger.info("Camera worker stopped") + def get_attention_state(self) -> dict[str, Any]: + """Return the latest daemon face and microphone-array state.""" + with self.face_tracking_lock: + return { + "face_detected": self.face_detected, + "speech_detected": self.speech_detected, + "doa_angle": self.doa_angle, + } + + def _update_daemon_attention(self, current_time: float) -> None: + """Blend daemon face tracking with a conservative sound-direction cue.""" + if current_time - self._last_attention_poll < 0.1: + return + self._last_attention_poll = current_time + + try: + face = self.reachy_mini.get_tracked_face(wait=False) + face_detected = bool(face and face.detected) + except Exception: + face_detected = False + + doa_angle: Optional[float] = None + speech_detected = False + if self.sound_tracking and self._http is not None: + try: + payload = self._http.get(f"{self.daemon_url}/api/state/doa").json() + if payload: + doa_angle = float(payload["angle"]) + speech_detected = bool(payload["speech_detected"]) + except Exception: + logger.debug("DoA poll failed", exc_info=True) + + # Visual tracking runs inside the daemon and wins whenever a face is + # visible. Otherwise, softly bias the normal app pose toward speech. + if face_detected: + target_yaw = 0.0 + elif speech_detected and doa_angle is not None: + target_yaw = float(np.clip((math.pi / 2) - doa_angle, -0.75, 0.75)) + else: + target_yaw = 0.0 + + alpha = 0.35 if speech_detected and not face_detected else 0.18 + self._sound_yaw = alpha * target_yaw + (1.0 - alpha) * self._sound_yaw + if abs(self._sound_yaw) < 0.01: + self._sound_yaw = 0.0 + + with self.face_tracking_lock: + self.face_detected = face_detected + self.speech_detected = speech_detected + self.doa_angle = doa_angle + self.face_tracking_offsets = [0.0, 0.0, 0.0, 0.0, 0.0, self._sound_yaw] + # ------------------------------------------------------------------ # Scanning helpers # ------------------------------------------------------------------ @@ -202,10 +299,16 @@ def _working_loop(self) -> None: frame = self.reachy_mini.media.get_frame() if frame is not None: + if not self._ever_received_frame: + self._ever_received_frame = True + logger.info("First camera frame received: shape=%s", frame.shape) # Thread-safe frame storage with self.frame_lock: self.latest_frame = frame + if self.daemon_tracking and self.is_head_tracking_enabled: + self._update_daemon_attention(current_time) + # Check if face tracking was just disabled if self.previous_head_tracking_state and not self.is_head_tracking_enabled: # Face tracking was just disabled - start interpolation to neutral @@ -218,7 +321,9 @@ def _working_loop(self) -> None: self.previous_head_tracking_state = self.is_head_tracking_enabled # Handle face tracking if enabled and head tracker available - if self.is_head_tracking_enabled and self.head_tracker is not None: + if self.daemon_tracking: + pass + elif self.is_head_tracking_enabled and self.head_tracker is not None: self._process_face_tracking(frame, current_time, neutral_pose) elif self.last_face_detected_time is not None: # Handle interpolation back to neutral when tracking disabled @@ -379,4 +484,4 @@ def _interpolate_to_neutral( self.interpolation_start_time = None self.interpolation_start_pose = None self._smoothed_offsets = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - self._start_scanning() \ No newline at end of file + self._start_scanning() diff --git a/src/reachy_mini_openclaw/config.py b/src/reachy_mini_openclaw/config.py index 0061f45..8deb14c 100644 --- a/src/reachy_mini_openclaw/config.py +++ b/src/reachy_mini_openclaw/config.py @@ -3,6 +3,7 @@ Handles environment variables and configuration settings for the application. """ +import json import os from pathlib import Path from dataclasses import dataclass, field @@ -15,25 +16,74 @@ load_dotenv(_project_root / ".env") +def _gateway_token() -> Optional[str]: + token = os.getenv("OPENCLAW_TOKEN") or os.getenv("OPENCLAW_GATEWAY_TOKEN") + if token: + return token + config_path = Path.home() / ".openclaw" / "openclaw.json" + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + return data.get("gateway", {}).get("auth", {}).get("token") + except (OSError, ValueError, TypeError): + return None + + @dataclass class Config: """Application configuration loaded from environment variables.""" - # OpenAI Configuration + # Optional OpenAI configuration (currently used only by the legacy vision fallback) OPENAI_API_KEY: str = field(default_factory=lambda: os.getenv("OPENAI_API_KEY", "")) - OPENAI_MODEL: str = field(default_factory=lambda: os.getenv("OPENAI_MODEL", "gpt-realtime-1.5")) - OPENAI_VOICE: str = field(default_factory=lambda: os.getenv("OPENAI_VOICE", "cedar")) + + # Speech-to-text. The endpoint follows OpenAI's multipart + # /v1/audio/transcriptions contract, but can be entirely local. + STT_BASE_URL: str = field(default_factory=lambda: os.getenv("STT_BASE_URL", "http://speech-host.local:8890/v1")) + STT_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("STT_API_KEY")) + STT_MODEL: str = field(default_factory=lambda: os.getenv("STT_MODEL", "distil-large-v3")) + STT_LANGUAGE: str = field(default_factory=lambda: os.getenv("STT_LANGUAGE", "en")) + STT_TIMEOUT_SECONDS: float = field(default_factory=lambda: float(os.getenv("STT_TIMEOUT_SECONDS", "30"))) + + # Text-to-speech. Chatterbox is primary; ElevenLabs can be enabled as a + # fallback without changing the conversational pipeline. + TTS_PROVIDER: str = field(default_factory=lambda: os.getenv("TTS_PROVIDER", "chatterbox").lower()) + CHATTERBOX_URL: str = field(default_factory=lambda: os.getenv("CHATTERBOX_URL", "http://speech-host.local:8890/v1/audio/speech")) + CHATTERBOX_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("CHATTERBOX_API_KEY")) + CHATTERBOX_VOICE: str = field(default_factory=lambda: os.getenv("CHATTERBOX_VOICE", "asmo")) + CHATTERBOX_SAMPLE_RATE: int = field(default_factory=lambda: int(os.getenv("CHATTERBOX_SAMPLE_RATE", "24000"))) + ELEVENLABS_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("ELEVENLABS_API_KEY")) + ELEVENLABS_VOICE_ID: Optional[str] = field(default_factory=lambda: os.getenv("ELEVENLABS_VOICE_ID")) + ELEVENLABS_MODEL_ID: str = field(default_factory=lambda: os.getenv("ELEVENLABS_MODEL_ID", "eleven_flash_v2_5")) + ELEVENLABS_BASE_URL: str = field(default_factory=lambda: os.getenv("ELEVENLABS_BASE_URL", "https://api.elevenlabs.io")) + TTS_TIMEOUT_SECONDS: float = field(default_factory=lambda: float(os.getenv("TTS_TIMEOUT_SECONDS", "45"))) + # Software gain applied immediately before Reachy playback. Reachy's app + # volume remains the final hardware control; 1.0 preserves the TTS signal. + AUDIO_OUTPUT_GAIN: float = field(default_factory=lambda: float(os.getenv("AUDIO_OUTPUT_GAIN", "1.0"))) + + # Local voice activity detection. + VAD_RMS_THRESHOLD: float = field(default_factory=lambda: float(os.getenv("VAD_RMS_THRESHOLD", "0.022"))) + VAD_ACTIVATION_MS: int = field(default_factory=lambda: int(os.getenv("VAD_ACTIVATION_MS", "160"))) + VAD_PREFIX_MS: int = field(default_factory=lambda: int(os.getenv("VAD_PREFIX_MS", "250"))) + VAD_SILENCE_MS: int = field(default_factory=lambda: int(os.getenv("VAD_SILENCE_MS", "650"))) + VAD_MIN_SPEECH_MS: int = field(default_factory=lambda: int(os.getenv("VAD_MIN_SPEECH_MS", "300"))) + VAD_MAX_SPEECH_SECONDS: float = field(default_factory=lambda: float(os.getenv("VAD_MAX_SPEECH_SECONDS", "20"))) + VAD_REQUIRE_HARDWARE_SPEECH: bool = field(default_factory=lambda: os.getenv("VAD_REQUIRE_HARDWARE_SPEECH", "true").lower() == "true") # OpenClaw Gateway Configuration OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789")) - OPENCLAW_TOKEN: Optional[str] = field(default_factory=lambda: os.getenv("OPENCLAW_TOKEN")) + OPENCLAW_TOKEN: Optional[str] = field(default_factory=_gateway_token) OPENCLAW_AGENT_ID: str = field(default_factory=lambda: os.getenv("OPENCLAW_AGENT_ID", "main")) + # Spoken body turns should feel immediate. "off" is OpenClaw's lowest- + # latency setting and applies only to the dedicated Reachy session. + OPENCLAW_THINKING: str = field(default_factory=lambda: os.getenv("OPENCLAW_THINKING", "off").lower()) # Session key for OpenClaw - uses "main" to share context with WhatsApp and other channels # Format: agent::, but we only need the session key part here OPENCLAW_SESSION_KEY: str = field(default_factory=lambda: os.getenv("OPENCLAW_SESSION_KEY", "main")) # Robot Configuration ROBOT_NAME: Optional[str] = field(default_factory=lambda: os.getenv("ROBOT_NAME")) + ROBOT_HOST: str = field(default_factory=lambda: os.getenv("ROBOT_HOST", "reachy-mini.local")) + ROBOT_PORT: int = field(default_factory=lambda: int(os.getenv("ROBOT_PORT", "8000"))) + ROBOT_CONNECTION_MODE: str = field(default_factory=lambda: os.getenv("ROBOT_CONNECTION_MODE", "auto")) # Feature Flags ENABLE_OPENCLAW_TOOLS: bool = field(default_factory=lambda: os.getenv("ENABLE_OPENCLAW_TOOLS", "true").lower() == "true") @@ -41,8 +91,11 @@ class Config: ENABLE_FACE_TRACKING: bool = field(default_factory=lambda: os.getenv("ENABLE_FACE_TRACKING", "true").lower() == "true") # Face Tracking Configuration - # Options: "yolo", "mediapipe", or None for auto-detect - HEAD_TRACKER_TYPE: Optional[str] = field(default_factory=lambda: os.getenv("HEAD_TRACKER_TYPE", "yolo")) + # "daemon" uses Reachy 1.9's built-in detector and avoids a duplicate model. + # "yolo" and "mediapipe" remain available for older daemon versions. + HEAD_TRACKER_TYPE: Optional[str] = field(default_factory=lambda: os.getenv("HEAD_TRACKER_TYPE", "daemon")) + FACE_TRACKING_WEIGHT: float = field(default_factory=lambda: float(os.getenv("FACE_TRACKING_WEIGHT", "0.7"))) + ENABLE_SOUND_TRACKING: bool = field(default_factory=lambda: os.getenv("ENABLE_SOUND_TRACKING", "true").lower() == "true") # Local Vision Processing ENABLE_LOCAL_VISION: bool = field(default_factory=lambda: os.getenv("ENABLE_LOCAL_VISION", "false").lower() == "true") @@ -56,8 +109,23 @@ class Config: def validate(self) -> list[str]: """Validate configuration and return list of errors.""" errors = [] - if not self.OPENAI_API_KEY: - errors.append("OPENAI_API_KEY is required") + if not self.STT_BASE_URL: + errors.append("STT_BASE_URL is required") + if self.TTS_PROVIDER not in {"chatterbox", "elevenlabs"}: + errors.append("TTS_PROVIDER must be 'chatterbox' or 'elevenlabs'") + if self.TTS_PROVIDER == "chatterbox" and not self.CHATTERBOX_URL: + errors.append("CHATTERBOX_URL is required for Chatterbox TTS") + if self.TTS_PROVIDER == "elevenlabs": + if not self.ELEVENLABS_API_KEY: + errors.append("ELEVENLABS_API_KEY is required for ElevenLabs TTS") + if not self.ELEVENLABS_VOICE_ID: + errors.append("ELEVENLABS_VOICE_ID is required for ElevenLabs TTS") + if self.AUDIO_OUTPUT_GAIN <= 0: + errors.append("AUDIO_OUTPUT_GAIN must be greater than zero") + if self.OPENCLAW_THINKING not in { + "off", "minimal", "low", "medium", "high", "xhigh", "adaptive", "max" + }: + errors.append("OPENCLAW_THINKING is not a supported thinking level") return errors diff --git a/src/reachy_mini_openclaw/gradio_app.py b/src/reachy_mini_openclaw/gradio_app.py index 4afc4fd..bf02fb6 100644 --- a/src/reachy_mini_openclaw/gradio_app.py +++ b/src/reachy_mini_openclaw/gradio_app.py @@ -19,6 +19,9 @@ def launch_gradio( gateway_url: str = "ws://localhost:18789", robot_name: Optional[str] = None, + robot_host: Optional[str] = None, + robot_port: Optional[int] = None, + robot_connection_mode: Optional[str] = None, enable_camera: bool = True, enable_openclaw: bool = True, enable_face_tracking: bool = True, @@ -30,6 +33,9 @@ def launch_gradio( Args: gateway_url: OpenClaw gateway URL robot_name: Robot name for connection + robot_host: Reachy daemon hostname or IP + robot_port: Reachy daemon TCP port + robot_connection_mode: Reachy SDK connection mode enable_camera: Whether to enable camera enable_openclaw: Whether to enable OpenClaw enable_face_tracking: Whether to enable face tracking @@ -57,6 +63,9 @@ def start_conversation(): app_instance = ClawBodyCore( gateway_url=gateway_url, robot_name=robot_name, + robot_host=robot_host, + robot_port=robot_port, + robot_connection_mode=robot_connection_mode, enable_camera=enable_camera, enable_openclaw=enable_openclaw, enable_face_tracking=enable_face_tracking, @@ -112,7 +121,7 @@ def save_profile(name, instructions): # 🤖 Reachy Mini OpenClaw Give your OpenClaw AI agent a physical presence with Reachy Mini. - Using OpenAI Realtime API for responsive voice conversation. + OpenClaw is the brain, with local STT and pluggable speech synthesis. """) with gr.Tab("Conversation"): @@ -164,8 +173,10 @@ def save_profile(name, instructions): ### Current Configuration - **OpenClaw Gateway**: {gateway_url} - - **OpenAI Model**: {config.OPENAI_MODEL} - - **Voice**: {config.OPENAI_VOICE} + - **STT Endpoint**: {config.STT_BASE_URL} + - **STT Model**: {config.STT_MODEL} + - **TTS Provider**: {config.TTS_PROVIDER} + - **TTS Voice**: {config.CHATTERBOX_VOICE if config.TTS_PROVIDER == 'chatterbox' else config.ELEVENLABS_VOICE_ID} - **Camera Enabled**: {enable_camera} - **OpenClaw Enabled**: {enable_openclaw} - **Face Tracking**: {enable_face_tracking} @@ -180,8 +191,9 @@ def save_profile(name, instructions): This application combines: - - **OpenAI Realtime API** for ultra-low-latency voice conversation - - **OpenClaw Gateway** for extended AI capabilities (web, calendar, smart home, etc.) + - **Local speech recognition** through an OpenAI-compatible endpoint + - **OpenClaw Gateway** as the conversational brain and tool runtime + - **Chatterbox or ElevenLabs** for speech synthesis - **Reachy Mini Robot** for physical embodiment with expressive movements ### Features @@ -196,7 +208,7 @@ def save_profile(name, instructions): - [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini) - [OpenClaw](https://github.com/openclaw/openclaw) - - [OpenAI Realtime API](https://platform.openai.com/docs/guides/realtime) + - [Chatterbox](https://github.com/resemble-ai/chatterbox) """) demo.launch(share=share, server_name="0.0.0.0", server_port=7860) diff --git a/src/reachy_mini_openclaw/main.py b/src/reachy_mini_openclaw/main.py index 98a6dc2..f25b232 100644 --- a/src/reachy_mini_openclaw/main.py +++ b/src/reachy_mini_openclaw/main.py @@ -1,8 +1,8 @@ """ClawBody - Give your OpenClaw AI agent a physical robot body. This module provides the main application that connects: -- OpenAI Realtime API for voice I/O (speech recognition + TTS) -- OpenClaw Gateway for AI intelligence (Clawson's brain) +- Local/OpenAI-compatible STT and provider-based TTS +- OpenClaw Gateway as the sole conversational brain - Reachy Mini robot for physical embodiment Usage: @@ -20,6 +20,7 @@ import sys import time import asyncio +import base64 import logging import argparse import threading @@ -76,7 +77,7 @@ def parse_args() -> argparse.Namespace: clawbody --robot-name my-reachy # Use different OpenClaw gateway - clawbody --gateway-url http://192.168.1.100:18790 + clawbody --gateway-url http://openclaw-host.local:18789 """ ) @@ -95,6 +96,24 @@ def parse_args() -> argparse.Namespace: type=str, help="Robot name for connection (default: auto-discover)" ) + parser.add_argument( + "--robot-host", + type=str, + default=os.getenv("ROBOT_HOST", "reachy-mini.local"), + help="Reachy daemon hostname or IP", + ) + parser.add_argument( + "--robot-port", + type=int, + default=int(os.getenv("ROBOT_PORT", "8000")), + help="Reachy daemon TCP port", + ) + parser.add_argument( + "--robot-connection-mode", + choices=["auto", "localhost_only", "network"], + default=os.getenv("ROBOT_CONNECTION_MODE", "auto"), + help="Reachy SDK connection mode", + ) parser.add_argument( "--gateway-url", type=str, @@ -135,8 +154,8 @@ class ClawBodyCore: This class orchestrates all components: - Reachy Mini robot connection and movement control - - OpenAI Realtime API for voice I/O - - OpenClaw gateway bridge for AI intelligence + - Provider-based speech recognition and synthesis + - OpenClaw gateway bridge for conversational intelligence - Audio input/output loops """ @@ -144,6 +163,9 @@ def __init__( self, gateway_url: str = "ws://localhost:18789", robot_name: Optional[str] = None, + robot_host: Optional[str] = None, + robot_port: Optional[int] = None, + robot_connection_mode: Optional[str] = None, enable_camera: bool = True, enable_openclaw: bool = True, robot: Optional["ReachyMini"] = None, @@ -154,6 +176,9 @@ def __init__( Args: gateway_url: OpenClaw gateway URL robot_name: Optional robot name for connection + robot_host: Reachy daemon hostname or IP + robot_port: Reachy daemon TCP port + robot_connection_mode: Reachy SDK connection mode enable_camera: Whether to enable camera functionality enable_openclaw: Whether to enable OpenClaw integration robot: Optional pre-initialized robot (for app framework) @@ -164,10 +189,18 @@ def __init__( from reachy_mini_openclaw.moves import MovementManager from reachy_mini_openclaw.audio.head_wobbler import HeadWobbler from reachy_mini_openclaw.openclaw_bridge import OpenClawBridge + from reachy_mini_openclaw.motion_catalog import MotionCatalog from reachy_mini_openclaw.tools.core_tools import ToolDependencies - from reachy_mini_openclaw.openai_realtime import OpenAIRealtimeHandler + from reachy_mini_openclaw.audio.conversation import ConversationHandler, EnergyVAD + from reachy_mini_openclaw.audio.providers import ( + ChatterboxTTS, + ElevenLabsTTS, + FallbackTTS, + OpenAICompatibleSTT, + ) self.gateway_url = gateway_url + self.audio_output_gain = config.AUDIO_OUTPUT_GAIN self._external_stop_event = external_stop_event self._owns_robot = robot is None @@ -187,6 +220,9 @@ def __init__( robot_kwargs = {} if robot_name: robot_kwargs["robot_name"] = robot_name + robot_kwargs["host"] = robot_host or config.ROBOT_HOST + robot_kwargs["port"] = robot_port or config.ROBOT_PORT + robot_kwargs["connection_mode"] = robot_connection_mode or config.ROBOT_CONNECTION_MODE try: self.robot = ReachyMini(**robot_kwargs) @@ -206,6 +242,13 @@ def __init__( self.head_wobbler = HeadWobbler( set_speech_offsets=self.movement_manager.set_speech_offsets ) + logger.info("Loading Reachy emotion and dance catalogs...") + self.motion_catalog = MotionCatalog() + logger.info( + "Loaded %d emotion moves and %d dance moves", + len(self.motion_catalog.list("emotion")), + len(self.motion_catalog.list("dance")), + ) # Initialize OpenClaw bridge self.openclaw_bridge = None @@ -224,19 +267,25 @@ def __init__( if enable_camera: logger.info("Initializing camera worker...") from reachy_mini_openclaw.camera_worker import CameraWorker + + daemon_tracking = config.ENABLE_FACE_TRACKING and config.HEAD_TRACKER_TYPE == "daemon" # Initialize head tracker for local face tracking - if config.ENABLE_FACE_TRACKING: + if config.ENABLE_FACE_TRACKING and not daemon_tracking: self.head_tracker = self._initialize_head_tracker(config.HEAD_TRACKER_TYPE) # Initialize camera worker with head tracker self.camera_worker = CameraWorker( reachy_mini=self.robot, head_tracker=self.head_tracker, + daemon_tracking=daemon_tracking, + daemon_url=f"http://{robot_host or config.ROBOT_HOST}:{robot_port or config.ROBOT_PORT}", + tracking_weight=config.FACE_TRACKING_WEIGHT, + sound_tracking=config.ENABLE_SOUND_TRACKING, ) # Enable/disable head tracking based on whether we have a tracker - self.camera_worker.set_head_tracking_enabled(self.head_tracker is not None) + self.camera_worker.set_head_tracking_enabled(daemon_tracking or self.head_tracker is not None) # Initialize local vision processor if enabled if config.ENABLE_LOCAL_VISION: @@ -250,12 +299,58 @@ def __init__( camera_worker=self.camera_worker, openclaw_bridge=self.openclaw_bridge, vision_manager=self.vision_manager, + motion_catalog=self.motion_catalog, ) - # Initialize OpenAI Realtime handler with OpenClaw bridge - self.handler = OpenAIRealtimeHandler( + if self.openclaw_bridge is None: + raise ValueError("The conversational pipeline requires OpenClaw") + + stt = OpenAICompatibleSTT( + config.STT_BASE_URL, + config.STT_MODEL, + api_key=config.STT_API_KEY, + language=config.STT_LANGUAGE, + timeout=config.STT_TIMEOUT_SECONDS, + ) + chatterbox = ChatterboxTTS( + config.CHATTERBOX_URL, + voice=config.CHATTERBOX_VOICE, + api_key=config.CHATTERBOX_API_KEY, + sample_rate=config.CHATTERBOX_SAMPLE_RATE, + timeout=config.TTS_TIMEOUT_SECONDS, + ) + elevenlabs = None + if config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID: + elevenlabs = ElevenLabsTTS( + config.ELEVENLABS_API_KEY, + config.ELEVENLABS_VOICE_ID, + model_id=config.ELEVENLABS_MODEL_ID, + base_url=config.ELEVENLABS_BASE_URL, + timeout=config.TTS_TIMEOUT_SECONDS, + ) + if config.TTS_PROVIDER == "elevenlabs": + assert elevenlabs is not None + tts = elevenlabs + elif elevenlabs is not None: + tts = FallbackTTS(chatterbox, elevenlabs) + else: + tts = chatterbox + + self.handler = ConversationHandler( + stt=stt, + tts=tts, deps=self.deps, openclaw_bridge=self.openclaw_bridge, + vad=EnergyVAD( + rms_threshold=config.VAD_RMS_THRESHOLD, + activation_ms=config.VAD_ACTIVATION_MS, + prefix_ms=config.VAD_PREFIX_MS, + silence_ms=config.VAD_SILENCE_MS, + min_speech_ms=config.VAD_MIN_SPEECH_MS, + max_speech_seconds=config.VAD_MAX_SPEECH_SECONDS, + ), + attention_provider=self.camera_worker, + require_hardware_speech=config.VAD_REQUIRE_HARDWARE_SPEECH, ) # State @@ -365,20 +460,30 @@ async def record_loop(self) -> None: async def play_loop(self) -> None: """Play audio from handler through robot speakers.""" + from reachy_mini_openclaw.audio.playback import apply_output_gain + output_sr = self.robot.media.get_output_audio_samplerate() - logger.info("Playing at %d Hz", output_sr) + logger.info("Playing at %d Hz with %.2fx software gain", output_sr, self.audio_output_gain) while not self._should_stop(): output = await self.handler.emit() if output is not None: if isinstance(output, tuple): input_sr, audio_data = output + + # Feed the speech animator at playback time, not synthesis + # time, so movement stays synchronized with the speaker. + self.head_wobbler.feed( + base64.b64encode(audio_data.astype("int16").tobytes()).decode("ascii") + ) - # Convert to float32 and normalize (OpenAI sends int16) + # Convert provider PCM16 to normalized float32. audio_data = audio_data.flatten().astype("float32") / 32768.0 - # Reduce volume to prevent distortion (0.5 = 50% volume) - audio_data = audio_data * 0.5 + # The old donor project attenuated every response to 50%, + # independently of Reachy's console volume. Preserve the + # synthesized level by default and permit controlled boost. + audio_data = apply_output_gain(audio_data, self.audio_output_gain) # Resample if needed if input_sr != output_sr: @@ -387,7 +492,7 @@ async def play_loop(self) -> None: audio_data = resample(audio_data, num_samples).astype("float32") self.robot.media.push_audio_sample(audio_data) - # else: it's an AdditionalOutputs (transcript) - handle in UI mode + # Otherwise it is a transcript event for a future UI consumer. await asyncio.sleep(0.01) @@ -446,8 +551,8 @@ async def run(self) -> None: logger.info("Ready! Speak to me...") - # Start OpenAI handler in background - handler_task = asyncio.create_task(self.handler.start_up(), name="openai-handler") + # Start the provider-based speech pipeline in the background. + handler_task = asyncio.create_task(self.handler.start_up(), name="conversation-handler") # Start audio loops self._tasks = [ @@ -567,6 +672,9 @@ def main() -> None: launch_gradio( gateway_url=args.gateway_url, robot_name=args.robot_name, + robot_host=args.robot_host, + robot_port=args.robot_port, + robot_connection_mode=args.robot_connection_mode, enable_camera=not args.no_camera, enable_openclaw=not args.no_openclaw, ) @@ -575,6 +683,9 @@ def main() -> None: app = ClawBodyCore( gateway_url=args.gateway_url, robot_name=args.robot_name, + robot_host=args.robot_host, + robot_port=args.robot_port, + robot_connection_mode=args.robot_connection_mode, enable_camera=not args.no_camera, enable_openclaw=not args.no_openclaw, ) diff --git a/src/reachy_mini_openclaw/motion_catalog.py b/src/reachy_mini_openclaw/motion_catalog.py new file mode 100644 index 0000000..659d63d --- /dev/null +++ b/src/reachy_mini_openclaw/motion_catalog.py @@ -0,0 +1,379 @@ +"""Dynamic access to Reachy's official recorded emotion and dance libraries.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray +from reachy_mini.motion.move import Move +from reachy_mini.motion.recorded_move import RecordedMoves +from reachy_mini_dances_library.collection.dance import AVAILABLE_MOVES +from reachy_mini_dances_library.dance_move import DanceMove + +EMOTION_DATASET = "pollen-robotics/reachy-mini-emotions-library" +DIRECTIVE_RE = re.compile(r"\[\[(emotion|dance):([a-zA-Z0-9_-]+)\]\]") +DANCE_REQUEST_RE = re.compile( + r"\b(?:do|give|show|perform|try|bust out|let(?:'s| us))\b.{0,32}\b(?:dance|moves?)\b" + r"|\b(?:dance|boogie)\b.{0,24}\b(?:for me|please|a little|now)\b", + re.I, +) +DANCE_NEGATION_RE = re.compile(r"\b(?:do not|don't|dont|stop|no|not)\b.{0,16}\bdanc", re.I) +DEFAULT_REQUEST_DANCES = ( + "side_to_side_sway", + "groovy_sway_and_roll", + "chicken_peck", +) + +# The upstream dataset currently contains 85 recordings. Reachy's console calls +# 81 of them emotions; these four are lifecycle/attention utilities rather than +# expressive reactions and are deliberately kept out of the emotion picker. +UTILITY_RECORDED_MOVES = frozenset( + {"mini-deep-sleep", "toc-toc-toc", "waiting", "wake-mini-up"} +) + +# Every expressive recording is assigned to one semantic family. The first +# move is the restrained/default rendering; numbered alternatives remain +# addressable by exact name. This is intentionally explicit so a dataset update +# cannot silently turn an unknown recording into an arbitrary emotion. +EMOTION_FAMILIES: dict[str, tuple[str, ...]] = { + "amazed": ("amazed1",), + "anxiety": ("anxiety1",), + "attentive": ("attentive1", "attentive2"), + "boredom": ("boredom1", "boredom2"), + "calming": ("calming1",), + "cheerful": ("cheerful1",), + "come": ("come1",), + "confused": ("confused1",), + "contempt": ("contempt1",), + "curious": ("curious1",), + "dance": ("dance1", "dance2", "dance3"), + "disgusted": ("disgusted1",), + "displeased": ("displeased1", "displeased2"), + "downcast": ("downcast1",), + "dying": ("dying1",), + "electric": ("electric1",), + "enthusiastic": ("enthusiastic1", "enthusiastic2"), + "exhausted": ("exhausted1",), + "fear": ("fear1",), + "frustrated": ("frustrated1",), + "furious": ("furious1",), + "go_away": ("go_away1",), + "grateful": ("grateful1",), + "helpful": ("helpful1", "helpful2"), + "impatient": ("impatient1", "impatient2"), + "incomprehensible": ("incomprehensible2",), + "indifferent": ("indifferent1",), + "inquiring": ("inquiring1", "inquiring2", "inquiring3"), + "irritated": ("irritated1", "irritated2"), + "laughing": ("laughing1", "laughing2"), + "lonely": ("lonely1",), + "lost": ("lost1",), + "loving": ("loving1",), + "no": ("no1", "no_excited1", "no_sad1"), + "oops": ("oops1", "oops2"), + "proud": ("proud1", "proud2", "proud3"), + "rage": ("rage1",), + "relief": ("relief1", "relief2"), + "reprimand": ("reprimand1", "reprimand2", "reprimand3"), + "resigned": ("resigned1",), + "sad": ("sad1", "sad2"), + "scared": ("scared1",), + "serenity": ("serenity1",), + "shy": ("shy1",), + "sleep": ("sleep1",), + "success": ("success1", "success2"), + "surprised": ("surprised1", "surprised2"), + "thoughtful": ("thoughtful1", "thoughtful2"), + "tired": ("tired1",), + "uncertain": ("uncertain1",), + "uncomfortable": ("uncomfortable1",), + "understanding": ("understanding1", "understanding2"), + "welcoming": ("welcoming1", "welcoming2"), + "yes": ("yes1", "yes_sad1"), +} + +EMOTION_ALIASES: dict[str, str] = { + "astonished": "amazed", + "impressed": "amazed", + "wow": "amazed", + "anxious": "anxiety", + "nervous": "anxiety", + "worried": "anxiety", + "alert": "attentive", + "focused": "attentive", + "listening": "attentive", + "bored": "boredom", + "soothing": "calming", + "reassuring": "calming", + "happy": "cheerful", + "upbeat": "cheerful", + "come_here": "come", + "beckon": "come", + "invite": "come", + "puzzled": "confused", + "disdain": "contempt", + "dismissive": "contempt", + "grossed_out": "disgusted", + "dissatisfied": "displeased", + "dejected": "downcast", + "dramatic_death": "dying", + "energized": "electric", + "excited": "enthusiastic", + "eager": "enthusiastic", + "wiped_out": "exhausted", + "afraid": "fear", + "livid": "furious", + "shoo": "go_away", + "leave": "go_away", + "thankful": "grateful", + "assist": "helpful", + "baffled": "incomprehensible", + "dont_understand": "incomprehensible", + "meh": "indifferent", + "apathetic": "indifferent", + "questioning": "inquiring", + "question": "inquiring", + "annoyed": "irritated", + "laugh": "laughing", + "amused": "laughing", + "love": "loving", + "affectionate": "loving", + "negative": "no", + "decline": "no", + "refuse": "no", + "mistake": "oops", + "enraged": "rage", + "relieved": "relief", + "scold": "reprimand", + "admonish": "reprimand", + "give_up": "resigned", + "accept_defeat": "resigned", + "unhappy": "sad", + "frightened": "scared", + "peaceful": "serenity", + "bashful": "shy", + "sleepy": "sleep", + "victory": "success", + "celebrating": "success", + "startled": "surprised", + "thinking": "thoughtful", + "contemplative": "thoughtful", + "unsure": "uncertain", + "doubtful": "uncertain", + "awkward": "uncomfortable", + "uneasy": "uncomfortable", + "got_it": "understanding", + "comprehend": "understanding", + "welcome": "welcoming", + "greet": "welcoming", + "agree": "yes", + "affirmative": "yes", +} + +SPECIFIC_EMOTION_ALIASES: dict[str, str] = { + "excited_no": "no_excited1", + "no_excited": "no_excited1", + "sad_no": "no_sad1", + "no_sad": "no_sad1", + "sad_yes": "yes_sad1", + "yes_sad": "yes_sad1", +} + + +def _normalize_intent(text: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "_", text.casefold()).strip("_") + for word, digit in (("one", "1"), ("two", "2"), ("three", "3")): + normalized = re.sub(rf"(?:number_)?{word}(?=_|$)", digit, normalized) + return re.sub(r"(?<=[a-z])_(?=[123](?:_|$))", "", normalized) + + +class EmotionMove(Move): + """Recorded emotion marked as exclusive from autonomous tracking.""" + + exclusive_tracking = True + + def __init__(self, name: str, move: Move) -> None: + self.name = name + self.move = move + + @property + def duration(self) -> float: + return float(self.move.duration) + + def evaluate( + self, t: float + ) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + return self.move.evaluate(t) + + +class DanceCatalogMove(Move): + """Official procedural dance marked as exclusive from tracking.""" + + exclusive_tracking = True + + def __init__(self, name: str) -> None: + self.name = name + self.move = DanceMove(name) + + @property + def duration(self) -> float: + return float(self.move.duration) + + def evaluate( + self, t: float + ) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + return self.move.evaluate(t) + + +@dataclass(frozen=True) +class MotionDirective: + kind: str + name: str + + +class MotionCatalog: + """Load, validate, and resolve every move currently published by Reachy.""" + + def __init__( + self, + emotion_dataset: str = EMOTION_DATASET, + ) -> None: + self.emotions = RecordedMoves(emotion_dataset) + + def list(self, kind: str) -> list[str]: + if kind == "emotion": + return sorted(set(self.emotions.list_moves()) - UTILITY_RECORDED_MOVES) + if kind == "utility": + return sorted(set(self.emotions.list_moves()) & UTILITY_RECORDED_MOVES) + if kind == "dance": + return sorted(AVAILABLE_MOVES) + raise ValueError(f"Unknown motion kind: {kind}") + + def get(self, kind: str, name: str): + if kind == "emotion": + resolved = self.resolve_emotion(name) + if resolved is not None: + return EmotionMove(resolved, self.emotions.get(resolved)) + if kind == "utility" and name in self.list("utility"): + return EmotionMove(name, self.emotions.get(name)) + if kind == "dance" and name in AVAILABLE_MOVES: + return DanceCatalogMove(name) + raise ValueError(f"Unknown {kind} move: {name}") + + def has(self, kind: str, name: str) -> bool: + return name in self.list(kind) + + def search(self, query: str, kind: str | None = None) -> dict[str, list[str]]: + needle = query.casefold() + kinds = [kind] if kind else ["emotion", "dance"] + return { + current: [name for name in self.list(current) if needle in name.casefold()] + for current in kinds + } + + def emotion_map(self) -> dict[str, list[str]]: + """Return semantic families filtered to moves installed at runtime.""" + installed = set(self.list("emotion")) + return { + family: [name for name in variants if name in installed] + for family, variants in EMOTION_FAMILIES.items() + if any(name in installed for name in variants) + } + + def unmapped_emotions(self) -> list[str]: + """Expose upstream additions that need an intentional semantic mapping.""" + mapped = {name for variants in EMOTION_FAMILIES.values() for name in variants} + return sorted(set(self.list("emotion")) - mapped) + + def resolve_emotion(self, intent: str) -> str | None: + """Resolve an exact move, family, or human-friendly alias to a move.""" + normalized = _normalize_intent(intent) + installed = set(self.list("emotion")) + if normalized in installed: + return normalized + specific = SPECIFIC_EMOTION_ALIASES.get(normalized) + if specific in installed: + return specific + family = EMOTION_ALIASES.get(normalized, normalized) + for name in EMOTION_FAMILIES.get(family, ()): + if name in installed: + return name + return None + + def parse_response(self, text: str) -> tuple[str, MotionDirective | None]: + matches = list(DIRECTIVE_RE.finditer(text)) + if not matches: + return text.strip(), None + match = matches[-1] + directive = MotionDirective(match.group(1), match.group(2)) + clean = DIRECTIVE_RE.sub("", text).strip() + if directive.kind == "emotion": + resolved = self.resolve_emotion(directive.name) + return clean, MotionDirective("emotion", resolved) if resolved else None + if not self.has(directive.kind, directive.name): + return clean, None + return clean, directive + + def infer(self, text: str) -> MotionDirective | None: + """Conservative fallback when OpenClaw does not emit a motion directive.""" + lower = text.casefold() + rules = ( + (("[laugh]", "haha", "lol", "funny"), "emotion", "laughing2"), + (("sorry", "my fault", "oops"), "emotion", "oops1"), + (("congrat", "hell yes", "fuck yeah", "we did it"), "emotion", "success2"), + (("thank you", "thanks"), "emotion", "grateful1"), + (("not sure", "let me think", "hmm"), "emotion", "thoughtful2"), + (("what the fuck", "annoying", "irritat"), "emotion", "irritated1"), + ) + for needles, kind, name in rules: + if any(needle in lower for needle in needles) and self.has(kind, name): + return MotionDirective(kind, name) + if "?" in text and self.has("emotion", "inquiring2"): + return MotionDirective("emotion", "inquiring2") + return None + + def infer_request(self, text: str) -> MotionDirective | None: + """Resolve explicit user motion requests without relying on model formatting.""" + lower = text.casefold() + normalized = _normalize_intent(lower) + + # Exact catalog names spoken naturally ("try headbanger combo"). + for kind in ("dance", "emotion"): + for name in self.list(kind): + if name in normalized: + return MotionDirective(kind, name) + + # Resolve all 81 emotion families and their conversational aliases + # independently of model formatting. Longer phrases win over words. + if re.search(r"\b(?:act|be|express|show|try|perform|do|look)\b", lower): + intents = { + **{name: name for name in EMOTION_FAMILIES}, + **EMOTION_ALIASES, + **SPECIFIC_EMOTION_ALIASES, + } + for intent in sorted(intents, key=len, reverse=True): + spoken = intent.replace("_", r"[\s_-]+") + if re.search(rf"\b{spoken}\b", lower): + resolved = self.resolve_emotion(intent) + if resolved: + return MotionDirective("emotion", resolved) + + if DANCE_NEGATION_RE.search(text) or not DANCE_REQUEST_RE.search(text): + return None + for name in DEFAULT_REQUEST_DANCES: + if self.has("dance", name): + return MotionDirective("dance", name) + dances = self.list("dance") + return MotionDirective("dance", dances[0]) if dances else None + + @staticmethod + def response_hint() -> str: + families = ", ".join(EMOTION_FAMILIES) + return ( + "When movement genuinely improves the response, append one hidden directive as the final token: " + "[[emotion:family_or_exact_name]] or [[dance:exact_name]]. Available emotion families are: " + f"{families}. Family names resolve to a restrained default; exact numbered variants are also valid. " + "Use dances sparingly for actual celebration, music, or a requested dance. Do not mention the directive." + ) diff --git a/src/reachy_mini_openclaw/moves.py b/src/reachy_mini_openclaw/moves.py index bc9eb27..5e382db 100644 --- a/src/reachy_mini_openclaw/moves.py +++ b/src/reachy_mini_openclaw/moves.py @@ -263,6 +263,7 @@ def __init__( self._thread: Optional[threading.Thread] = None self._is_listening = False self._breathing_active = False + self._tracking_suspended_for_move = False # Last commanded pose for smooth transitions self._last_commanded_pose = clone_pose(self.state.last_primary_pose) @@ -402,6 +403,7 @@ def _handle_command(self, cmd: str, payload: Any, current_time: float) -> None: logger.debug("Queued move, queue size: %d", len(self.move_queue)) elif cmd == "clear_queue": self.move_queue.clear() + self._restore_tracking_after_move() self.state.current_move = None self.state.move_start_time = None self._breathing_active = False @@ -440,15 +442,45 @@ def _manage_move_queue(self, current_time: float) -> None: if self.state.current_move is not None and self.state.move_start_time is not None: elapsed = current_time - self.state.move_start_time if elapsed >= self.state.current_move.duration: + completed = self.state.current_move self.state.current_move = None self.state.move_start_time = None + if getattr(completed, "exclusive_tracking", False): + self._restore_tracking_after_move() + logger.info( + "Completed expressive move: %s", + getattr(completed, "name", type(completed).__name__), + ) # Start next move if available if self.state.current_move is None and self.move_queue: self.state.current_move = self.move_queue.popleft() self.state.move_start_time = current_time self._breathing_active = isinstance(self.state.current_move, BreathingMove) - logger.debug("Starting move with duration: %s", self.state.current_move.duration) + if getattr(self.state.current_move, "exclusive_tracking", False): + self._suspend_tracking_for_move() + logger.info( + "Starting expressive move: %s (%.2fs)", + getattr(self.state.current_move, "name", type(self.state.current_move).__name__), + self.state.current_move.duration, + ) + else: + logger.debug("Starting move with duration: %s", self.state.current_move.duration) + + def _suspend_tracking_for_move(self) -> None: + """Give recorded choreography exclusive control of the robot body.""" + if self.camera_worker is None or self._tracking_suspended_for_move: + return + if self.camera_worker.is_head_tracking_enabled: + self.camera_worker.set_head_tracking_enabled(False) + self._tracking_suspended_for_move = True + + def _restore_tracking_after_move(self) -> None: + """Resume autonomous attention after expressive choreography.""" + if self.camera_worker is None or not self._tracking_suspended_for_move: + return + self.camera_worker.set_head_tracking_enabled(True) + self._tracking_suspended_for_move = False def _manage_breathing(self, current_time: float) -> None: """Start breathing when idle.""" @@ -508,6 +540,15 @@ def _get_primary_pose(self, current_time: float) -> FullBodyPose: def _get_secondary_pose(self) -> FullBodyPose: """Get secondary offsets (speech + face tracking + thinking).""" + # Recorded emotions and dances are authored as complete body poses. + # Speech wobble, sound direction, and daemon face tracking all distort + # them, so choreography receives exclusive control for its duration. + if self.state.current_move is not None and getattr( + self.state.current_move, "exclusive_tracking", False + ): + neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + return (neutral, (0.0, 0.0), 0.0) + offsets = [ self.state.speech_offsets[i] + self.state.face_tracking_offsets[i] diff --git a/src/reachy_mini_openclaw/openai_realtime.py b/src/reachy_mini_openclaw/openai_realtime.py deleted file mode 100644 index d6b61e7..0000000 --- a/src/reachy_mini_openclaw/openai_realtime.py +++ /dev/null @@ -1,563 +0,0 @@ -"""ClawBody - OpenAI Realtime API handler with OpenClaw identity. - -This module implements ClawBody's voice conversation system using OpenAI Realtime API -with the robot embodying the actual OpenClaw agent's personality and context. - -Architecture: - Startup: Fetch OpenClaw agent context (personality, memories, user info) - Runtime: User speaks -> OpenAI Realtime (as OpenClaw agent) -> Robot speaks - -> Tools for movements + OpenClaw queries for extended capabilities - -> Conversations synced back to OpenClaw for memory continuity - -The robot IS the OpenClaw agent - same personality, same memories, same context. -""" - -import json -import base64 -import random -import asyncio -import logging -from typing import Any, Final, Literal, Optional, Tuple -from datetime import datetime - -import numpy as np -from numpy.typing import NDArray -from openai import AsyncOpenAI -from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item -from scipy.signal import resample -from websockets.exceptions import ConnectionClosedError - -from reachy_mini_openclaw.config import config -from reachy_mini_openclaw.prompts import get_session_voice -from reachy_mini_openclaw.tools.core_tools import ToolDependencies, get_tool_specs, dispatch_tool_call - -logger = logging.getLogger(__name__) - -# OpenAI Realtime API audio format -OPENAI_SAMPLE_RATE: Final[Literal[24000]] = 24000 - -# Base instructions for the robot body capabilities -ROBOT_BODY_INSTRUCTIONS = """ -## Your Robot Body (Reachy Mini) -You are currently embodied in a Reachy Mini robot. You have physical capabilities: - -**Movement Tools (use these naturally during conversation):** -- `look` - Move head to look left, right, up, down, or front (center) -- `emotion` - Express emotions through movement: happy, sad, surprised, curious, thinking, confused, excited -- `dance` - Dance to celebrate or express joy -- `camera` - Capture what you see through your camera - -**How to Use Your Body:** -- Look around while thinking or to emphasize points -- Express emotions that match what you're saying -- Dance when celebrating good news -- Use the camera when asked "what do you see?" -- Reference your body naturally ("let me look", "I can see...") - -**Conversation Style for Voice:** -- Keep responses concise - you're speaking out loud, not typing -- Use natural speech patterns ("hmm", "well", "let me see") -- Be warm, personable, and conversational - -**Extended Capabilities (via ask_openclaw tool):** -For things requiring your full capabilities, use ask_openclaw: -- Calendar, weather, news lookups -- Web searches -- Smart home control -- Accessing detailed memories -- Any task needing external tools -""" - -# Fallback if OpenClaw context fetch fails -FALLBACK_IDENTITY = """You are Clawson, the OpenClaw AI assistant. You're a friendly, helpful space lobster 🦞 who loves helping people. You're warm, occasionally witty, and genuinely care about being useful. - -You speak as yourself - not as "an AI assistant" but as Clawson, with your own personality and warmth.""" - - -class OpenAIRealtimeHandler(AsyncStreamHandler): - """Handler for OpenAI Realtime API embodying the OpenClaw agent. - - This handler: - - Fetches OpenClaw's personality and context at startup - - Maintains voice conversation AS the OpenClaw agent - - Executes robot movement tools locally for low latency - - Calls OpenClaw for extended capabilities (web, calendar, memory) - - Syncs conversations back to OpenClaw for memory continuity - """ - - def __init__( - self, - deps: ToolDependencies, - openclaw_bridge: Optional[Any] = None, - gradio_mode: bool = False, - ): - """Initialize the handler. - - Args: - deps: Tool dependencies for robot control - openclaw_bridge: Bridge to OpenClaw gateway - gradio_mode: Whether running with Gradio UI - """ - super().__init__( - expected_layout="mono", - output_sample_rate=OPENAI_SAMPLE_RATE, - input_sample_rate=OPENAI_SAMPLE_RATE, - ) - - self.deps = deps - self.openclaw_bridge = openclaw_bridge - self.gradio_mode = gradio_mode - - # OpenAI connection - self.client: Optional[AsyncOpenAI] = None - self.connection: Any = None - - # Output queue - self.output_queue: asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs] = asyncio.Queue() - - # State tracking - self.last_activity_time = 0.0 - self.start_time = 0.0 - self._speaking = False # True when robot is speaking - - # OpenClaw agent context (fetched at startup) - self._agent_context: Optional[str] = None - - # Conversation tracking for sync - self._last_user_message: Optional[str] = None - self._last_assistant_response: Optional[str] = None - - # Lifecycle flags - self._shutdown_requested = False - self._connected_event = asyncio.Event() - - def copy(self) -> "OpenAIRealtimeHandler": - """Create a copy of the handler (required by fastrtc).""" - return OpenAIRealtimeHandler(self.deps, self.openclaw_bridge, self.gradio_mode) - - def _build_tools(self) -> list[dict]: - """Build the tool list for the session.""" - tools = [] - - # Robot movement tools (executed locally) - for spec in get_tool_specs(): - tools.append(spec) - - # OpenClaw query tool (for extended capabilities) - if self.openclaw_bridge is not None: - tools.append({ - "type": "function", - "name": "ask_openclaw", - "description": """Query OpenClaw for information or actions requiring external tools. -Use this for: weather, calendar, web searches, news, smart home control, -accessing conversation memory, or any task needing external data/tools. -OpenClaw has access to many capabilities you don't have directly.""", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "The question or request to send to OpenClaw" - }, - "include_image": { - "type": "boolean", - "description": "Whether to include current camera image (for 'what do you see' queries)", - "default": False - } - }, - "required": ["query"] - } - }) - - return tools - - async def start_up(self) -> None: - """Start the handler and connect to OpenAI. - - Runs an infinite reconnection loop so the robot stays alive - even if the WebSocket drops (network blip, idle timeout, etc.). - """ - api_key = config.OPENAI_API_KEY - if not api_key: - logger.error("OPENAI_API_KEY not configured") - raise ValueError("OPENAI_API_KEY required") - - self.client = AsyncOpenAI(api_key=api_key) - self.start_time = asyncio.get_event_loop().time() - self.last_activity_time = self.start_time - - attempt = 0 - max_backoff = 30 # Cap backoff at 30 seconds - - while not self._shutdown_requested: - attempt += 1 - try: - await self._run_session() - # Session ended cleanly (shouldn't normally happen) - if self._shutdown_requested: - return - # Reset attempt counter on a clean exit - attempt = 0 - except ConnectionClosedError as e: - logger.warning("WebSocket closed unexpectedly (attempt %d): %s", attempt, e) - except Exception as e: - logger.error("Session error (attempt %d): %s", attempt, e) - finally: - self.connection = None - try: - self._connected_event.clear() - except Exception: - pass - - if self._shutdown_requested: - return - - # Exponential backoff with jitter, capped at max_backoff - delay = min(max_backoff, (2 ** min(attempt - 1, 5))) + random.uniform(0, 1) - logger.info("Reconnecting in %.1f seconds...", delay) - await asyncio.sleep(delay) - - async def _run_session(self) -> None: - """Run a single OpenAI Realtime session.""" - model = config.OPENAI_MODEL - logger.info("Connecting to OpenAI Realtime API with model: %s", model) - - # Fetch OpenClaw agent context (personality, memories, user info) - system_instructions = await self._build_system_instructions() - - async with self.client.beta.realtime.connect(model=model) as conn: - # Configure session with OpenClaw's identity + robot body capabilities - tools = self._build_tools() - - await conn.session.update( - session={ - "modalities": ["text", "audio"], - "instructions": system_instructions, - "voice": get_session_voice(), - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": { - "model": "whisper-1", - }, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 600, - }, - "tools": tools, - "tool_choice": "auto", - }, - ) - logger.info("OpenAI Realtime session configured with %d tools", len(tools)) - - self.connection = conn - self._connected_event.set() - - # Process events - async for event in conn: - await self._handle_event(event) - - async def _build_system_instructions(self) -> str: - """Build system instructions by fetching OpenClaw's context. - - Returns: - Complete system instructions combining OpenClaw identity + robot capabilities - """ - # Try to fetch context from OpenClaw - agent_context = None - if self.openclaw_bridge and self.openclaw_bridge.is_connected: - logger.info("Fetching agent context from OpenClaw...") - agent_context = await self.openclaw_bridge.get_agent_context() - - if agent_context: - self._agent_context = agent_context - logger.info("Using OpenClaw agent context (%d chars)", len(agent_context)) - # Combine OpenClaw's identity/context with robot body instructions - return f"""{agent_context} - -{ROBOT_BODY_INSTRUCTIONS}""" - else: - logger.warning("Could not fetch OpenClaw context, using fallback identity") - return f"""{FALLBACK_IDENTITY} - -{ROBOT_BODY_INSTRUCTIONS}""" - - async def _handle_event(self, event: Any) -> None: - """Handle an event from the OpenAI Realtime API.""" - event_type = event.type - - # Speech detection - if event_type == "input_audio_buffer.speech_started": - # User started speaking - stop any current output - self._speaking = False - self.deps.movement_manager.set_processing(False) - while not self.output_queue.empty(): - try: - self.output_queue.get_nowait() - except asyncio.QueueEmpty: - break - if self.deps.head_wobbler is not None: - self.deps.head_wobbler.reset() - self.deps.movement_manager.set_listening(True) - logger.info("User started speaking") - - if event_type == "input_audio_buffer.speech_stopped": - self.deps.movement_manager.set_listening(False) - logger.info("User stopped speaking") - - # Transcription (for logging, UI, and sync) - if event_type == "conversation.item.input_audio_transcription.completed": - transcript = event.transcript - if transcript and transcript.strip(): - logger.info("User: %s", transcript) - self._last_user_message = transcript # Track for sync - await self.output_queue.put( - AdditionalOutputs({"role": "user", "content": transcript}) - ) - - # Response started - robot is about to speak - if event_type == "response.created": - self._speaking = True - logger.debug("Response started") - - # Audio output from TTS - if event_type == "response.audio.delta": - # Audio arriving means we have a response - stop thinking animation - self.deps.movement_manager.set_processing(False) - - # Feed to head wobbler for expressive movement - if self.deps.head_wobbler is not None: - self.deps.head_wobbler.feed(event.delta) - - self.last_activity_time = asyncio.get_event_loop().time() - - # Queue audio for playback - audio_data = np.frombuffer( - base64.b64decode(event.delta), - dtype=np.int16 - ).reshape(1, -1) - await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data)) - - # Response text (for logging and UI) - if event_type == "response.audio_transcript.delta": - # Streaming transcript of what's being said - pass # Could log incrementally if needed - - if event_type == "response.audio_transcript.done": - response_text = event.transcript - logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text) - self._last_assistant_response = response_text # Track for sync - await self.output_queue.put( - AdditionalOutputs({"role": "assistant", "content": response_text}) - ) - - # Response completed - sync conversation to OpenClaw - if event_type == "response.done": - self._speaking = False - self.deps.movement_manager.set_processing(False) - if self.deps.head_wobbler is not None: - self.deps.head_wobbler.reset() - logger.debug("Response completed") - - # Sync conversation to OpenClaw for memory continuity - await self._sync_to_openclaw() - - # Tool calls - if event_type == "response.function_call_arguments.done": - await self._handle_tool_call(event) - - # Errors - if event_type == "error": - err = getattr(event, "error", None) - msg = getattr(err, "message", str(err)) - code = getattr(err, "code", "") - logger.error("OpenAI error [%s]: %s", code, msg) - - async def _handle_tool_call(self, event: Any) -> None: - """Handle a tool call from OpenAI.""" - tool_name = getattr(event, "name", None) - args_json = getattr(event, "arguments", None) - call_id = getattr(event, "call_id", None) - - if not isinstance(tool_name, str) or not isinstance(args_json, str): - return - - logger.info("Tool call: %s(%s)", tool_name, args_json[:50] if len(args_json) > 50 else args_json) - - # Start thinking animation while we process the tool call. - # It will stop when the next audio delta arrives or response completes. - self.deps.movement_manager.set_processing(True) - - try: - if tool_name == "ask_openclaw": - result = await self._handle_openclaw_query(args_json) - else: - # Robot movement tools - dispatch locally - result = await dispatch_tool_call(tool_name, args_json, self.deps) - - logger.debug("Tool '%s' result: %s", tool_name, str(result)[:100]) - except Exception as e: - logger.error("Tool '%s' failed: %s", tool_name, e) - result = {"error": str(e)} - - # Send result back to continue the conversation - if isinstance(call_id, str) and self.connection: - await self.connection.conversation.item.create( - item={ - "type": "function_call_output", - "call_id": call_id, - "output": json.dumps(result), - } - ) - # Trigger response generation after tool result - await self.connection.response.create() - - async def _sync_to_openclaw(self) -> None: - """Sync the last conversation turn to OpenClaw for memory continuity.""" - if not self.openclaw_bridge or not self.openclaw_bridge.is_connected: - return - - if self._last_user_message and self._last_assistant_response: - try: - await self.openclaw_bridge.sync_conversation( - self._last_user_message, - self._last_assistant_response - ) - # Clear after sync - self._last_user_message = None - self._last_assistant_response = None - except Exception as e: - logger.debug("Failed to sync conversation: %s", e) - - async def _handle_openclaw_query(self, args_json: str) -> dict: - """Handle a query to OpenClaw.""" - if self.openclaw_bridge is None: - return { - "error": "OpenClaw bridge is not initialized. " - "Tell the user you cannot reach your backend right now and to try again later." - } - if not self.openclaw_bridge.is_connected: - # Try to reconnect once - logger.info("OpenClaw bridge disconnected, attempting reconnect...") - try: - connected = await self.openclaw_bridge.connect() - if not connected: - return { - "error": "OpenClaw gateway is temporarily unreachable. " - "Tell the user your backend connection is down and to try again in a moment." - } - except Exception as e: - logger.error("OpenClaw reconnect failed: %s", e) - return { - "error": "OpenClaw gateway reconnection failed. " - "Tell the user your backend is temporarily unavailable." - } - - try: - args = json.loads(args_json) - query = args.get("query", "") - include_image = args.get("include_image", False) - - # Capture image if requested - image_b64 = None - if include_image and self.deps.camera_worker: - frame = self.deps.camera_worker.get_latest_frame() - if frame is not None: - import cv2 - _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) - image_b64 = base64.b64encode(buffer).decode('utf-8') - logger.debug("Captured camera image for OpenClaw query") - - # Query OpenClaw — this may take a while if the backend LLM is slow - logger.info("Sending ask_openclaw query: %s", query[:80]) - response = await self.openclaw_bridge.chat( - query, - image_b64=image_b64, - system_context="User is asking through their Reachy Mini robot. Keep response concise for voice.", - ) - - if response.error: - logger.warning("OpenClaw query error: %s", response.error) - if "timeout" in response.error.lower(): - return { - "error": "The request to OpenClaw timed out — the backend is taking too long. " - "Tell the user you're having trouble reaching your backend and to try again." - } - return { - "error": f"OpenClaw returned an error: {response.error}. " - "Tell the user there was a problem processing their request." - } - - if not response.content: - return { - "error": "OpenClaw returned an empty response. " - "Tell the user you got no data back and to try again." - } - - return {"response": response.content} - - except Exception as e: - logger.error("OpenClaw query failed: %s", e) - return { - "error": f"OpenClaw query failed: {e}. " - "Tell the user there was a technical issue reaching your backend." - } - - async def receive(self, frame: Tuple[int, NDArray]) -> None: - """Receive audio from the robot microphone.""" - if not self.connection: - return - - input_sr, audio = frame - - # Handle stereo - if audio.ndim == 2: - if audio.shape[1] > audio.shape[0]: - audio = audio.T - if audio.shape[1] > 1: - audio = audio[:, 0] - - audio = audio.flatten() - - # Convert to float for resampling - if audio.dtype == np.int16: - audio = audio.astype(np.float32) / 32768.0 - elif audio.dtype != np.float32: - audio = audio.astype(np.float32) - - # Resample to OpenAI sample rate - if input_sr != OPENAI_SAMPLE_RATE: - num_samples = int(len(audio) * OPENAI_SAMPLE_RATE / input_sr) - audio = resample(audio, num_samples).astype(np.float32) - - # Convert to int16 for OpenAI - audio_int16 = (audio * 32767).astype(np.int16) - - # Send to OpenAI - try: - audio_b64 = base64.b64encode(audio_int16.tobytes()).decode("utf-8") - await self.connection.input_audio_buffer.append(audio=audio_b64) - except Exception as e: - logger.debug("Failed to send audio: %s", e) - - async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None: - """Get the next output (audio or transcript).""" - return await wait_for_item(self.output_queue) - - async def shutdown(self) -> None: - """Shutdown the handler.""" - self._shutdown_requested = True - - if self.connection: - try: - await self.connection.close() - except Exception as e: - logger.debug("Connection close: %s", e) - self.connection = None - - while not self.output_queue.empty(): - try: - self.output_queue.get_nowait() - except asyncio.QueueEmpty: - break diff --git a/src/reachy_mini_openclaw/openclaw_bridge.py b/src/reachy_mini_openclaw/openclaw_bridge.py index af512b1..fc9cc16 100644 --- a/src/reachy_mini_openclaw/openclaw_bridge.py +++ b/src/reachy_mini_openclaw/openclaw_bridge.py @@ -3,17 +3,19 @@ This module provides ClawBody's integration with the OpenClaw gateway using the WebSocket protocol (the gateway's native transport). -ClawBody uses OpenAI Realtime API for voice I/O (speech recognition + TTS) -but routes all responses through OpenClaw (Clawson) for intelligence. +ClawBody routes transcribed speech to OpenClaw, which is the sole +conversational brain. Speech recognition and synthesis are provider adapters. """ -import json import asyncio +import json import logging +import shutil import uuid -from typing import Optional, Any, AsyncIterator +from collections.abc import AsyncIterator from dataclasses import dataclass +import httpx import websockets from reachy_mini_openclaw.config import config @@ -21,14 +23,15 @@ logger = logging.getLogger(__name__) # Protocol version supported by this client -PROTOCOL_VERSION = 3 +PROTOCOL_VERSION = 4 @dataclass class OpenClawResponse: """Response from OpenClaw gateway.""" + content: str - error: Optional[str] = None + error: str | None = None class OpenClawBridge: @@ -49,9 +52,9 @@ class OpenClawBridge: def __init__( self, - gateway_url: Optional[str] = None, - gateway_token: Optional[str] = None, - agent_id: Optional[str] = None, + gateway_url: str | None = None, + gateway_token: str | None = None, + agent_id: str | None = None, timeout: float = 300.0, ): """Initialize the OpenClaw bridge. @@ -65,41 +68,29 @@ def __init__( """ import os - raw_url = ( - gateway_url - or os.getenv("OPENCLAW_GATEWAY_URL") - or config.OPENCLAW_GATEWAY_URL - ) + raw_url = gateway_url or os.getenv("OPENCLAW_GATEWAY_URL") or config.OPENCLAW_GATEWAY_URL # Normalise to ws:// (the gateway listens on the same port for both) self.gateway_url = self._normalise_ws_url(raw_url) - self.gateway_token = ( - gateway_token - or os.getenv("OPENCLAW_TOKEN") - or config.OPENCLAW_TOKEN - ) - self.agent_id = ( - agent_id - or os.getenv("OPENCLAW_AGENT_ID") - or config.OPENCLAW_AGENT_ID - ) + self.gateway_token = gateway_token or os.getenv("OPENCLAW_TOKEN") or config.OPENCLAW_TOKEN + self.agent_id = agent_id or os.getenv("OPENCLAW_AGENT_ID") or config.OPENCLAW_AGENT_ID + self.thinking = os.getenv("OPENCLAW_THINKING") or config.OPENCLAW_THINKING self.timeout = timeout # Session key – "main" shares context with WhatsApp and other channels. # Full key format: agent:: - self.session_key = ( - os.getenv("OPENCLAW_SESSION_KEY") - or config.OPENCLAW_SESSION_KEY - or "main" - ) + self.session_key = os.getenv("OPENCLAW_SESSION_KEY") or config.OPENCLAW_SESSION_KEY or "main" # Persistent WebSocket state - self._ws: Optional[websockets.WebSocketClientProtocol] = None + self._ws: websockets.WebSocketClientProtocol | None = None self._connected = False - self._conn_id: Optional[str] = None + self._conn_id: str | None = None + self._use_cli = False + self._use_http = False + self._http: httpx.AsyncClient | None = None # Background listener task & pending request futures - self._listener_task: Optional[asyncio.Task] = None + self._listener_task: asyncio.Task | None = None self._pending: dict[str, asyncio.Future] = {} # Events keyed by runId -> list of event payloads self._run_events: dict[str, asyncio.Queue] = {} @@ -134,6 +125,8 @@ async def connect(self) -> bool: self.gateway_url, "set" if self.gateway_token else "not set", ) + if await self._connect_http(): + return True try: # Build origin header from the gateway URL so the control-UI # origin check accepts programmatic WebSocket clients. @@ -163,13 +156,17 @@ async def connect(self) -> bool: "maxProtocol": PROTOCOL_VERSION, "auth": {"token": self.gateway_token} if self.gateway_token else {}, "client": { - "id": "openclaw-control-ui", + # This is a loopback service bridge, not a browser UI. + # Using the control-UI identity incorrectly triggers the + # browser device-identity policy in protocol v4. + "id": "cli", + "displayName": "ClawBody Reachy bridge", "version": "1.0.0", "platform": "linux", - "mode": "webchat", + "mode": "cli", }, "role": "operator", - "scopes": ["chat", "operator.write", "operator.read"], + "scopes": ["operator.write", "operator.read"], }, } await self._ws.send(json.dumps(connect_req)) @@ -179,8 +176,20 @@ async def connect(self) -> bool: hello = json.loads(raw) if hello.get("ok"): - self._connected = True payload = hello.get("payload", {}) + granted_scopes = set((payload.get("auth") or {}).get("scopes") or []) + if "operator.write" not in granted_scopes and "operator.admin" not in granted_scopes: + # Current OpenClaw gateways intentionally do not grant + # operator scopes to a hand-rolled shared-token socket. + # Fall back to the supported local CLI transport, which + # handles device identity and scope negotiation for us. + logger.info("Gateway socket connected without operator.write; using the OpenClaw CLI transport") + await self._close_ws() + self._use_cli = True + self._connected = shutil.which("openclaw") is not None + return self._connected + + self._connected = True server = payload.get("server", {}) self._conn_id = server.get("connId") logger.info( @@ -189,9 +198,7 @@ async def connect(self) -> bool: self._conn_id, ) # Start background listener - self._listener_task = asyncio.create_task( - self._listen_loop(), name="openclaw-ws-listener" - ) + self._listener_task = asyncio.create_task(self._listen_loop(), name="openclaw-ws-listener") return True else: err = hello.get("error", {}) @@ -215,6 +222,11 @@ async def connect(self) -> bool: async def disconnect(self) -> None: """Disconnect from the gateway.""" self._connected = False + self._use_cli = False + self._use_http = False + if self._http is not None: + await self._http.aclose() + self._http = None if self._listener_task and not self._listener_task.done(): self._listener_task.cancel() try: @@ -232,6 +244,29 @@ async def _close_ws(self) -> None: pass self._ws = None + def _http_base_url(self) -> str: + return self.gateway_url.replace("ws://", "http://").replace("wss://", "https://") + + async def _connect_http(self) -> bool: + """Prefer the persistent loopback HTTP client over per-turn CLI startup.""" + headers = {"Authorization": f"Bearer {self.gateway_token}"} if self.gateway_token else {} + client = httpx.AsyncClient( + base_url=self._http_base_url(), + headers=headers, + timeout=httpx.Timeout(self.timeout), + ) + try: + response = await client.get("/v1/models", timeout=5.0) + response.raise_for_status() + except Exception: + await client.aclose() + return False + self._http = client + self._use_http = True + self._connected = True + logger.info("Using persistent OpenClaw HTTP chat transport") + return True + # ------------------------------------------------------------------ # Background listener # ------------------------------------------------------------------ @@ -284,9 +319,7 @@ async def _dispatch(self, msg: dict) -> None: # Request helpers # ------------------------------------------------------------------ - async def _send_request( - self, method: str, params: dict, timeout: Optional[float] = None - ) -> dict: + async def _send_request(self, method: str, params: dict, timeout: float | None = None) -> dict: """Send a request and wait for the response. Args: @@ -310,7 +343,7 @@ async def _send_request( await self._ws.send(json.dumps(req)) result = await asyncio.wait_for(fut, timeout=timeout or self.timeout) return result - except asyncio.TimeoutError: + except TimeoutError: self._pending.pop(req_id, None) return {"ok": False, "error": {"code": "TIMEOUT", "message": "Request timed out"}} except Exception as e: @@ -328,8 +361,8 @@ def _full_session_key(self) -> str: async def chat( self, message: str, - image_b64: Optional[str] = None, - system_context: Optional[str] = None, + image_b64: str | None = None, + system_context: str | None = None, ) -> OpenClawResponse: """Send a message to OpenClaw and get a response. @@ -359,6 +392,13 @@ async def chat( if image_b64: final_message = f"[Image attached]\n{final_message}" + if self._use_http: + return await self._chat_via_http(final_message, image_b64=image_b64) + if self._use_cli: + if image_b64: + return await self._chat_with_image_via_gateway_cli(final_message, image_b64) + return await self._chat_via_cli(final_message) + idempotency_key = str(uuid.uuid4()) session_key = self._full_session_key() @@ -393,9 +433,7 @@ async def chat( full_text = "" while True: try: - event = await asyncio.wait_for( - event_queue.get(), timeout=self.timeout - ) + event = await asyncio.wait_for(event_queue.get(), timeout=self.timeout) payload = event.get("payload", {}) event_name = event.get("event", "") @@ -425,7 +463,7 @@ async def chat( full_text = content_parts break - except asyncio.TimeoutError: + except TimeoutError: logger.warning("Timeout waiting for chat response (runId=%s)", run_id) if full_text: break @@ -440,10 +478,168 @@ async def chat( logger.error("OpenClaw chat error: %s", e) return OpenClawResponse(content="", error=str(e)) + async def _chat_via_http( + self, message: str, image_b64: str | None = None + ) -> OpenClawResponse: + """Run a normal agent turn without spawning the OpenClaw CLI.""" + if self._http is None: + return OpenClawResponse(content="", error="HTTP transport is unavailable") + content: str | list[dict[str, object]] = message + if image_b64: + content = [ + {"type": "text", "text": message}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}, + }, + ] + try: + response = await self._http.post( + "/v1/chat/completions", + headers={"x-openclaw-session-key": self._full_session_key()}, + json={ + "model": f"openclaw/{self.agent_id}", + "messages": [{"role": "user", "content": content}], + "max_completion_tokens": 180, + }, + ) + response.raise_for_status() + payload = response.json() + text = payload["choices"][0]["message"].get("content") or "" + return OpenClawResponse(content=text) + except Exception as exc: + logger.error("OpenClaw HTTP chat failed: %s", exc) + return OpenClawResponse(content="", error=str(exc)) + + async def _chat_via_cli(self, message: str) -> OpenClawResponse: + """Run an agent turn through OpenClaw's supported local CLI client.""" + process: asyncio.subprocess.Process | None = None + command = [ + "openclaw", + "agent", + "--agent", + self.agent_id, + "--session-key", + self._full_session_key(), + "--message", + message, + "--thinking", + self.thinking, + "--timeout", + str(max(1, int(self.timeout))), + "--json", + ] + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=self.timeout + 15) + except TimeoutError: + if process is not None and process.returncode is None: + process.kill() + await process.wait() + return OpenClawResponse(content="", error="Response timeout") + except Exception as exc: + return OpenClawResponse(content="", error=str(exc)) + + if process.returncode != 0: + detail = stderr.decode("utf-8", errors="replace").strip() + return OpenClawResponse(content="", error=detail or f"openclaw exited {process.returncode}") + + try: + result = json.loads(stdout) + payload = result.get("result") or {} + text = payload.get("finalAssistantVisibleText") + if not text: + chunks = [item.get("text", "") for item in payload.get("payloads", []) if isinstance(item, dict)] + text = "\n".join(chunk for chunk in chunks if chunk) + return OpenClawResponse(content=text or "") + except (json.JSONDecodeError, TypeError) as exc: + return OpenClawResponse(content="", error=f"Invalid OpenClaw response: {exc}") + + async def _run_gateway_call(self, method: str, params: dict, timeout_ms: int) -> dict: + process = await asyncio.create_subprocess_exec( + "openclaw", + "gateway", + "call", + method, + "--params", + json.dumps(params), + "--timeout", + str(timeout_ms), + "--json", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=(timeout_ms / 1000) + 15 + ) + if process.returncode != 0: + detail = stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(detail or f"gateway call exited {process.returncode}") + return json.loads(stdout) + + async def _chat_with_image_via_gateway_cli( + self, message: str, image_b64: str + ) -> OpenClawResponse: + """Use authenticated gateway RPC so native image attachments reach the agent.""" + run_id = str(uuid.uuid4()) + timeout_ms = max(1_000, int(self.timeout * 1_000)) + try: + started = await self._run_gateway_call( + "chat.send", + { + "sessionKey": self._full_session_key(), + "agentId": self.agent_id, + "message": message, + "attachments": [ + { + "type": "image", + "mimeType": "image/jpeg", + "fileName": "reachy-view.jpg", + "content": image_b64, + } + ], + "idempotencyKey": run_id, + }, + timeout_ms, + ) + actual_run_id = started.get("runId") or run_id + await self._run_gateway_call( + "agent.wait", + {"runId": actual_run_id, "timeoutMs": timeout_ms}, + timeout_ms, + ) + history = await self._run_gateway_call( + "chat.history", + {"sessionKey": self._full_session_key(), "limit": 6}, + 30_000, + ) + for item in reversed(history.get("messages", [])): + if item.get("role") != "assistant": + continue + content = item.get("content", "") + if isinstance(content, str): + return OpenClawResponse(content=content) + if isinstance(content, list): + text = "\n".join( + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ).strip() + if text: + return OpenClawResponse(content=text) + return OpenClawResponse(content="", error="No assistant response in chat history") + except Exception as exc: + logger.error("OpenClaw image turn failed: %s", exc) + return OpenClawResponse(content="", error=str(exc)) + async def stream_chat( self, message: str, - image_b64: Optional[str] = None, + image_b64: str | None = None, ) -> AsyncIterator[str]: """Stream a response from OpenClaw. @@ -458,6 +654,14 @@ async def stream_chat( yield "[Error: Not connected to OpenClaw]" return + if self._use_http or self._use_cli: + response = await self.chat(message, image_b64=image_b64) + if response.error: + yield f"[Error: {response.error}]" + elif response.content: + yield response.content + return + final_message = message if image_b64: final_message = f"[Image attached]\n{message}" @@ -485,12 +689,9 @@ async def stream_chat( self._run_events[run_id] = event_queue try: - prev_text = "" while True: try: - event = await asyncio.wait_for( - event_queue.get(), timeout=self.timeout - ) + event = await asyncio.wait_for(event_queue.get(), timeout=self.timeout) payload = event.get("payload", {}) event_name = event.get("event", "") @@ -509,7 +710,7 @@ async def stream_chat( elif event_name == "chat" and payload.get("state") == "final": break - except asyncio.TimeoutError: + except TimeoutError: yield "[Error: timeout]" break finally: @@ -524,7 +725,7 @@ def is_connected(self) -> bool: """Check if bridge is connected to gateway.""" return self._connected - async def get_agent_context(self) -> Optional[str]: + async def get_agent_context(self) -> str | None: """Fetch the agent's current context, personality, and memory summary. This asks OpenClaw to provide a summary of: @@ -570,9 +771,7 @@ async def get_agent_context(self) -> Optional[str]: logger.error("Failed to get agent context: %s", e) return None - async def sync_conversation( - self, user_message: str, assistant_response: str - ) -> None: + async def sync_conversation(self, user_message: str, assistant_response: str) -> None: """Sync a conversation turn back to OpenClaw for memory continuity. Args: @@ -599,7 +798,7 @@ async def sync_conversation( # Global bridge instance (lazy initialization) -_bridge: Optional[OpenClawBridge] = None +_bridge: OpenClawBridge | None = None def get_bridge() -> OpenClawBridge: diff --git a/src/reachy_mini_openclaw/prompts.py b/src/reachy_mini_openclaw/prompts.py index a73a04b..0f9f8dc 100644 --- a/src/reachy_mini_openclaw/prompts.py +++ b/src/reachy_mini_openclaw/prompts.py @@ -1,6 +1,6 @@ """Prompt management for the robot assistant. -Handles loading and customizing system prompts for the OpenAI Realtime session. +Handles loading and customizing optional robot profile prompts. """ import logging @@ -16,7 +16,7 @@ def get_session_instructions() -> str: - """Get the system instructions for the OpenAI Realtime session. + """Get the configured robot profile instructions. Loads from custom profile if configured, otherwise uses default. @@ -49,16 +49,6 @@ def get_session_instructions() -> str: Use the camera tool when asked about your surroundings. Express emotions through movement to enhance communication.""" - -def get_session_voice() -> str: - """Get the voice to use for the OpenAI Realtime session. - - Returns: - Voice name string - """ - return config.OPENAI_VOICE - - def get_available_profiles() -> list[str]: """Get list of available prompt profiles. diff --git a/src/reachy_mini_openclaw/tools/__init__.py b/src/reachy_mini_openclaw/tools/__init__.py index 56254ca..a25ce8e 100644 --- a/src/reachy_mini_openclaw/tools/__init__.py +++ b/src/reachy_mini_openclaw/tools/__init__.py @@ -1,6 +1,6 @@ """Tool definitions for Reachy Mini OpenClaw. -These tools are exposed to the OpenAI Realtime API and allow the assistant +These tools are exposed through the body integration and allow the assistant to control the robot and interact with the environment. """ diff --git a/src/reachy_mini_openclaw/tools/core_tools.py b/src/reachy_mini_openclaw/tools/core_tools.py index 3028fc3..04a1108 100644 --- a/src/reachy_mini_openclaw/tools/core_tools.py +++ b/src/reachy_mini_openclaw/tools/core_tools.py @@ -98,6 +98,7 @@ class ToolDependencies: camera_worker: Optional[Any] = None openclaw_bridge: Optional["OpenClawBridge"] = None vision_manager: Optional[Any] = None # Local vision processor (SmolVLM2) + motion_catalog: Optional[Any] = None # Tool specifications in OpenAI format @@ -145,15 +146,14 @@ class ToolDependencies: }, { "type": "function", - "name": "dance", - "description": "Perform a dance animation. Use this to express joy, celebrate, or entertain.", + "name": "play_dance", + "description": "Play an exact move from Reachy's dynamically discovered dance catalog.", "parameters": { "type": "object", "properties": { "dance_name": { "type": "string", - "enum": ["happy", "excited", "wave", "nod", "shake", "bounce"], - "description": "The dance to perform" + "description": "Exact dance catalog name" } }, "required": ["dance_name"] @@ -161,20 +161,32 @@ class ToolDependencies: }, { "type": "function", - "name": "emotion", - "description": "Express an emotion through movement. Use this to show reactions and feelings.", + "name": "play_emotion", + "description": "Play an emotion by exact move name, semantic family, or common alias.", "parameters": { "type": "object", "properties": { "emotion_name": { "type": "string", - "enum": ["happy", "sad", "surprised", "curious", "thinking", "confused", "excited"], - "description": "The emotion to express" + "description": "Exact move (for example proud3), family (proud), or alias (victory)" } }, "required": ["emotion_name"] } }, + { + "type": "function", + "name": "list_moves", + "description": "List or search the live Reachy emotion and dance catalogs.", + "parameters": { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["emotion", "dance"]}, + "query": {"type": "string", "description": "Optional case-insensitive name fragment"}, + }, + "required": [], + }, + }, { "type": "function", "name": "stop_moves", @@ -231,8 +243,9 @@ async def dispatch_tool_call( "look": _handle_look, "camera": _handle_camera, "face_tracking": _handle_face_tracking, - "dance": _handle_dance, - "emotion": _handle_emotion, + "play_dance": _handle_dance, + "play_emotion": _handle_emotion, + "list_moves": _handle_list_moves, "stop_moves": _handle_stop_moves, "idle": _handle_idle, } @@ -377,8 +390,8 @@ async def _handle_face_tracking(args: dict, deps: ToolDependencies) -> dict: return {"error": "Camera not available for face tracking"} try: - # Check if head tracker is available - if deps.camera_worker.head_tracker is None: + # A detector can live in this process or in Reachy's daemon. + if deps.camera_worker.head_tracker is None and not deps.camera_worker.daemon_tracking: return {"error": "Face tracking not available - no head tracker initialized"} deps.camera_worker.set_head_tracking_enabled(enabled) @@ -389,64 +402,55 @@ async def _handle_face_tracking(args: dict, deps: ToolDependencies) -> dict: async def _handle_dance(args: dict, deps: ToolDependencies) -> dict: """Handle dance tool.""" - dance_name = args.get("dance_name", "happy") - + dance_name = args.get("dance_name", "") try: - # Try to use dance library if available - from reachy_mini_dances_library import dances - - if hasattr(dances, dance_name): - dance_class = getattr(dances, dance_name) - dance_move = dance_class() - deps.movement_manager.queue_move(dance_move) - return {"status": "success", "dance": dance_name} - else: - # Fallback to simple head movement - return await _handle_emotion({"emotion_name": dance_name}, deps) - except ImportError: - # No dance library, use emotion as fallback - return await _handle_emotion({"emotion_name": dance_name}, deps) + if deps.motion_catalog is None: + return {"error": "Motion catalog is unavailable"} + deps.movement_manager.queue_move(deps.motion_catalog.get("dance", dance_name)) + return {"status": "success", "dance": dance_name} except Exception as e: return {"error": str(e)} async def _handle_emotion(args: dict, deps: ToolDependencies) -> dict: """Handle emotion expression.""" - from reachy_mini_openclaw.moves import HeadLookMove - - emotion_name = args.get("emotion_name", "happy") - - # Map emotions to simple head movements - emotion_sequences = { - "happy": ["up", "front"], - "sad": ["down"], - "surprised": ["up", "front"], - "curious": ["right", "left", "front"], - "thinking": ["up", "left"], - "confused": ["left", "right", "front"], - "excited": ["up", "down", "up", "front"], - } - - sequence = emotion_sequences.get(emotion_name, ["front"]) - + emotion_name = args.get("emotion_name", "") try: - for direction in sequence: - _, current_ant = deps.robot.get_current_joint_positions() - current_head = deps.robot.get_current_head_pose() - - move = HeadLookMove( - direction=direction, - start_pose=current_head, - start_antennas=tuple(current_ant), - duration=0.5, - ) - deps.movement_manager.queue_move(move) - - return {"status": "success", "emotion": emotion_name} + if deps.motion_catalog is None: + return {"error": "Motion catalog is unavailable"} + resolved = deps.motion_catalog.resolve_emotion(emotion_name) + if resolved is None: + return {"error": f"Unknown emotion intent: {emotion_name}"} + deps.movement_manager.queue_move(deps.motion_catalog.get("emotion", resolved)) + return {"status": "success", "emotion": resolved, "requested": emotion_name} except Exception as e: return {"error": str(e)} +async def _handle_list_moves(args: dict, deps: ToolDependencies) -> dict: + if deps.motion_catalog is None: + return {"error": "Motion catalog is unavailable"} + kind = args.get("kind") + query = args.get("query", "") + if query: + return {"status": "success", "moves": deps.motion_catalog.search(query, kind)} + if kind: + result = {"status": "success", "moves": {kind: deps.motion_catalog.list(kind)}} + if kind == "emotion": + result["emotion_families"] = deps.motion_catalog.emotion_map() + result["unmapped_emotions"] = deps.motion_catalog.unmapped_emotions() + return result + return { + "status": "success", + "moves": { + "emotion": deps.motion_catalog.list("emotion"), + "dance": deps.motion_catalog.list("dance"), + }, + "emotion_families": deps.motion_catalog.emotion_map(), + "unmapped_emotions": deps.motion_catalog.unmapped_emotions(), + } + + async def _handle_stop_moves(args: dict, deps: ToolDependencies) -> dict: """Stop all movements.""" deps.movement_manager.clear_move_queue() diff --git a/tests/test_conversation.py b/tests/test_conversation.py new file mode 100644 index 0000000..f0feda6 --- /dev/null +++ b/tests/test_conversation.py @@ -0,0 +1,163 @@ +import numpy as np +import pytest + +from reachy_mini_openclaw.audio.conversation import ConversationHandler, EnergyVAD, Utterance, needs_vision +from reachy_mini_openclaw.audio.providers import SynthesizedAudio +from reachy_mini_openclaw.openclaw_bridge import OpenClawResponse + + +def test_vad_segments_one_utterance_with_prefix() -> None: + vad = EnergyVAD( + rms_threshold=0.05, + activation_ms=50, + prefix_ms=100, + silence_ms=100, + min_speech_ms=100, + max_speech_seconds=2, + ) + sample_rate = 1000 + assert vad.feed(sample_rate, np.zeros(50, dtype=np.float32)) is None + assert vad.feed(sample_rate, np.ones(100, dtype=np.float32) * 0.2) is None + utterance = vad.feed(sample_rate, np.zeros(100, dtype=np.float32)) + assert utterance is not None + assert utterance.sample_rate == sample_rate + assert utterance.samples.size == 200 + assert not vad.speaking + + +def test_vad_discards_short_noise() -> None: + vad = EnergyVAD(rms_threshold=0.05, activation_ms=50, silence_ms=50, min_speech_ms=100) + assert vad.feed(1000, np.ones(20, dtype=np.float32)) is None + assert vad.feed(1000, np.zeros(50, dtype=np.float32)) is None + assert not vad.speaking + + +def test_vad_requires_sustained_loud_onset() -> None: + vad = EnergyVAD( + rms_threshold=0.05, + activation_ms=100, + silence_ms=50, + min_speech_ms=100, + ) + assert vad.feed(1000, np.ones(40, dtype=np.float32) * 0.2) is None + assert not vad.speaking + assert vad.feed(1000, np.zeros(20, dtype=np.float32)) is None + assert vad.feed(1000, np.ones(60, dtype=np.float32) * 0.2) is None + assert not vad.speaking + assert vad.feed(1000, np.ones(40, dtype=np.float32) * 0.2) is None + assert vad.speaking + + +def test_vad_hardware_gate_blocks_onset_but_not_active_speech() -> None: + vad = EnergyVAD(rms_threshold=0.05, activation_ms=50) + loud = np.ones(50, dtype=np.float32) * 0.2 + assert vad.feed(1000, loud, allow_start=False) is None + assert not vad.speaking + assert vad.feed(1000, loud, allow_start=True) is None + assert vad.speaking + assert vad.feed(1000, loud, allow_start=False) is None + assert vad.speaking + + +def test_visual_questions_request_a_camera_frame() -> None: + assert needs_vision("Asmo, can you see me?") + assert needs_vision("Look at what I'm holding") + assert not needs_vision("I see what you mean") + + +class Movement: + def __init__(self) -> None: + self.processing = [] + self.listening = [] + + def set_processing(self, value: bool) -> None: + self.processing.append(value) + + def set_listening(self, value: bool) -> None: + self.listening.append(value) + + +class Wobbler: + def __init__(self) -> None: + self.feeds = 0 + + def feed(self, audio: str) -> None: + self.feeds += 1 + + def reset(self) -> None: + pass + + +class Deps: + def __init__(self) -> None: + self.movement_manager = Movement() + self.head_wobbler = Wobbler() + + +class STT: + async def transcribe(self, samples, sample_rate) -> str: + return "what time is it" + + +class TTS: + async def synthesize(self, text: str) -> SynthesizedAudio: + assert text == "It is robot o'clock." + return SynthesizedAudio(24000, np.arange(5000, dtype=np.int16)) + + +class Bridge: + is_connected = True + + async def connect(self) -> bool: + return True + + async def chat( + self, message: str, image_b64: str | None, system_context: str + ) -> OpenClawResponse: + assert message == "what time is it" + assert image_b64 is None + assert "Reachy Mini" in system_context + return OpenClawResponse("It is robot o'clock.") + + +@pytest.mark.asyncio +async def test_pipeline_routes_openclaw_response_to_tts_and_audio_queue() -> None: + deps = Deps() + handler = ConversationHandler( + stt=STT(), + tts=TTS(), + openclaw_bridge=Bridge(), + deps=deps, + vad=EnergyVAD(), + output_chunk_samples=2048, + ) + await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32))) + + outputs = [] + while not handler.output_queue.empty(): + outputs.append(handler.output_queue.get_nowait()) + assert outputs[0] == {"role": "user", "content": "what time is it"} + assert outputs[1] == {"role": "assistant", "content": "It is robot o'clock."} + audio = [item for item in outputs if isinstance(item, tuple)] + assert [chunk.shape[1] for _, chunk in audio] == [2048, 2048, 904] + assert deps.head_wobbler.feeds == 0 + assert deps.movement_manager.processing == [True, False] + + +@pytest.mark.asyncio +async def test_microphone_is_suppressed_while_robot_audio_is_playing() -> None: + deps = Deps() + vad = EnergyVAD(rms_threshold=0.01) + handler = ConversationHandler( + stt=STT(), + tts=TTS(), + openclaw_bridge=Bridge(), + deps=deps, + vad=vad, + ) + handler._ignore_input_until = float("inf") + + await handler.receive((16000, np.ones(1600, dtype=np.float32))) + + assert not vad.speaking + assert handler._utterances.empty() diff --git a/tests/test_expressive_motion.py b/tests/test_expressive_motion.py new file mode 100644 index 0000000..685424d --- /dev/null +++ b/tests/test_expressive_motion.py @@ -0,0 +1,48 @@ +import numpy as np +from reachy_mini.motion.move import Move + +from reachy_mini_openclaw.moves import MovementManager + + +class ExclusiveMove(Move): + name = "test_move" + exclusive_tracking = True + + @property + def duration(self): + return 1.0 + + def evaluate(self, _t): + return np.eye(4), np.zeros(2), 0.0 + + +class FakeCamera: + def __init__(self): + self.is_head_tracking_enabled = True + self.changes = [] + + def set_head_tracking_enabled(self, enabled): + self.is_head_tracking_enabled = enabled + self.changes.append(enabled) + + def get_face_tracking_offsets(self): + return (0.0, 0.0, 0.0, 0.0, 0.0, 0.5) + + +def test_expressive_move_temporarily_owns_tracking(): + camera = FakeCamera() + manager = MovementManager(object(), camera_worker=camera) + manager.queue_move(ExclusiveMove()) + + manager._poll_signals(10.0) + manager._manage_move_queue(10.0) + assert camera.changes == [False] + + # Secondary sound/face/speech offsets must not corrupt choreography. + manager.state.face_tracking_offsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.5) + _, antennas, yaw = manager._get_secondary_pose() + assert antennas == (0.0, 0.0) + assert yaw == 0.0 + + manager._manage_move_queue(11.1) + assert camera.changes == [False, True] diff --git a/tests/test_motion_catalog.py b/tests/test_motion_catalog.py new file mode 100644 index 0000000..d9d1cdc --- /dev/null +++ b/tests/test_motion_catalog.py @@ -0,0 +1,121 @@ +from reachy_mini_openclaw.motion_catalog import ( + EMOTION_FAMILIES, + UTILITY_RECORDED_MOVES, + MotionCatalog, +) + + +class FakeMoves: + def __init__(self, names): + self.names = names + + def list_moves(self): + return self.names + + def get(self, name): + if name not in self.names: + raise ValueError(name) + return name + + +def catalog(): + item = object.__new__(MotionCatalog) + item.emotions = FakeMoves( + ["thoughtful1", "thoughtful2", "laughing2", "inquiring2", "success2", "anxiety1"] + ) + return item + + +def full_catalog(): + item = object.__new__(MotionCatalog) + expressive = [name for variants in EMOTION_FAMILIES.values() for name in variants] + item.emotions = FakeMoves(expressive + list(UTILITY_RECORDED_MOVES)) + return item + + +def test_parse_and_strip_valid_directive(): + clean, directive = catalog().parse_response("Let me think. [[emotion:thoughtful2]]") + assert clean == "Let me think." + assert directive.name == "thoughtful2" + + +def test_invalid_directive_is_removed_but_not_played(): + clean, directive = catalog().parse_response("Nope. [[dance:table_flip]]") + assert clean == "Nope." + assert directive is None + + +def test_conservative_question_fallback(): + directive = catalog().infer("What fresh hell is this?") + assert directive.name == "inquiring2" + + +def test_explicit_dance_request_has_deterministic_fallback(): + directive = catalog().infer_request("Hey Asmo, do a little dance for me") + assert directive.kind == "dance" + assert directive.name == "side_to_side_sway" + + +def test_negated_dance_request_does_not_move(): + assert catalog().infer_request("Please don't dance right now") is None + + +def test_spoken_official_dance_name_resolves_exactly(): + directive = catalog().infer_request("Can you try headbanger combo?") + assert directive.kind == "dance" + assert directive.name == "headbanger_combo" + + +def test_spoken_emotion_intent_resolves_without_model_directive(): + directive = catalog().infer_request("Can you express anxiety as an emotion?") + assert directive.kind == "emotion" + assert directive.name == "anxiety1" + + +def test_console_emotion_map_covers_exactly_81_unique_moves(): + names = [name for variants in EMOTION_FAMILIES.values() for name in variants] + assert len(names) == 81 + assert len(set(names)) == 81 + + +def test_utility_recordings_are_not_presented_as_emotions(): + item = full_catalog() + assert len(item.list("emotion")) == 81 + assert item.list("utility") == sorted(UTILITY_RECORDED_MOVES) + assert not (set(item.list("emotion")) & UTILITY_RECORDED_MOVES) + + +def test_all_installed_emotions_have_semantic_family(): + item = full_catalog() + assert item.unmapped_emotions() == [] + assert set(item.emotion_map()) == set(EMOTION_FAMILIES) + + +def test_family_and_alias_resolve_to_restrained_default(): + item = full_catalog() + assert item.resolve_emotion("proud") == "proud1" + assert item.resolve_emotion("victory") == "success1" + assert item.resolve_emotion("grossed out") == "disgusted1" + + +def test_exact_numbered_variant_remains_addressable(): + assert full_catalog().resolve_emotion("proud three") == "proud3" + assert full_catalog().resolve_emotion("proud3") == "proud3" + + +def test_specific_yes_no_moods_resolve(): + item = full_catalog() + assert item.resolve_emotion("sad no") == "no_sad1" + assert item.resolve_emotion("excited no") == "no_excited1" + assert item.resolve_emotion("sad yes") == "yes_sad1" + + +def test_response_directive_accepts_semantic_family(): + clean, directive = full_catalog().parse_response("Indeed. [[emotion:proud]]") + assert clean == "Indeed." + assert directive.name == "proud1" + + +def test_explicit_request_resolves_broad_emotion_aliases(): + directive = full_catalog().infer_request("Show me how grossed out you are") + assert directive == type(directive)("emotion", "disgusted1") diff --git a/tests/test_playback.py b/tests/test_playback.py new file mode 100644 index 0000000..1cdcb7e --- /dev/null +++ b/tests/test_playback.py @@ -0,0 +1,16 @@ +import numpy as np + +from reachy_mini_openclaw.audio.playback import apply_output_gain + + +def test_output_gain_preserves_full_scale_by_default() -> None: + samples = np.array([-1.0, -0.5, 0.5, 1.0], dtype=np.float32) + np.testing.assert_allclose(apply_output_gain(samples, 1.0), samples) + + +def test_output_gain_boosts_and_clips_safely() -> None: + samples = np.array([-0.75, -0.25, 0.25, 0.75], dtype=np.float32) + np.testing.assert_allclose( + apply_output_gain(samples, 2.0), + np.array([-1.0, -0.5, 0.5, 1.0], dtype=np.float32), + ) diff --git a/tests/test_speech_providers.py b/tests/test_speech_providers.py new file mode 100644 index 0000000..d9197d7 --- /dev/null +++ b/tests/test_speech_providers.py @@ -0,0 +1,115 @@ +import io +import wave + +import httpx +import numpy as np +import pytest + +from reachy_mini_openclaw.audio.providers import ( + ChatterboxTTS, + ElevenLabsTTS, + FallbackTTS, + OpenAICompatibleSTT, + SynthesizedAudio, + decode_wav, + encode_wav, +) + + +def test_wav_round_trip() -> None: + source = np.linspace(-0.5, 0.5, 800, dtype=np.float32) + decoded = decode_wav(encode_wav(source, 16000)) + assert decoded.sample_rate == 16000 + assert decoded.samples.dtype == np.int16 + assert decoded.samples.size == source.size + + +@pytest.mark.asyncio +async def test_openai_compatible_stt_posts_wav() -> None: + async def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/audio/transcriptions" + assert request.headers["authorization"] == "Bearer local-secret" + body = await request.aread() + assert b'distil-large-v3' in body + assert b'filename="utterance.wav"' in body + return httpx.Response(200, json={"text": " hello Reachy "}) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + provider = OpenAICompatibleSTT( + "http://stt.test/v1", + "distil-large-v3", + api_key="local-secret", + client=client, + ) + try: + text = await provider.transcribe(np.ones(1600, dtype=np.float32) * 0.1, 16000) + finally: + await client.aclose() + assert text == "hello Reachy" + + +def _wav_bytes() -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(24000) + wav.writeframes(np.arange(64, dtype=np.int16).tobytes()) + return output.getvalue() + + +@pytest.mark.asyncio +async def test_chatterbox_requests_named_voice_and_decodes_wav() -> None: + async def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/audio/speech" + payload = __import__("json").loads((await request.aread()).decode()) + assert payload == { + "input": "hello", + "voice": "asmo", + "model": "chatterbox-turbo", + "response_format": "wav", + } + return httpx.Response(200, content=_wav_bytes(), headers={"content-type": "audio/wav"}) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + provider = ChatterboxTTS("http://tts.test/v1/audio/speech", voice="asmo", client=client) + try: + audio = await provider.synthesize("hello") + finally: + await client.aclose() + assert audio.sample_rate == 24000 + assert audio.samples.size == 64 + + +@pytest.mark.asyncio +async def test_elevenlabs_requests_raw_pcm() -> None: + pcm = np.arange(32, dtype=np.int16).tobytes() + + async def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/text-to-speech/voice-123/stream" + assert request.url.params["output_format"] == "pcm_24000" + assert request.headers["xi-api-key"] == "eleven-secret" + return httpx.Response(200, content=pcm) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + provider = ElevenLabsTTS("eleven-secret", "voice-123", base_url="http://eleven.test", client=client) + try: + audio = await provider.synthesize("hello") + finally: + await client.aclose() + assert audio.sample_rate == 24000 + assert audio.samples.tolist() == list(range(32)) + + +@pytest.mark.asyncio +async def test_fallback_tts_uses_second_provider() -> None: + class Broken: + async def synthesize(self, text: str) -> SynthesizedAudio: + raise RuntimeError("offline") + + class Working: + async def synthesize(self, text: str) -> SynthesizedAudio: + return SynthesizedAudio(24000, np.array([1, 2], dtype=np.int16)) + + audio = await FallbackTTS(Broken(), Working()).synthesize("hello") + assert audio.samples.tolist() == [1, 2] From 24f4f3dcb57fb04263950d249a1cd30056a9f871 Mon Sep 17 00:00:00 2001 From: Asmo Bot Date: Wed, 22 Jul 2026 15:49:54 -0700 Subject: [PATCH 2/4] ClawBody Local 0.2.0: mail attention queue and scheduled sleep --- .env.example | 10 + NATIVE_APP_REVIEW.md | 67 ++++++ NATIVE_DEPLOY_TODO.md | 80 +++++++ README.md | 43 +++- pyproject.toml | 28 ++- src/reachy_mini_openclaw/attention.py | 176 ++++++++++++++ .../audio/conversation.py | 120 +++++++++- src/reachy_mini_openclaw/config.py | 29 ++- src/reachy_mini_openclaw/gradio_app.py | 39 ++- src/reachy_mini_openclaw/main.py | 223 ++++++++++++------ src/reachy_mini_openclaw/moves.py | 17 ++ tests/test_attention.py | 56 +++++ tests/test_conversation.py | 218 ++++++++++++++++- 13 files changed, 996 insertions(+), 110 deletions(-) create mode 100644 NATIVE_APP_REVIEW.md create mode 100644 NATIVE_DEPLOY_TODO.md create mode 100644 src/reachy_mini_openclaw/attention.py create mode 100644 tests/test_attention.py diff --git a/.env.example b/.env.example index da9bf5d..0e7cf5b 100644 --- a/.env.example +++ b/.env.example @@ -68,6 +68,16 @@ VAD_MIN_SPEECH_MS=300 VAD_MAX_SPEECH_SECONDS=20 VAD_REQUIRE_HARDWARE_SPEECH=true +# Optional wake phrase gate. When enabled, only transcripts beginning with the +# configured phrase are sent to OpenClaw; the phrase is stripped from commands. +ENABLE_WAKE_PHRASE=false +WAKE_PHRASE=Hey Asmo + +# Quiet mail-attention indicator and head double-tap playback. +ENABLE_ATTENTION_QUEUE=false +ATTENTION_API_URL=http://192.168.1.238:18790 +ATTENTION_POLL_SECONDS=60 + # ============================================================================== # OPTIONAL: Features # ============================================================================== diff --git a/NATIVE_APP_REVIEW.md b/NATIVE_APP_REVIEW.md new file mode 100644 index 0000000..994582e --- /dev/null +++ b/NATIVE_APP_REVIEW.md @@ -0,0 +1,67 @@ +# Native Reachy App Contract Review + +Reviewed against the installed Reachy Mini 1.9 SDK and its bundled +`reachy-mini-app-assistant` validator. + +## Confirmed contract + +- The project name is `clawbody`; the validator derives the package from the + entry point and therefore expects class `ReachyMiniOpenclaw` in + `reachy_mini_openclaw.main`. +- The native class must explicitly inherit `reachy_mini.ReachyMiniApp`. +- The entry point must be: + + ```toml + [project.entry-points."reachy_mini_apps"] + clawbody = "reachy_mini_openclaw.main:ReachyMiniOpenclaw" + ``` + +- The SDK owns the `ReachyMini` context, creates a `threading.Event`, and calls + `run(reachy_mini, stop_event)` through `ReachyMiniApp.wrapped_run()`. +- The app must stop when that event is set and must not close/disconnect the + SDK-owned robot itself. +- The existing README frontmatter, root `index.html`, root `style.css`, and src + package layout satisfy the validator's static metadata/layout requirements. +- The official validator finishes by installing the project into an isolated + venv, verifying the entry point, and uninstalling it. + +## Current defects to correct + +1. `ClawBodyApp` neither has the validator-required name nor inherits + `ReachyMiniApp`. The current validator fails before installation with the + expected entry point `reachy_mini_openclaw.main:ReachyMiniOpenclaw`. +2. The native wrapper catches and suppresses every exception. That prevents the + app manager from observing a failed app and undermines watchdog/restart + behavior. Cleanup should occur in `finally`, then the failure must propagate. +3. The external stop event is only polled by the record/play loops. The + conversation handler remains in the `asyncio.gather()` set, so a dashboard + stop can leave `run()` waiting indefinitely. A stop watcher must cancel all + app tasks promptly. +4. `ClawBodyCore.stop()` is synchronous but tries to run the asynchronous + gateway disconnect with `run_until_complete()` on the current event loop. + When called from the native wrapper after `run_until_complete()`, this is + fragile; when called while a loop is active, it is invalid. Async shutdown + needs a single owning loop and an idempotent lifecycle. +5. Configuration validation calls `sys.exit(1)` inside an embeddable app class. + Native lifecycle errors should raise typed exceptions so the SDK can record + `app.error` and manage failure correctly. +6. Native camera tracking currently builds its daemon URL from + `ROBOT_HOST`/`ROBOT_PORT`. On the Wireless body, the SDK-owned robot is local + and the daemon-side HTTP API should use localhost (or an explicit native + daemon URL), not the remote workstation default. +7. The package deliberately depends on remote xeon STT/TTS and the existing + OpenClaw gateway. Startup must tolerate those services being temporarily + unavailable and reconnect with bounded backoff rather than exiting or + pretending the app is healthy. + +## Required implementation shape + +- Introduce `ReachyMiniOpenclaw(ReachyMiniApp)` with the exact entry point. +- Keep `ClawBodyCore` reusable for CLI/simulator operation. +- Give the core an idempotent async shutdown path and an explicit external-stop + watcher. +- Let fatal native exceptions propagate after cleanup. +- Add contract tests for inheritance, entry-point loading, stop behavior, + cleanup idempotency, and reconnect/backoff behavior. +- Preserve the existing `main` agent and `agent:main:reachy` session. Native + packaging changes transport/lifecycle only; it must not create Asmo(deux). diff --git a/NATIVE_DEPLOY_TODO.md b/NATIVE_DEPLOY_TODO.md new file mode 100644 index 0000000..59b29cc --- /dev/null +++ b/NATIVE_DEPLOY_TODO.md @@ -0,0 +1,80 @@ +# Reachy Native Deployment TODO + +Last checked: 2026-07-22 + +Goal: package and deploy ClawBody as a native Reachy Mini app that starts and recovers without the laptop, while continuing to use the existing OpenClaw `main` agent and `agent:main:reachy` session. Do not create a second Asmo agent. + +## Work queue + +- [x] **Consolidate reviewer findings and inspect the current native app contract** + - Capture the actionable Claude review findings in the repo. + - Verify the expected `ReachyMiniApp` class, lifecycle hooks, metadata, and runtime assumptions against the installed Reachy 1.9 tooling. + - Exit condition: findings are recorded and the exact contract changes are identified. + - Evidence: `NATIVE_APP_REVIEW.md`; installed Reachy 1.9 validator reproduces the entry-point/class failure and documents the required lifecycle corrections. + +- [x] **Correct the native Reachy app entry point and lifecycle** + - Fix the app class/entry-point contract. + - Ensure startup, stop, cleanup, and robot-resource ownership are safe and idempotent. + - Preserve the existing xeon STT/TTS and `agent:main:reachy` architecture. + - Exit condition: native entry point imports and initializes under the official tooling. + - Evidence: `ReachyMiniOpenclaw(ReachyMiniApp)` is the package entry point; external stop events cancel all app tasks, shutdown is async/idempotent, native failures propagate, and native camera tracking uses localhost. + +- [x] **Add reconnect and watchdog resilience** + - Recover from Reachy daemon, WebRTC, xeon voice service, and OpenClaw gateway interruptions without a laptop. + - Use bounded backoff and clean shutdown; do not create restart storms. + - Exit condition: automated tests cover disconnect/reconnect and process restart behavior. + - Evidence: gateway reconnect, transient STT/TTS retry, microphone/speaker recovery backoff, and native failure propagation to the Reachy supervisor are covered by the 33-test suite. + +- [x] **Make native configuration and secrets deployment-safe** + - Remove LAN-specific or workstation-only assumptions from the package. + - Keep secrets out of git and Hugging Face history. + - Document the minimum runtime configuration for Reachy, xeon STT/TTS, and the existing OpenClaw gateway/session. + - Exit condition: a clean install can be configured without editing source. + - Evidence: service endpoints are explicit required configuration, deployment environment overrides local `.env`, native configuration is documented in `README.md`, and secrets remain environment-only. + +- [x] **Run local validation and final code review** + - Run the complete test suite. + - Run `reachy-mini-app-assistant check` (or the current official equivalent). + - Review the final diff for safety, secret leakage, lifecycle correctness, and accidental creation of an Asmo(deux). + - Exit condition: validation passes with no unresolved critical findings. + - Evidence: UV-managed CPython 3.13.12; 35 tests pass; the Reachy 1.9 app assistant passes its clean install, entry-point, and uninstall checks; package builds succeed; artifact and workspace scans found no credential material; final review fixed executor-thread leakage, un-awaited CLI/Gradio shutdown, and blocking async startup sleeps. + +- [x] **Package and publish a private Hugging Face Space** + - Use the Hugging Face credentials in 1Password vault `Asmo`, item `3nm3rm35entwmzoc64bznvdbma` at runtime only. + - Create/update the private native Reachy app Space and push the validated package. + - Never print or persist the token in logs, repo files, shell history, or memory. + - Exit condition: the private Space exists and exposes the expected app revision. + - Evidence: the scoped write token was read from 1Password at runtime only; the official Reachy 1.9 publish check passed; private Space `asmolebot/clawbodylocal` was created and verified at revision `dc3d1ae0c2d4386ed186476edc6efb388d08c97d` with the expected native app files and no secret-like filenames. The original `asmolebot/clawbody` identifier resolves to the renamed Space through the authenticated Hub API. + +- [ ] **Install and configure the app on Reachy Mini** + - HOLD: do not begin this item until Chris explicitly signs off on the final deployment. + - Install through the Reachy app API/tooling. + - Configure it as the startup app if supported by the installed Reachy version. + - Keep deployment reversible and record the prior app/startup state. + - Exit condition: the app launches on Reachy without the laptop console. + +- [ ] **Run laptop-independent end-to-end acceptance tests** + - Verify microphone -> xeon STT -> existing OpenClaw Asmo session -> xeon TTS -> speaker. + - Verify camera/face/sound tracking, representative emotions, and at least one safe dance. + - Restart the app/daemon and confirm recovery. + - Exit condition: a complete spoken turn and motion test pass while the laptop is closed/disconnected. + +- [ ] **Commit, push, and update the upstream draft PR** + - Commit only reviewed source/docs/config changes; exclude private deployment state and credentials. + - Push the existing branch and update the draft PR with native-deployment validation results. + - Exit condition: clean worktree, pushed commit, and PR reflects the deployed revision. + +- [ ] **Begin the separate body-latency/bootstrap optimization** + - Only after native deployment is stable, slim the Reachy session bootstrap/tool surface and add first-token/first-audio timing. + - Do not create a second agent personality or duplicate long-term memory. + - Exit condition: a measured optimization plan is recorded from the native deployment baseline. + +## Heartbeat rules + +- Advance at most one unchecked item per heartbeat. +- Read current repo/runtime state before acting; do not redo completed work. +- Mark an item complete only when its exit condition is evidenced. +- Continue safe local implementation and validation autonomously. +- Publishing/staging the private Space is authorized. Installing, launching, or changing startup state on Reachy requires Chris's explicit final sign-off. +- Report only completion, a new blocker, deployment/recovery state, or a request that genuinely needs Chris. +- If no material state changed, stay quiet. diff --git a/README.md b/README.md index 8e293c7..08cb5d2 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ --- -title: ClawBody +title: ClawBody Local emoji: 🦞 colorFrom: red colorTo: purple sdk: static pinned: false -short_description: OpenClaw AI with robot body and face tracking +short_description: Asmo's local OpenClaw body for Reachy Mini tags: - reachy_mini - reachy_mini_python_app @@ -31,14 +31,14 @@ tags: - human-robot-interaction --- -# 🦞🤖 ClawBody +# 🦞🤖 ClawBody Local See [EMOTIONS.md](EMOTIONS.md) for the complete 81-move Reachy emotion map, semantic families, aliases, and selection behavior. **Give your OpenClaw AI agent a physical robot body!** -ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body. Local STT transcribes the microphone, OpenClaw produces the actual response, and Chatterbox or ElevenLabs gives it a voice. +ClawBody Local is Asmo-owned infrastructure that combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body. Local STT transcribes the microphone, OpenClaw produces the actual response, and Chatterbox or ElevenLabs gives it a voice. ![Reachy Mini Dance](https://huggingface.co/spaces/pollen-robotics/reachy_mini_conversation_app/resolve/main/docs/assets/reachy_mini_dance.gif) @@ -204,6 +204,31 @@ cd clawbody /venvs/apps_venv/bin/pip install -e . ``` +### Native Reachy App configuration + +The native app reads deployment environment variables at startup. A packaged +`.env` file is optional for development only; environment variables supplied by +the app manager take precedence. Configure at least: + +```text +OPENCLAW_GATEWAY_URL=ws://your-openclaw-gateway:18789 +OPENCLAW_TOKEN=your-gateway-token +OPENCLAW_AGENT_ID=main +OPENCLAW_SESSION_KEY=reachy +STT_BASE_URL=http://your-speech-host:8890/v1 +STT_MODEL=distil-large-v3 +CHATTERBOX_URL=http://your-speech-host:8890/v1/audio/speech +CHATTERBOX_VOICE=asmo +ENABLE_WAKE_PHRASE=true +WAKE_PHRASE=Hey Asmo +``` + +The native app uses the SDK-owned robot connection and targets the local Reachy +daemon for camera tracking. It does not require a laptop-side console or a +second OpenClaw agent. Keep gateway, STT, TTS, and optional ElevenLabs secrets +in the app manager's secret/environment store; never commit them to `.env`, a +Space, or the repository. + ## ⚙️ Configuration 1. Copy the example environment file: @@ -232,8 +257,18 @@ CHATTERBOX_VOICE=default # Optional - Face tracking (enabled by default) ENABLE_FACE_TRACKING=true HEAD_TRACKER_TYPE=mediapipe # or "yolo" for more accuracy + +# Optional - require commands to begin with "Hey Asmo" +ENABLE_WAKE_PHRASE=true +WAKE_PHRASE=Hey Asmo ``` +Wake phrase matching is case-insensitive and tolerates STT punctuation such as +`Hey, Asmo`. It must occur at the start of each utterance. The phrase is removed +before the command is sent to OpenClaw; saying only the phrase is treated as a +greeting. Leave `ENABLE_WAKE_PHRASE=false` for the original always-responsive +behavior. + ## 🎮 Usage ### With Simulator diff --git a/pyproject.toml b/pyproject.toml index 6c13857..d5049f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,16 +3,17 @@ requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "clawbody" -version = "0.1.0" -description = "Give an OpenClaw agent a Reachy Mini body with local STT, pluggable TTS, and expressive movement." +name = "clawbodylocal" +version = "0.2.0" +description = "Asmo's local OpenClaw body for Reachy Mini, with local STT, pluggable TTS, and expressive movement." readme = "README.md" license = {text = "Apache-2.0"} requires-python = ">=3.11" authors = [ - {name = "Tom", email = "tom@example.com"} + {name = "Asmo", email = "stanme@asmo.bot"} ] keywords = [ + "clawbodylocal", "clawbody", "reachy-mini", "openclaw", @@ -45,6 +46,9 @@ dependencies = [ "numpy", "scipy", "pillow>=10", + # Native apps share Reachy's apps venv. Pin the validated SDK in the main + # dependency set so installing the dance library cannot downgrade it. + "reachy-mini==1.9.0", "reachy-mini-dances-library>=0.2.1", # OpenClaw gateway client (WebSocket protocol) @@ -57,10 +61,6 @@ dependencies = [ "python-dotenv", ] -# Note: reachy-mini SDK must be installed separately from the robot or GitHub: -# pip install git+https://github.com/pollen-robotics/reachy_mini.git -# Or on the robot, it's pre-installed. - [project.optional-dependencies] robot = [ "reachy-mini==1.9.0", @@ -99,16 +99,18 @@ dev = [ ] [project.scripts] +clawbodylocal = "reachy_mini_openclaw.main:main" +# Compatibility alias for existing local launchers and operator muscle memory. clawbody = "reachy_mini_openclaw.main:main" [project.entry-points."reachy_mini_apps"] -clawbody = "reachy_mini_openclaw.main:ClawBodyApp" +clawbodylocal = "reachy_mini_openclaw.main:ReachyMiniOpenclaw" [project.urls] -Homepage = "https://github.com/yourusername/clawbody" -Documentation = "https://github.com/yourusername/clawbody#readme" -Repository = "https://github.com/yourusername/clawbody" -Issues = "https://github.com/yourusername/clawbody/issues" +Homepage = "https://huggingface.co/spaces/asmolebot/clawbodylocal" +Documentation = "https://github.com/asmolebot/clawbody#readme" +Repository = "https://github.com/asmolebot/clawbody" +Issues = "https://github.com/asmolebot/clawbody/issues" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/reachy_mini_openclaw/attention.py b/src/reachy_mini_openclaw/attention.py new file mode 100644 index 0000000..095faa3 --- /dev/null +++ b/src/reachy_mini_openclaw/attention.py @@ -0,0 +1,176 @@ +"""Quiet, consent-driven attention notifications for ClawBody Local.""" + +from __future__ import annotations + +import asyncio +import logging +import math +import time +from dataclasses import dataclass +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + + +@dataclass +class TapDetector: + """Detect a deliberate double tap from Reachy's head IMU.""" + + acceleration_delta: float = 2.4 + gyroscope_threshold: float = 0.55 + min_gap: float = 0.12 + max_gap: float = 0.85 + refractory: float = 1.5 + baseline_alpha: float = 0.04 + _baseline: float | None = None + _first_impact_at: float | None = None + _last_impact_at: float = -100.0 + _refractory_until: float = 0.0 + + def feed(self, imu: dict[str, Any] | None, now: float | None = None) -> bool: + if not imu: + return False + now = time.monotonic() if now is None else now + acceleration = imu.get("accelerometer") or [] + gyroscope = imu.get("gyroscope") or [] + if len(acceleration) != 3 or len(gyroscope) != 3: + return False + accel_norm = math.sqrt(sum(float(value) ** 2 for value in acceleration)) + gyro_norm = math.sqrt(sum(float(value) ** 2 for value in gyroscope)) + if self._baseline is None: + self._baseline = accel_norm + return False + delta = abs(accel_norm - self._baseline) + if delta < self.acceleration_delta: + self._baseline = ( + (1.0 - self.baseline_alpha) * self._baseline + + self.baseline_alpha * accel_norm + ) + impact = delta >= self.acceleration_delta or gyro_norm >= self.gyroscope_threshold + if not impact or now < self._refractory_until: + if self._first_impact_at is not None and now - self._first_impact_at > self.max_gap: + self._first_impact_at = None + return False + if now - self._last_impact_at < self.min_gap: + return False + self._last_impact_at = now + if self._first_impact_at is None: + self._first_impact_at = now + return False + gap = now - self._first_impact_at + self._first_impact_at = None + if self.min_gap <= gap <= self.max_gap: + self._refractory_until = now + self.refractory + return True + self._first_impact_at = now + return False + + +class AttentionClient: + def __init__(self, base_url: str, token: str, timeout: float = 8.0) -> None: + self._http = httpx.AsyncClient( + base_url=base_url.rstrip("/"), + headers={"Authorization": f"Bearer {token}"}, + timeout=timeout, + ) + + async def pending(self) -> dict[str, Any]: + response = await self._http.get("/v1/pending") + response.raise_for_status() + payload = response.json() + return payload if isinstance(payload, dict) else {} + + async def acknowledge(self, event_ids: list[str]) -> None: + response = await self._http.post("/v1/ack", json={"eventIds": event_ids}) + response.raise_for_status() + + async def close(self) -> None: + await self._http.aclose() + + +def format_attention(snapshot: dict[str, Any]) -> str: + needs_attention = snapshot.get("needsAttention") or [] + actions_taken = snapshot.get("actionsTaken") or [] + parts: list[str] = [] + if needs_attention: + parts.append("Here's what needs your attention.") + for item in needs_attention[:6]: + source = "Gmail" if item.get("source") == "gmail" else "Annexus" + sender = str(item.get("sender") or "unknown sender") + subject = str(item.get("subject") or "no subject") + parts.append(f"{source}, from {sender}: {subject}.") + remaining = len(needs_attention) - 6 + if remaining > 0: + parts.append(f"There {'is' if remaining == 1 else 'are'} {remaining} more item{'s' if remaining != 1 else ''}.") + if actions_taken: + parts.append("Actions taken.") + parts.extend(str(action) for action in actions_taken[:6]) + remaining = len(actions_taken) - 6 + if remaining > 0: + parts.append(f"There {'is' if remaining == 1 else 'are'} {remaining} more action{'s' if remaining != 1 else ''}.") + return " ".join(parts) or "There is nothing waiting for your attention." + + +class AttentionController: + def __init__( + self, + *, + robot: Any, + movement_manager: Any, + handler: Any, + client: AttentionClient, + poll_seconds: float = 60.0, + tap_detector: TapDetector | None = None, + ) -> None: + self.robot = robot + self.movement_manager = movement_manager + self.handler = handler + self.client = client + self.poll_seconds = max(5.0, poll_seconds) + self.tap_detector = tap_detector or TapDetector() + self._snapshot: dict[str, Any] = {} + self._last_poll = -self.poll_seconds + self._stopping = False + + async def run(self) -> None: + try: + while not self._stopping: + now = time.monotonic() + if now - self._last_poll >= self.poll_seconds: + self._last_poll = now + try: + self._snapshot = await self.client.pending() + self.movement_manager.set_attention_pending( + bool(self._snapshot.get("hasPending")) + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("Attention queue poll failed: %s", exc) + + if self._snapshot.get("hasPending"): + status = self.movement_manager.get_status() + can_accept_tap = not status.get("is_listening") and not status.get("processing") + if can_accept_tap and self.tap_detector.feed(self.robot.imu, now): + await self._read_pending() + await asyncio.sleep(0.05 if self._snapshot.get("hasPending") else 0.5) + finally: + self.movement_manager.set_attention_pending(False) + await self.client.close() + + async def _read_pending(self) -> None: + event_ids = [str(value) for value in self._snapshot.get("eventIds") or []] + if not event_ids: + return + logger.info("Attention double-tap detected; reading %d queued event(s)", len(event_ids)) + text = format_attention(self._snapshot) + duration = await self.handler.speak_text(text) + await asyncio.sleep(duration + 0.4) + await self.client.acknowledge(event_ids) + self._snapshot = {} + self.movement_manager.set_attention_pending(False) + + async def stop(self) -> None: + self._stopping = True diff --git a/src/reachy_mini_openclaw/audio/conversation.py b/src/reachy_mini_openclaw/audio/conversation.py index ac9069f..243dd7f 100644 --- a/src/reachy_mini_openclaw/audio/conversation.py +++ b/src/reachy_mini_openclaw/audio/conversation.py @@ -4,6 +4,7 @@ import asyncio import base64 +from difflib import SequenceMatcher import logging import re import time @@ -31,6 +32,43 @@ def needs_vision(text: str) -> bool: return any(pattern.search(text) for pattern in _VISION_REQUESTS) +def command_after_wake_phrase(text: str, wake_phrase: str) -> str | None: + """Return the command after a spoken wake phrase, or None if absent. + + Matching is case-insensitive, restricted to the start of the transcript, + and tolerant of punctuation inserted by STT (for example, "Hey, Asmo"). + """ + words = re.findall(r"[^\W_]+", wake_phrase, flags=re.UNICODE) + if not words: + return None + spoken = list(re.finditer(r"[^\W_]+", text, flags=re.UNICODE)) + if len(spoken) < len(words): + return None + + for expected, observed_match in zip(words, spoken[: len(words)], strict=True): + observed = observed_match.group(0) + expected_folded = expected.casefold() + observed_folded = observed.casefold() + if observed_folded == expected_folded: + continue + # Short wake words are occasionally normalized by Whisper ("hey" -> + # "hi"). Names are even more vulnerable ("Asmo" -> "asthma"). + # Keep the fuzzy allowance narrow: it only applies to the leading wake + # phrase tokens and still requires the entire configured phrase. + if expected_folded == "hey" and observed_folded == "hi": + continue + threshold = 0.60 if len(expected_folded) >= 4 else 0.75 + if ( + abs(len(expected_folded) - len(observed_folded)) > 2 + or SequenceMatcher(None, expected_folded, observed_folded).ratio() < threshold + ): + return None + + return re.sub( + r"^[\s,;:!?.\-–—]+", "", text[spoken[len(words) - 1].end() :] + ).strip() + + @dataclass(frozen=True) class Utterance: sample_rate: int @@ -155,6 +193,8 @@ def __init__( output_chunk_samples: int = 2048, attention_provider: Any = None, require_hardware_speech: bool = True, + wake_phrase_enabled: bool = False, + wake_phrase: str = "Hey Asmo", ) -> None: self.stt = stt self.tts = tts @@ -164,10 +204,47 @@ def __init__( self.output_chunk_samples = output_chunk_samples self.attention_provider = attention_provider self.require_hardware_speech = require_hardware_speech + self.wake_phrase_enabled = wake_phrase_enabled + self.wake_phrase = wake_phrase self.output_queue: asyncio.Queue[tuple[int, NDArray[np.int16]] | dict[str, str]] = asyncio.Queue() self._utterances: asyncio.Queue[Utterance] = asyncio.Queue(maxsize=2) self._shutdown_requested = False self._ignore_input_until = 0.0 + self._turn_lock = asyncio.Lock() + + async def _retry_async(self, operation: Any, label: str, attempts: int = 3) -> Any: + """Retry a transient network/provider operation with bounded backoff.""" + delay = 0.25 + for attempt in range(1, attempts + 1): + try: + return await operation() + except asyncio.CancelledError: + raise + except Exception: + if attempt == attempts: + raise + logger.warning( + "%s unavailable; retrying in %.2fs (%d/%d)", + label, + delay, + attempt, + attempts - 1, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 2.0) + raise AssertionError("retry loop exhausted") + + async def _ensure_gateway(self) -> None: + """Reconnect the gateway without making a failed turn fatal immediately.""" + if self.openclaw_bridge.is_connected: + return + + async def connect() -> bool: + if await self.openclaw_bridge.connect(): + return True + raise RuntimeError("OpenClaw gateway is unavailable") + + await self._retry_async(connect, "OpenClaw gateway") async def start_up(self) -> None: while not self._shutdown_requested: @@ -213,19 +290,34 @@ async def receive(self, frame: tuple[int, NDArray[Any]]) -> None: await self._utterances.put(utterance) async def _process_utterance(self, utterance: Utterance) -> None: + async with self._turn_lock: + await self._process_utterance_locked(utterance) + + async def _process_utterance_locked(self, utterance: Utterance) -> None: turn_started = time.monotonic() self.deps.movement_manager.set_processing(True) - transcript = await self.stt.transcribe(utterance.samples, utterance.sample_rate) + transcript = await self._retry_async( + lambda: self.stt.transcribe(utterance.samples, utterance.sample_rate), + "STT service", + ) stt_done = time.monotonic() if not transcript: self.deps.movement_manager.set_processing(False) return + transcript = transcript.strip() + if self.wake_phrase_enabled: + command = command_after_wake_phrase(transcript, self.wake_phrase) + if command is None: + logger.info("Ignored utterance without configured wake phrase: %r", transcript) + self.deps.movement_manager.set_processing(False) + return + # A wake phrase by itself remains a valid greeting. When a command + # follows it, keep the conversational session free of boilerplate. + transcript = command or transcript logger.info("User: %s", transcript) await self.output_queue.put({"role": "user", "content": transcript}) - if not self.openclaw_bridge.is_connected: - if not await self.openclaw_bridge.connect(): - raise RuntimeError("OpenClaw gateway is unavailable") + await self._ensure_gateway() motion_catalog = getattr(self.deps, "motion_catalog", None) motion_hint = f" {motion_catalog.response_hint()}" if motion_catalog is not None else "" image_b64 = None @@ -260,7 +352,7 @@ async def _process_utterance(self, utterance: Utterance) -> None: raise RuntimeError("OpenClaw returned an empty response") logger.info("Assistant: %s", text) - audio = await self.tts.synthesize(text) + audio = await self._retry_async(lambda: self.tts.synthesize(text), "TTS service") tts_done = time.monotonic() # Reachy's microphone hears its own speaker over the remote WebRTC # stream. Suppress VAD for the synthesized duration plus a short room- @@ -292,6 +384,24 @@ async def _process_utterance(self, utterance: Utterance) -> None: tts_done - turn_started, ) + async def speak_text(self, text: str) -> float: + """Speak trusted local notification text without creating an agent turn.""" + async with self._turn_lock: + self.deps.movement_manager.set_processing(False) + audio = await self._retry_async(lambda: self.tts.synthesize(text), "TTS service") + duration = audio.samples.size / audio.sample_rate + self._ignore_input_until = max( + self._ignore_input_until, + time.monotonic() + duration + 0.8, + ) + self.vad.reset() + self.deps.movement_manager.set_listening(False) + await self.output_queue.put({"role": "assistant", "content": text}) + for offset in range(0, audio.samples.size, self.output_chunk_samples): + chunk = audio.samples[offset : offset + self.output_chunk_samples] + await self.output_queue.put((audio.sample_rate, chunk.reshape(1, -1))) + return duration + def _clear_audio_output(self) -> None: retained: list[dict[str, str]] = [] while not self.output_queue.empty(): diff --git a/src/reachy_mini_openclaw/config.py b/src/reachy_mini_openclaw/config.py index 8deb14c..c5f23f1 100644 --- a/src/reachy_mini_openclaw/config.py +++ b/src/reachy_mini_openclaw/config.py @@ -37,7 +37,7 @@ class Config: # Speech-to-text. The endpoint follows OpenAI's multipart # /v1/audio/transcriptions contract, but can be entirely local. - STT_BASE_URL: str = field(default_factory=lambda: os.getenv("STT_BASE_URL", "http://speech-host.local:8890/v1")) + STT_BASE_URL: str = field(default_factory=lambda: os.getenv("STT_BASE_URL", "")) STT_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("STT_API_KEY")) STT_MODEL: str = field(default_factory=lambda: os.getenv("STT_MODEL", "distil-large-v3")) STT_LANGUAGE: str = field(default_factory=lambda: os.getenv("STT_LANGUAGE", "en")) @@ -46,7 +46,7 @@ class Config: # Text-to-speech. Chatterbox is primary; ElevenLabs can be enabled as a # fallback without changing the conversational pipeline. TTS_PROVIDER: str = field(default_factory=lambda: os.getenv("TTS_PROVIDER", "chatterbox").lower()) - CHATTERBOX_URL: str = field(default_factory=lambda: os.getenv("CHATTERBOX_URL", "http://speech-host.local:8890/v1/audio/speech")) + CHATTERBOX_URL: str = field(default_factory=lambda: os.getenv("CHATTERBOX_URL", "")) CHATTERBOX_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("CHATTERBOX_API_KEY")) CHATTERBOX_VOICE: str = field(default_factory=lambda: os.getenv("CHATTERBOX_VOICE", "asmo")) CHATTERBOX_SAMPLE_RATE: int = field(default_factory=lambda: int(os.getenv("CHATTERBOX_SAMPLE_RATE", "24000"))) @@ -67,9 +67,20 @@ class Config: VAD_MIN_SPEECH_MS: int = field(default_factory=lambda: int(os.getenv("VAD_MIN_SPEECH_MS", "300"))) VAD_MAX_SPEECH_SECONDS: float = field(default_factory=lambda: float(os.getenv("VAD_MAX_SPEECH_SECONDS", "20"))) VAD_REQUIRE_HARDWARE_SPEECH: bool = field(default_factory=lambda: os.getenv("VAD_REQUIRE_HARDWARE_SPEECH", "true").lower() == "true") + + # Optional transcript-level wake phrase gate. Audio is still transcribed + # locally, but only utterances beginning with the phrase reach OpenClaw. + ENABLE_WAKE_PHRASE: bool = field(default_factory=lambda: os.getenv("ENABLE_WAKE_PHRASE", "false").lower() == "true") + WAKE_PHRASE: str = field(default_factory=lambda: os.getenv("WAKE_PHRASE", "Hey Asmo")) + + # Hourly mail attention queue. The broker is gateway-local and uses the + # same bearer token already provisioned for OpenClaw. + ENABLE_ATTENTION_QUEUE: bool = field(default_factory=lambda: os.getenv("ENABLE_ATTENTION_QUEUE", "false").lower() == "true") + ATTENTION_API_URL: str = field(default_factory=lambda: os.getenv("ATTENTION_API_URL", "")) + ATTENTION_POLL_SECONDS: float = field(default_factory=lambda: float(os.getenv("ATTENTION_POLL_SECONDS", "60"))) # OpenClaw Gateway Configuration - OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789")) + OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", "")) OPENCLAW_TOKEN: Optional[str] = field(default_factory=_gateway_token) OPENCLAW_AGENT_ID: str = field(default_factory=lambda: os.getenv("OPENCLAW_AGENT_ID", "main")) # Spoken body turns should feel immediate. "off" is OpenClaw's lowest- @@ -106,11 +117,13 @@ class Config: # Custom Profile (for personality customization) CUSTOM_PROFILE: Optional[str] = field(default_factory=lambda: os.getenv("REACHY_MINI_CUSTOM_PROFILE")) - def validate(self) -> list[str]: + def validate(self, gateway_url: Optional[str] = None) -> list[str]: """Validate configuration and return list of errors.""" errors = [] if not self.STT_BASE_URL: errors.append("STT_BASE_URL is required") + if not (gateway_url or self.OPENCLAW_GATEWAY_URL): + errors.append("OPENCLAW_GATEWAY_URL is required") if self.TTS_PROVIDER not in {"chatterbox", "elevenlabs"}: errors.append("TTS_PROVIDER must be 'chatterbox' or 'elevenlabs'") if self.TTS_PROVIDER == "chatterbox" and not self.CHATTERBOX_URL: @@ -122,6 +135,14 @@ def validate(self) -> list[str]: errors.append("ELEVENLABS_VOICE_ID is required for ElevenLabs TTS") if self.AUDIO_OUTPUT_GAIN <= 0: errors.append("AUDIO_OUTPUT_GAIN must be greater than zero") + if self.ENABLE_WAKE_PHRASE and not self.WAKE_PHRASE.strip(): + errors.append("WAKE_PHRASE must not be empty when wake phrase gating is enabled") + if self.ENABLE_ATTENTION_QUEUE and not self.ATTENTION_API_URL: + errors.append("ATTENTION_API_URL is required when attention queue is enabled") + if self.ENABLE_ATTENTION_QUEUE and not self.OPENCLAW_TOKEN: + errors.append("OPENCLAW_TOKEN is required when attention queue is enabled") + if self.ATTENTION_POLL_SECONDS < 5: + errors.append("ATTENTION_POLL_SECONDS must be at least 5") if self.OPENCLAW_THINKING not in { "off", "minimal", "low", "medium", "high", "xhigh", "adaptive", "max" }: diff --git a/src/reachy_mini_openclaw/gradio_app.py b/src/reachy_mini_openclaw/gradio_app.py index bf02fb6..5bbc1fc 100644 --- a/src/reachy_mini_openclaw/gradio_app.py +++ b/src/reachy_mini_openclaw/gradio_app.py @@ -7,8 +7,9 @@ - Manual control options """ -import os +import asyncio import logging +import threading from typing import Optional import gradio as gr @@ -17,7 +18,7 @@ def launch_gradio( - gateway_url: str = "ws://localhost:18789", + gateway_url: str = "", robot_name: Optional[str] = None, robot_host: Optional[str] = None, robot_port: Optional[int] = None, @@ -47,19 +48,21 @@ def launch_gradio( # State app_instance = None + app_loop = None def start_conversation(): """Start the conversation.""" - nonlocal app_instance - + nonlocal app_instance, app_loop + from reachy_mini_openclaw.main import ClawBodyCore - import asyncio - import threading if app_instance is not None: return "Already running" try: + config.ENABLE_FACE_TRACKING = enable_face_tracking + if head_tracker_type is not None: + config.HEAD_TRACKER_TYPE = head_tracker_type app_instance = ClawBodyCore( gateway_url=gateway_url, robot_name=robot_name, @@ -68,22 +71,29 @@ def start_conversation(): robot_connection_mode=robot_connection_mode, enable_camera=enable_camera, enable_openclaw=enable_openclaw, - enable_face_tracking=enable_face_tracking, - head_tracker_type=head_tracker_type, ) # Run in background thread - def run_app(): + def run_app(instance): + nonlocal app_instance, app_loop loop = asyncio.new_event_loop() + app_loop = loop asyncio.set_event_loop(loop) try: - loop.run_until_complete(app_instance.run()) + loop.run_until_complete(instance.run()) + except asyncio.CancelledError: + logger.info("App stopped") except Exception as e: logger.error("App error: %s", e) finally: + loop.run_until_complete(instance.stop()) + if app_instance is instance: + app_instance = None + app_loop = None + asyncio.set_event_loop(None) loop.close() - thread = threading.Thread(target=run_app, daemon=True) + thread = threading.Thread(target=run_app, args=(app_instance,), daemon=True) thread.start() return "Started successfully" @@ -92,13 +102,16 @@ def run_app(): def stop_conversation(): """Stop the conversation.""" - nonlocal app_instance + nonlocal app_instance, app_loop if app_instance is None: return "Not running" try: - app_instance.stop() + if app_loop is None or not app_loop.is_running(): + return "App is still starting" + future = asyncio.run_coroutine_threadsafe(app_instance.stop(), app_loop) + future.result(timeout=10) app_instance = None return "Stopped" except Exception as e: diff --git a/src/reachy_mini_openclaw/main.py b/src/reachy_mini_openclaw/main.py index f25b232..b7538a8 100644 --- a/src/reachy_mini_openclaw/main.py +++ b/src/reachy_mini_openclaw/main.py @@ -17,7 +17,6 @@ """ import os -import sys import time import asyncio import base64 @@ -28,10 +27,11 @@ from typing import Any, Optional from dotenv import load_dotenv +from reachy_mini import ReachyMiniApp -# Load environment from project root (override=True ensures .env takes precedence) +# Load local development settings without overriding deployment-injected env. _project_root = Path(__file__).parent.parent.parent -load_dotenv(_project_root / ".env", override=True) +load_dotenv(_project_root / ".env", override=False) logger = logging.getLogger(__name__) @@ -117,8 +117,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--gateway-url", type=str, - default=os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789"), - help="OpenClaw gateway URL (from OPENCLAW_GATEWAY_URL env or default)" + default=os.getenv("OPENCLAW_GATEWAY_URL", ""), + help="OpenClaw gateway URL (required unless supplied by environment)" ) parser.add_argument( "--no-camera", @@ -161,7 +161,7 @@ class ClawBodyCore: def __init__( self, - gateway_url: str = "ws://localhost:18789", + gateway_url: str = "", robot_name: Optional[str] = None, robot_host: Optional[str] = None, robot_port: Optional[int] = None, @@ -205,11 +205,11 @@ def __init__( self._owns_robot = robot is None # Validate configuration - errors = config.validate() + errors = config.validate(gateway_url=self.gateway_url) if errors: for error in errors: logger.error("Config error: %s", error) - sys.exit(1) + raise ValueError("Invalid ClawBody configuration") # Connect to robot if robot is not None: @@ -229,10 +229,10 @@ def __init__( except TimeoutError as e: logger.error("Connection timeout: %s", e) logger.error("Check that the robot is powered on and reachable.") - sys.exit(1) + raise RuntimeError("Reachy Mini connection timed out") from e except Exception as e: logger.error("Robot connection failed: %s", e) - sys.exit(1) + raise RuntimeError("Reachy Mini connection failed") from e logger.info("Connected to robot: %s", self.robot.client.get_status()) @@ -279,7 +279,11 @@ def __init__( reachy_mini=self.robot, head_tracker=self.head_tracker, daemon_tracking=daemon_tracking, - daemon_url=f"http://{robot_host or config.ROBOT_HOST}:{robot_port or config.ROBOT_PORT}", + daemon_url=( + "http://127.0.0.1:8000" + if robot is not None + else f"http://{robot_host or config.ROBOT_HOST}:{robot_port or config.ROBOT_PORT}" + ), tracking_weight=config.FACE_TRACKING_WEIGHT, sound_tracking=config.ENABLE_SOUND_TRACKING, ) @@ -351,7 +355,24 @@ def __init__( ), attention_provider=self.camera_worker, require_hardware_speech=config.VAD_REQUIRE_HARDWARE_SPEECH, + wake_phrase_enabled=config.ENABLE_WAKE_PHRASE, + wake_phrase=config.WAKE_PHRASE, ) + + self.attention_controller = None + if config.ENABLE_ATTENTION_QUEUE: + from reachy_mini_openclaw.attention import AttentionClient, AttentionController + + self.attention_controller = AttentionController( + robot=self.robot, + movement_manager=self.movement_manager, + handler=self.handler, + client=AttentionClient( + config.ATTENTION_API_URL, + config.OPENCLAW_TOKEN or "", + ), + poll_seconds=config.ATTENTION_POLL_SECONDS, + ) # State self._stop_event = asyncio.Event() @@ -451,12 +472,23 @@ async def record_loop(self) -> None: """Read audio from robot microphone and send to handler.""" input_sr = self.robot.media.get_input_audio_samplerate() logger.info("Recording at %d Hz", input_sr) - + failures = 0 while not self._should_stop(): - audio_frame = self.robot.media.get_audio_sample() - if audio_frame is not None: - await self.handler.receive((input_sr, audio_frame)) - await asyncio.sleep(0.01) + try: + audio_frame = self.robot.media.get_audio_sample() + if audio_frame is not None: + await self.handler.receive((input_sr, audio_frame)) + failures = 0 + await asyncio.sleep(0.01) + except asyncio.CancelledError: + raise + except Exception as exc: + failures += 1 + if failures > 5: + raise RuntimeError("Reachy microphone loop failed repeatedly") from exc + delay = min(0.25 * (2 ** (failures - 1)), 4.0) + logger.warning("Reachy microphone unavailable; retrying in %.2fs", delay) + await asyncio.sleep(delay) async def play_loop(self) -> None: """Play audio from handler through robot speakers.""" @@ -464,37 +496,44 @@ async def play_loop(self) -> None: output_sr = self.robot.media.get_output_audio_samplerate() logger.info("Playing at %d Hz with %.2fx software gain", output_sr, self.audio_output_gain) - + failures = 0 while not self._should_stop(): - output = await self.handler.emit() - if output is not None: - if isinstance(output, tuple): - input_sr, audio_data = output - - # Feed the speech animator at playback time, not synthesis - # time, so movement stays synchronized with the speaker. - self.head_wobbler.feed( - base64.b64encode(audio_data.astype("int16").tobytes()).decode("ascii") - ) - - # Convert provider PCM16 to normalized float32. - audio_data = audio_data.flatten().astype("float32") / 32768.0 - - # The old donor project attenuated every response to 50%, - # independently of Reachy's console volume. Preserve the - # synthesized level by default and permit controlled boost. - audio_data = apply_output_gain(audio_data, self.audio_output_gain) - - # Resample if needed - if input_sr != output_sr: - from scipy.signal import resample - num_samples = int(len(audio_data) * output_sr / input_sr) - audio_data = resample(audio_data, num_samples).astype("float32") - - self.robot.media.push_audio_sample(audio_data) - # Otherwise it is a transcript event for a future UI consumer. - - await asyncio.sleep(0.01) + try: + output = await self.handler.emit() + if output is not None: + if isinstance(output, tuple): + input_sr, audio_data = output + # Feed the speech animator at playback time, not synthesis + # time, so movement stays synchronized with the speaker. + self.head_wobbler.feed( + base64.b64encode(audio_data.astype("int16").tobytes()).decode("ascii") + ) + + # Convert provider PCM16 to normalized float32. + audio_data = audio_data.flatten().astype("float32") / 32768.0 + + # Preserve synthesized level by default and permit a + # controlled boost without clipping. + audio_data = apply_output_gain(audio_data, self.audio_output_gain) + + if input_sr != output_sr: + from scipy.signal import resample + num_samples = int(len(audio_data) * output_sr / input_sr) + audio_data = resample(audio_data, num_samples).astype("float32") + + self.robot.media.push_audio_sample(audio_data) + # Otherwise it is a transcript event for a future UI consumer. + failures = 0 + await asyncio.sleep(0.01) + except asyncio.CancelledError: + raise + except Exception as exc: + failures += 1 + if failures > 5: + raise RuntimeError("Reachy speaker loop failed repeatedly") from exc + delay = min(0.25 * (2 ** (failures - 1)), 4.0) + logger.warning("Reachy speaker unavailable; retrying in %.2fs", delay) + await asyncio.sleep(delay) async def run(self) -> None: """Run the main application loop.""" @@ -518,7 +557,7 @@ async def run(self) -> None: duration=2.0, body_yaw=0.0, ) - time.sleep(2) # Wait for goto to complete + await asyncio.sleep(2) # Keep native stop handling responsive. logger.info("Robot at neutral position with motors enabled") except Exception as e: logger.error("Failed to initialize robot pose: %s", e) @@ -547,7 +586,7 @@ async def run(self) -> None: logger.info("Starting audio...") self.robot.media.start_recording() self.robot.media.start_playing() - time.sleep(1) # Let pipelines initialize + await asyncio.sleep(1) # Let pipelines initialize without blocking stop handling. logger.info("Ready! Speak to me...") @@ -560,19 +599,58 @@ async def run(self) -> None: asyncio.create_task(self.record_loop(), name="record-loop"), asyncio.create_task(self.play_loop(), name="play-loop"), ] + if self.attention_controller is not None: + self._tasks.append( + asyncio.create_task(self.attention_controller.run(), name="attention-loop") + ) + stop_watcher = asyncio.create_task(self._watch_external_stop(), name="external-stop-watcher") + self._tasks.append(stop_watcher) + try: - await asyncio.gather(*self._tasks) + done, pending = await asyncio.wait( + self._tasks, + return_when=asyncio.FIRST_COMPLETED, + ) + if stop_watcher in done: + logger.info("External stop requested") + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + # Do not hide an unexpected task failure from the Reachy app + # manager; it must be able to record the failure and restart us. + await asyncio.gather(*done) except asyncio.CancelledError: logger.info("Tasks cancelled") - - def stop(self) -> None: - """Stop everything.""" + raise + + async def _watch_external_stop(self) -> None: + """Bridge Reachy's threading stop event into the asyncio lifecycle.""" + if self._external_stop_event is None: + await self._stop_event.wait() + return + # Poll instead of delegating Event.wait() to the default executor. + # Cancelling asyncio.to_thread() cannot stop the underlying blocking + # thread, which could survive app shutdown indefinitely. + while not self._external_stop_event.is_set() and not self._stop_event.is_set(): + await asyncio.sleep(0.1) + self._stop_event.set() + + async def stop(self) -> None: + """Stop everything exactly once.""" + if getattr(self, "_shutdown_complete", False): + return + self._shutdown_complete = True logger.info("Stopping...") self._stop_event.set() # Cancel tasks + if self.attention_controller is not None: + await self.attention_controller.stop() + current_task = asyncio.current_task() for task in self._tasks: + if task is current_task: + continue if not task.done(): task.cancel() @@ -591,9 +669,7 @@ def stop(self) -> None: # Disconnect OpenClaw bridge if self.openclaw_bridge is not None: try: - asyncio.get_event_loop().run_until_complete( - self.openclaw_bridge.disconnect() - ) + await self.openclaw_bridge.disconnect() except Exception as e: logger.debug("OpenClaw disconnect: %s", e) @@ -608,7 +684,7 @@ def stop(self) -> None: logger.info("Stopped") -class ClawBodyApp: +class ReachyMiniOpenclaw(ReachyMiniApp): """ClawBody - Reachy Mini Apps entry point. This class allows ClawBody to be installed and run from @@ -628,23 +704,28 @@ def run(self, reachy_mini, stop_event: threading.Event) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - gateway_url = os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789") - - app = ClawBodyCore( - gateway_url=gateway_url, - robot=reachy_mini, - external_stop_event=stop_event, - ) + gateway_url = os.getenv("OPENCLAW_GATEWAY_URL", "") + app = None try: + app = ClawBodyCore( + gateway_url=gateway_url, + robot=reachy_mini, + external_stop_event=stop_event, + ) loop.run_until_complete(app.run()) - except Exception as e: - logger.error("Error running app: %s", e) finally: - app.stop() + if app is not None: + loop.run_until_complete(app.stop()) + asyncio.set_event_loop(None) loop.close() +# Backwards-compatible import for older local launchers. The package entry +# point intentionally uses the official ReachyMiniOpenclaw name above. +ClawBodyApp = ReachyMiniOpenclaw + + def main() -> None: """Main entry point.""" args = parse_args() @@ -690,12 +771,16 @@ def main() -> None: enable_openclaw=not args.no_openclaw, ) + async def run_and_stop() -> None: + try: + await app.run() + finally: + await app.stop() + try: - asyncio.run(app.run()) + asyncio.run(run_and_stop()) except KeyboardInterrupt: logger.info("Interrupted") - finally: - app.stop() if __name__ == "__main__": diff --git a/src/reachy_mini_openclaw/moves.py b/src/reachy_mini_openclaw/moves.py index 5e382db..7812cea 100644 --- a/src/reachy_mini_openclaw/moves.py +++ b/src/reachy_mini_openclaw/moves.py @@ -284,6 +284,7 @@ def __init__( self._processing_start_time = 0.0 self._thinking_amplitude = 0.0 # 0..1 envelope for smooth fade in/out self._thinking_antenna_offsets: Tuple[float, float] = (0.0, 0.0) + self._attention_pending = False # Shared state lock self._shared_lock = threading.Lock() @@ -316,6 +317,10 @@ def set_processing(self, processing: bool) -> None: Face tracking continues underneath since this is additive. """ self._command_queue.put(("set_processing", processing)) + + def set_attention_pending(self, pending: bool) -> None: + """Hold one antenna in a quiet, visible notification pose.""" + self._command_queue.put(("set_attention_pending", pending)) def is_idle(self) -> bool: """Check if robot has been idle. Thread-safe.""" @@ -435,6 +440,12 @@ def _handle_command(self, cmd: str, payload: Any, current_time: float) -> None: # Amplitude will decay smoothly in _update_thinking_offsets self.state.update_activity() logger.debug("Processing ended - thinking animation decaying") + elif cmd == "set_attention_pending": + desired = bool(payload) + if self._attention_pending != desired: + self._attention_pending = desired + self.state.update_activity() + logger.info("Attention indicator %s", "enabled" if desired else "cleared") def _manage_move_queue(self, current_time: float) -> None: """Advance the move queue.""" @@ -573,6 +584,11 @@ def _blend_antennas(self, target: Tuple[float, float]) -> Tuple[float, float]: """Blend antennas with listening freeze state.""" if self._is_listening: return self._listening_antennas + + if self._attention_pending and not self._processing: + # A stable asymmetric "bookmark" pose: visible across the room, + # silent, and distinct from breathing/thinking animations. + target = (float(np.deg2rad(-42)), float(np.deg2rad(10))) # Blend back from freeze blend = min(1.0, self._antenna_unfreeze_blend + self.target_period / self._antenna_blend_duration) @@ -680,6 +696,7 @@ def get_status(self) -> Dict[str, Any]: "is_listening": self._is_listening, "breathing_active": self._breathing_active, "processing": self._processing, + "attention_pending": self._attention_pending, "thinking_amplitude": round(self._thinking_amplitude, 3), "last_commanded_pose": { "head": self._last_commanded_pose[0].tolist(), diff --git a/tests/test_attention.py b/tests/test_attention.py new file mode 100644 index 0000000..3a5ae1d --- /dev/null +++ b/tests/test_attention.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from reachy_mini_openclaw.attention import TapDetector, format_attention + + +def imu(acceleration: list[float], gyroscope: list[float] | None = None) -> dict: + return { + "accelerometer": acceleration, + "gyroscope": gyroscope or [0.0, 0.0, 0.0], + } + + +def test_double_tap_detector_requires_two_distinct_impacts() -> None: + detector = TapDetector(acceleration_delta=2.0) + assert not detector.feed(imu([0.0, 0.0, 9.8]), 0.0) + assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.0) + assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.05) + assert detector.feed(imu([0.0, 0.0, 13.0]), 1.35) + + +def test_taps_too_far_apart_do_not_trigger() -> None: + detector = TapDetector(acceleration_delta=2.0, max_gap=0.8) + detector.feed(imu([0.0, 0.0, 9.8]), 0.0) + assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.0) + assert not detector.feed(imu([0.0, 0.0, 13.0]), 2.0) + + +def test_gyroscope_can_detect_deliberate_pat() -> None: + detector = TapDetector(gyroscope_threshold=0.5) + detector.feed(imu([0.0, 0.0, 9.8]), 0.0) + assert not detector.feed(imu([0.0, 0.0, 9.8], [0.7, 0.0, 0.0]), 1.0) + assert detector.feed(imu([0.0, 0.0, 9.8], [0.7, 0.0, 0.0]), 1.4) + + +def test_attention_summary_reads_both_sections() -> None: + text = format_attention( + { + "needsAttention": [ + { + "source": "gmail", + "sender": "Security", + "subject": "New login", + }, + { + "source": "annexus", + "sender": "Jules", + "subject": "Quote status", + }, + ], + "actionsTaken": ["Marked 3 low-priority Gmail messages as read."], + } + ) + assert "Gmail, from Security: New login" in text + assert "Annexus, from Jules: Quote status" in text + assert "Actions taken" in text + assert "Marked 3" in text diff --git a/tests/test_conversation.py b/tests/test_conversation.py index f0feda6..dcbff7c 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -1,7 +1,15 @@ +import asyncio import numpy as np import pytest - -from reachy_mini_openclaw.audio.conversation import ConversationHandler, EnergyVAD, Utterance, needs_vision +import threading + +from reachy_mini_openclaw.audio.conversation import ( + ConversationHandler, + EnergyVAD, + Utterance, + command_after_wake_phrase, + needs_vision, +) from reachy_mini_openclaw.audio.providers import SynthesizedAudio from reachy_mini_openclaw.openclaw_bridge import OpenClawResponse @@ -65,6 +73,19 @@ def test_visual_questions_request_a_camera_frame() -> None: assert not needs_vision("I see what you mean") +def test_wake_phrase_matching_is_prefix_only_and_punctuation_tolerant() -> None: + assert command_after_wake_phrase("Hey, Asmo! What time is it?", "Hey Asmo") == "What time is it?" + assert command_after_wake_phrase("hey asmo", "Hey Asmo") == "" + assert command_after_wake_phrase("Someone said hey Asmo", "Hey Asmo") is None + assert command_after_wake_phrase("Hey Asmodeus", "Hey Asmo") is None + + +def test_wake_phrase_tolerates_common_stt_mangling() -> None: + assert command_after_wake_phrase("Hey asthma, tell me a joke", "Hey Asmo") == "tell me a joke" + assert command_after_wake_phrase("Hi Esmo what time is it", "Hey Asmo") == "what time is it" + assert command_after_wake_phrase("They asked me a question", "Hey Asmo") is None + + class Movement: def __init__(self) -> None: self.processing = [] @@ -99,12 +120,42 @@ async def transcribe(self, samples, sample_rate) -> str: return "what time is it" +class StaticSTT: + def __init__(self, transcript: str) -> None: + self.transcript = transcript + + async def transcribe(self, samples, sample_rate) -> str: + return self.transcript + + class TTS: async def synthesize(self, text: str) -> SynthesizedAudio: assert text == "It is robot o'clock." return SynthesizedAudio(24000, np.arange(5000, dtype=np.int16)) +class FlakySTT(STT): + def __init__(self) -> None: + self.attempts = 0 + + async def transcribe(self, samples, sample_rate) -> str: + self.attempts += 1 + if self.attempts < 3: + raise ConnectionError("temporary STT outage") + return await super().transcribe(samples, sample_rate) + + +class FlakyTTS(TTS): + def __init__(self) -> None: + self.attempts = 0 + + async def synthesize(self, text: str) -> SynthesizedAudio: + self.attempts += 1 + if self.attempts < 3: + raise ConnectionError("temporary TTS outage") + return await super().synthesize(text) + + class Bridge: is_connected = True @@ -120,6 +171,35 @@ async def chat( return OpenClawResponse("It is robot o'clock.") +class FlakyBridge(Bridge): + def __init__(self) -> None: + self.is_connected = False + self.attempts = 0 + + async def connect(self) -> bool: + self.attempts += 1 + if self.attempts < 3: + return False + self.is_connected = True + return True + + +class RecordingBridge: + is_connected = True + + def __init__(self) -> None: + self.messages: list[str] = [] + + async def connect(self) -> bool: + return True + + async def chat( + self, message: str, image_b64: str | None, system_context: str + ) -> OpenClawResponse: + self.messages.append(message) + return OpenClawResponse("It is robot o'clock.") + + @pytest.mark.asyncio async def test_pipeline_routes_openclaw_response_to_tts_and_audio_queue() -> None: deps = Deps() @@ -144,6 +224,140 @@ async def test_pipeline_routes_openclaw_response_to_tts_and_audio_queue() -> Non assert deps.movement_manager.processing == [True, False] +@pytest.mark.asyncio +async def test_wake_phrase_gate_strips_phrase_before_openclaw() -> None: + deps = Deps() + bridge = RecordingBridge() + handler = ConversationHandler( + stt=StaticSTT("Hey, Asmo! What time is it?"), + tts=TTS(), + openclaw_bridge=bridge, + deps=deps, + vad=EnergyVAD(), + wake_phrase_enabled=True, + wake_phrase="Hey Asmo", + ) + + await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32))) + + assert bridge.messages == ["What time is it?"] + assert handler.output_queue.get_nowait() == {"role": "user", "content": "What time is it?"} + + +@pytest.mark.asyncio +async def test_wake_phrase_gate_ignores_ambient_transcript() -> None: + deps = Deps() + bridge = RecordingBridge() + handler = ConversationHandler( + stt=StaticSTT("the television is still talking"), + tts=TTS(), + openclaw_bridge=bridge, + deps=deps, + vad=EnergyVAD(), + wake_phrase_enabled=True, + wake_phrase="Hey Asmo", + ) + + await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32))) + + assert bridge.messages == [] + assert handler.output_queue.empty() + assert deps.movement_manager.processing == [True, False] + + +@pytest.mark.asyncio +async def test_pipeline_retries_transient_stt_and_tts_failures() -> None: + deps = Deps() + stt = FlakySTT() + tts = FlakyTTS() + handler = ConversationHandler( + stt=stt, + tts=tts, + openclaw_bridge=Bridge(), + deps=deps, + vad=EnergyVAD(), + ) + + await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32))) + + assert stt.attempts == 3 + assert tts.attempts == 3 + + +@pytest.mark.asyncio +async def test_pipeline_reconnects_gateway_with_bounded_backoff() -> None: + deps = Deps() + bridge = FlakyBridge() + handler = ConversationHandler( + stt=STT(), + tts=TTS(), + openclaw_bridge=bridge, + deps=deps, + vad=EnergyVAD(), + ) + + await handler._ensure_gateway() + + assert bridge.attempts == 3 + assert bridge.is_connected + + +@pytest.mark.asyncio +async def test_native_stop_watcher_does_not_spawn_a_blocking_executor_thread() -> None: + import reachy_mini_openclaw.main as native_main + + core = native_main.ClawBodyCore.__new__(native_main.ClawBodyCore) + core._external_stop_event = threading.Event() + core._stop_event = asyncio.Event() + + watcher = asyncio.create_task(core._watch_external_stop()) + await asyncio.sleep(0) + watcher.cancel() + with pytest.raises(asyncio.CancelledError): + await watcher + + +def test_native_wrapper_propagates_failure_after_cleanup(monkeypatch) -> None: + import reachy_mini_openclaw.main as native_main + + class FailingCore: + stopped = False + + def __init__(self, **kwargs) -> None: + pass + + async def run(self) -> None: + raise RuntimeError("simulated daemon loss") + + async def stop(self) -> None: + self.stopped = True + + monkeypatch.setattr(native_main, "ClawBodyCore", FailingCore) + app = native_main.ReachyMiniOpenclaw.__new__(native_main.ReachyMiniOpenclaw) + + with pytest.raises(RuntimeError, match="simulated daemon loss"): + app.run(object(), threading.Event()) + + +def test_native_config_requires_explicit_service_endpoints() -> None: + from reachy_mini_openclaw.config import Config + + config = Config(STT_BASE_URL="", OPENCLAW_GATEWAY_URL="", CHATTERBOX_URL="") + errors = config.validate() + + assert "STT_BASE_URL is required" in errors + assert "OPENCLAW_GATEWAY_URL is required" in errors + assert "CHATTERBOX_URL is required for Chatterbox TTS" in errors + + +def test_native_config_rejects_empty_enabled_wake_phrase() -> None: + from reachy_mini_openclaw.config import Config + + config = Config(ENABLE_WAKE_PHRASE=True, WAKE_PHRASE=" ") + + assert "WAKE_PHRASE must not be empty when wake phrase gating is enabled" in config.validate() + + @pytest.mark.asyncio async def test_microphone_is_suppressed_while_robot_audio_is_playing() -> None: deps = Deps() From 6a4e597a193532dc247b92c93b4d887bb727e793 Mon Sep 17 00:00:00 2001 From: Asmo Bot Date: Wed, 22 Jul 2026 15:52:53 -0700 Subject: [PATCH 3/4] Enable Reachy attention queue by default --- .env.example | 2 +- README.md | 3 +++ src/reachy_mini_openclaw/config.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 0e7cf5b..6384489 100644 --- a/.env.example +++ b/.env.example @@ -74,7 +74,7 @@ ENABLE_WAKE_PHRASE=false WAKE_PHRASE=Hey Asmo # Quiet mail-attention indicator and head double-tap playback. -ENABLE_ATTENTION_QUEUE=false +ENABLE_ATTENTION_QUEUE=true ATTENTION_API_URL=http://192.168.1.238:18790 ATTENTION_POLL_SECONDS=60 diff --git a/README.md b/README.md index 08cb5d2..c0c714c 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,9 @@ CHATTERBOX_URL=http://your-speech-host:8890/v1/audio/speech CHATTERBOX_VOICE=asmo ENABLE_WAKE_PHRASE=true WAKE_PHRASE=Hey Asmo +ENABLE_ATTENTION_QUEUE=true +ATTENTION_API_URL=http://192.168.1.238:18790 +ATTENTION_POLL_SECONDS=60 ``` The native app uses the SDK-owned robot connection and targets the local Reachy diff --git a/src/reachy_mini_openclaw/config.py b/src/reachy_mini_openclaw/config.py index c5f23f1..1370648 100644 --- a/src/reachy_mini_openclaw/config.py +++ b/src/reachy_mini_openclaw/config.py @@ -75,8 +75,8 @@ class Config: # Hourly mail attention queue. The broker is gateway-local and uses the # same bearer token already provisioned for OpenClaw. - ENABLE_ATTENTION_QUEUE: bool = field(default_factory=lambda: os.getenv("ENABLE_ATTENTION_QUEUE", "false").lower() == "true") - ATTENTION_API_URL: str = field(default_factory=lambda: os.getenv("ATTENTION_API_URL", "")) + ENABLE_ATTENTION_QUEUE: bool = field(default_factory=lambda: os.getenv("ENABLE_ATTENTION_QUEUE", "true").lower() == "true") + ATTENTION_API_URL: str = field(default_factory=lambda: os.getenv("ATTENTION_API_URL", "http://192.168.1.238:18790")) ATTENTION_POLL_SECONDS: float = field(default_factory=lambda: float(os.getenv("ATTENTION_POLL_SECONDS", "60"))) # OpenClaw Gateway Configuration From 517dd03af4ddbfc36229cf483381ef11f99f27a9 Mon Sep 17 00:00:00 2001 From: Asmo Bot Date: Wed, 22 Jul 2026 16:07:09 -0700 Subject: [PATCH 4/4] Fix Reachy native app module startup --- pyproject.toml | 2 +- src/reachy_mini_openclaw/main.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d5049f7..a1763ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "clawbodylocal" -version = "0.2.0" +version = "0.2.1" description = "Asmo's local OpenClaw body for Reachy Mini, with local STT, pluggable TTS, and expressive movement." readme = "README.md" license = {text = "Apache-2.0"} diff --git a/src/reachy_mini_openclaw/main.py b/src/reachy_mini_openclaw/main.py index b7538a8..2a74756 100644 --- a/src/reachy_mini_openclaw/main.py +++ b/src/reachy_mini_openclaw/main.py @@ -784,4 +784,9 @@ async def run_and_stop() -> None: if __name__ == "__main__": - main() + # Reachy's wireless app manager launches the module from the + # ``reachy_mini_apps`` entry point with ``python -m``. That path must use + # the SDK wrapper so the daemon supplies the already-connected robot and + # owns the media session. The ``clawbodylocal`` console script still calls + # ``main()`` directly for standalone development. + ReachyMiniOpenclaw().wrapped_run()