From d63d143dbee30d3033a03565b0cc376ca1868c71 Mon Sep 17 00:00:00 2001 From: Ying Date: Sat, 4 Apr 2026 22:33:15 +0100 Subject: [PATCH 01/17] fix: mirror camera feed before backend processing and darken bounding box colors Mirror camera frames horizontally (selfie-style) before sending to the backend so the streaming video matches what the user sees. Only applies to camera source, not video file uploads. Also darken the object detection bounding box color palette so white label text is readable on screen. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 11 ++++++++++ README.md | 4 ++-- .../mediapipe_object_detector_processor.py | 22 +++++++++---------- frontend/src/components/CameraCapture.tsx | 5 +++++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bc20a..a1108f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to the Pose Spatial Studio project will be documented in thi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.4.2] - 4 April 2026 + +### Changed +- Avatar Voice Control hidden behind "Coming Soon" placeholder with 15-click easter egg to unlock +- Added `hidden` flag to FunctionDefinition type for hiding unreleased features + +### Infrastructure +- Added supervisord process supervisor to backend Docker container for automatic app restart on crash +- Backend container image committed with supervisord entrypoint (`/root/entrypoint.sh`) +- Container restart policy set to `unless-stopped` + ## [1.4.1] - 3 April 2026 ### Added diff --git a/README.md b/README.md index e37e8e3..b5907bf 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Real-time pose estimation, object detection, and 3D avatar rendering with WebSoc - **Object Detection** — EfficientDet-Lite2 bounding boxes and labels - **Hand Gesture Recognition** — Per-hand landmark tracking with gesture classification - **Live 3D Avatar** — Mixamo-rigged avatar driven by FK quaternions from pose estimation -- **Avatar Voice Control** — Voice/text commands to control 3D avatar via SecondBrain AI +- **Avatar Voice Control** — Voice/text commands to control 3D avatar via SecondBrain AI (coming soon) - **Camera & Video Input** — Live camera streams or video file upload - **Real-time Log Streaming** — Backend logs streamed live to the frontend - **Auto-Deployment** — GitHub Actions CI/CD to production and staging @@ -61,7 +61,7 @@ Open `http://localhost:8585` in your browser. - 3D view supports orbit controls (rotate, pan, zoom) and toggle between Avatar and Skeleton rendering - 3D Pose supports model switching between MediaPipe and YOLO+RTMPose 5. **View logs** in the right sidebar panel for real-time backend diagnostics -6. **Avatar Voice Control** — select from function menu, type or speak commands like "wave your right hand" +6. **Avatar Voice Control** — coming soon (hidden easter egg: click 15 times to unlock) ## Architecture diff --git a/backend/processors/mediapipe_object_detector_processor.py b/backend/processors/mediapipe_object_detector_processor.py index 5a87cbe..523315c 100644 --- a/backend/processors/mediapipe_object_detector_processor.py +++ b/backend/processors/mediapipe_object_detector_processor.py @@ -20,18 +20,18 @@ logger = logging.getLogger(__name__) -# Color palette for different object categories (BGR) +# Color palette for different object categories (BGR) — dark tones for readability _CATEGORY_COLORS = [ - (0, 255, 0), # green - (255, 0, 0), # blue - (0, 0, 255), # red - (255, 255, 0), # cyan - (0, 255, 255), # yellow - (255, 0, 255), # magenta - (128, 255, 0), # spring green - (255, 128, 0), # sky blue - (0, 128, 255), # orange - (128, 0, 255), # violet + (0, 160, 0), # dark green + (200, 0, 0), # dark blue + (0, 0, 200), # dark red + (160, 120, 0), # dark teal + (0, 120, 180), # dark orange + (160, 0, 160), # dark magenta + (60, 140, 0), # dark spring green + (180, 80, 0), # dark sky blue + (0, 80, 180), # dark amber + (120, 0, 180), # dark violet ] diff --git a/frontend/src/components/CameraCapture.tsx b/frontend/src/components/CameraCapture.tsx index b1ded3a..8c0717d 100644 --- a/frontend/src/components/CameraCapture.tsx +++ b/frontend/src/components/CameraCapture.tsx @@ -182,7 +182,12 @@ export function CameraCapture({ canvasRef.current.height = videoRef.current.videoHeight; } + if (sourceType === 'camera') { + ctx.translate(canvasRef.current.width, 0); + ctx.scale(-1, 1); + } ctx.drawImage(videoRef.current, 0, 0); + ctx.setTransform(1, 0, 0, 1, 0, 0); isProcessingFrameRef.current = true; canvasRef.current.toBlob( (blob) => { From 3f4d5591e217c3bff3476ceae222c2c0f8130723 Mon Sep 17 00:00:00 2001 From: Ying Date: Sat, 4 Apr 2026 22:40:14 +0100 Subject: [PATCH 02/17] fix: use sudo to create /var/www/staging in deploy workflow The SSH user does not have write permission to /var/www. Use sudo mkdir and chown to match the ownership pattern of the production directory. Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/PROJECT_STRUCTURE.md | 341 ------------------ .claude/settings.local.json | 13 - .claude/skills/code-review/SKILL.md | 255 ------------- .claude/skills/develop/SKILL.md | 286 --------------- .claude/skills/ssh-servers/SKILL.md | 145 -------- .claude/skills/test/SKILL.md | 189 ---------- .github/workflows/deploy_frontend_staging.yml | 2 +- frontend/src/components/Skeleton3DViewer.tsx | 4 +- 8 files changed, 3 insertions(+), 1232 deletions(-) delete mode 100644 .claude/PROJECT_STRUCTURE.md delete mode 100644 .claude/settings.local.json delete mode 100644 .claude/skills/code-review/SKILL.md delete mode 100644 .claude/skills/develop/SKILL.md delete mode 100644 .claude/skills/ssh-servers/SKILL.md delete mode 100644 .claude/skills/test/SKILL.md diff --git a/.claude/PROJECT_STRUCTURE.md b/.claude/PROJECT_STRUCTURE.md deleted file mode 100644 index f218406..0000000 --- a/.claude/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,341 +0,0 @@ -# Project Structure - -## Directory Layout - -``` -pose-spatial-studio/ -├── backend/ -│ ├── app.py # FastAPI + Socket.IO server, health/info endpoints -│ ├── config.py # Settings (host, port, CORS, GPU detection, config merging) -│ ├── config_template.json # Default processor configuration template -│ ├── requirements.txt # Python deps (FastAPI, MediaPipe, torch, rtmlib, ultralytics) -│ ├── run_server.sh # Local dev startup script -│ ├── yolov8m-pose.pt # YOLOv8-M Pose model weights -│ ├── core/ -│ │ └── websocket_handler.py # Socket.IO stream management, processor pipeline, log streaming -│ ├── processors/ -│ │ ├── __init__.py -│ │ ├── base_processor.py # Abstract processor interface -│ │ ├── data_processor.py # Temporal smoothing, windowing, feature extraction -│ │ ├── image_processor.py # Frame preprocessing (resize, flip, normalize) -│ │ ├── mediapipe_processor.py # MediaPipe pose estimation (2D/3D landmarks) -│ │ ├── mediapipe_object_detector_processor.py # Object detection (EfficientDet) -│ │ ├── mediapipe_hand_gesture_processor.py # Hand gesture recognition -│ │ ├── rtmpose_processor.py # RTMPose3D via rtmlib (RTMW3D-X + YOLOX-M, FK output) -│ │ ├── yolo_pose_2d_processor.py # YOLOv8-Pose 2D detection -│ │ └── yolo_tcpformer_processor.py # YOLO 2D + TCPFormer 3D lifting (81-frame temporal) -│ ├── utils/ -│ │ ├── __init__.py -│ │ ├── cache.py # Model caching utilities -│ │ ├── filters.py # MedianFilter, GaussianFilter for temporal smoothing -│ │ ├── io.py # Frame I/O, encoding/decoding -│ │ ├── kinetic.py # Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24) -│ │ ├── locate_path.py # Project root detection -│ │ ├── log_streamer.py # SocketIOLogHandler for real-time log streaming -│ │ └── logger.py # Structured logging setup -│ └── models/ -│ ├── tcpformer/ # TCPFormer 2D→3D lifting model (AAAI 2025) -│ │ ├── model.py -│ │ └── *.pth.tr # Checkpoint (auto-downloaded) -│ ├── efficientdet_lite2.tflite # MediaPipe object detection model -│ └── pose_landmarker_full.task # MediaPipe pose landmarker model -│ -├── frontend/ -│ ├── src/ -│ │ ├── main.tsx # App entry point -│ │ ├── App.tsx # Root layout: sidebar + viewer + log panel -│ │ ├── App.css # Global styles -│ │ ├── components/ -│ │ │ ├── CameraCapture.tsx # Camera/video source lifecycle, 10 FPS capture, backpressure -│ │ │ ├── Controls.tsx # Function selector, source/device picker, start/stop -│ │ │ ├── FunctionViewer.tsx # Routes to View2D, View3D, RoboticControlView, or placeholder by viewMode -│ │ │ ├── View2D.tsx # 2D canvas viewer with camera mirroring -│ │ │ ├── View3D.tsx # 3D scene with model selector, renderer toggle -│ │ │ ├── Skeleton3DViewer.tsx # Three.js canvas: video plane + skeleton + controls -│ │ │ ├── LogPanel.tsx # Real-time backend log viewer (right sidebar) -│ │ │ ├── ChatPanel.tsx # Chat UI with text input, voice recognition, SSE streaming -│ │ │ ├── RoboticControlView.tsx # Avatar + ChatPanel side-by-side layout for voice control -│ │ │ └── DataAnalysis_.tsx # Placeholder (unused) -│ │ ├── hooks/ -│ │ │ ├── useCameraDevices.ts # Camera device enumeration + permission management -│ │ │ ├── useLogStream.ts # Backend log subscription (rolling 1000-entry buffer) -│ │ │ ├── useSessionTimer.ts # 60s guest session countdown timer -│ │ │ ├── useVoiceRecognition.ts # Web Speech API wrapper for voice input -│ │ │ └── useWebSocket.ts # Socket connection + stale result filtering -│ │ ├── services/ -│ │ │ ├── socketService.ts # Socket.IO singleton client (VITE_BACKEND_URL) -│ │ │ ├── streamInitService.ts # Async stream init with model switching, 60s timeout -│ │ │ ├── streamService.ts # Per-stream frame transmission -│ │ │ └── secondBrainService.ts # SSE streaming to SecondBrain guest chat endpoint -│ │ ├── stores/ -│ │ │ └── appStore.ts # Zustand store (function, source, stream, renderer state) -│ │ ├── three/ -│ │ │ ├── AvatarRenderer.tsx # Mixamo rigged avatar with FK quaternion animation -│ │ │ ├── StickBallRenderer.tsx # Procedural ball-and-stick skeleton -│ │ │ ├── VideoPlane.tsx # Video feed texture on XY plane, camera mirroring -│ │ │ ├── boneMapping.ts # Mixamo bone names, joint→bone maps, FK transforms -│ │ │ └── connections.ts # Skeleton bone connection topology -│ │ └── types/ -│ │ ├── functions.ts # Function definitions, processor types, view modes -│ │ ├── chat.ts # Chat message, session, and streaming types -│ │ └── pose.ts # Landmark, PoseData, DetectedObject, DetectedHand types -│ ├── public/ -│ │ └── avatars/skeleton.glb # Mixamo rigged skeleton model -│ ├── .env.local # Dev: VITE_BACKEND_URL=http://localhost:49101 -│ ├── .env.production # Prod: VITE_BACKEND_URL=https://pose-backend.yingliu.site -│ ├── package.json -│ ├── vite.config.ts # Vite config (port 8585, @ path alias) -│ └── run_ui.sh # Local dev startup script -│ -├── .github/workflows/ -│ ├── deploy_backend.yml # CI/CD: backend to production (main branch) -│ ├── deploy_backend_staging.yml # CI/CD: backend to staging (staging branch) -│ ├── deploy_frontend.yml # CI/CD: frontend to production (main branch) -│ └── deploy_frontend_staging.yml # CI/CD: frontend to staging (staging branch) -│ -├── .claude/ -│ ├── PROJECT_STRUCTURE.md # This file -│ └── skills/ -│ ├── develop/SKILL.md # Full development workflow -│ ├── code-review/SKILL.md # Code review workflow -│ ├── test/SKILL.md # Test workflow -│ └── ssh-servers/SKILL.md # Remote server access -│ -├── tests/ -│ ├── test_solve_ik.py # Integration test for solve_ik (local + Socket.IO) -│ ├── playwright.config.ts # Playwright config (production) -│ ├── playwright.staging.config.ts # Playwright config (staging) -│ └── specs/ -│ ├── pose-validation.spec.ts # E2E pose detection + 3D rendering tests -│ ├── avatar-voice-control.spec.ts # E2E avatar voice control tests (8 tests) -│ └── staging-video-test.spec.ts # Staging-specific video upload tests -│ -├── README.md # Project overview and setup guide -├── CHANGELOG.md # Version history -├── output/ # Processing output directory -├── logs/ # Backend log files (date-stamped) -└── .cache/ # Runtime model cache -``` - -## Function Modes - -The app operates in single-function mode with five available functions: - -| Function | Processor | View | Description | -|----------|-----------|------|-------------| -| 2D Pose Estimation | `yolo_pose_2d` | 2D | YOLOv8-Pose 2D keypoint detection | -| 3D Pose Estimation | `mediapipe` or `rtmpose` | 3D | Switchable 3D model with avatar/skeleton rendering | -| Object Detection | `mediapipe_object_detection` | 2D | EfficientDet bounding boxes + labels | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | 2D | Per-hand landmarks + gesture classification | -| Avatar Voice Control | — (SecondBrain) | voice | Voice/text commands → SecondBrain → solve_ik → avatar | - -## Core Components - -### Backend - -**app.py** - FastAPI server with Socket.IO, CORS, endpoints: `/` (info), `/health` (stats), `/info` (features) - -**config.py** - Centralized settings with env var support: -- `POSE_STUDIO_HOST` (default `0.0.0.0`), `POSE_STUDIO_PORT` (default `49101`) -- `POSE_WORKERS` — thread pool size (default `min(cpu_count, 16)`) -- `MAX_CONCURRENT_STREAMS` — server-wide limit (default `3`) -- BLAS thread pinning (`OMP_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`) -- CORS origins: `localhost:8585`, `robot.yingliu.site`, `staging.robot.yingliu.site` -- GPU detection (ONNX Runtime CUDA, PyTorch CUDA) -- Config merging from `config_template.json` - -**websocket_handler.py** - Manages: -- Client connections/disconnections with per-client cleanup -- Stream initialization with multi-processor pipeline -- Concurrent frame processing via `ThreadPoolExecutor(max_workers=POSE_WORKERS)` -- Per-stream timing metrics exposed on `/health` -- Real-time log streaming (`subscribe_logs` / `unsubscribe_logs`) -- Model switching (`switch_model` event) -- IK solving (`solve_ik` event → FK quaternion result via Converter) -- Concurrent stream limit enforcement - -**Processors** (all inherit from `base_processor.py`): - -| Processor | Output | Notes | -|-----------|--------|-------| -| `mediapipe_processor` | 2D/3D landmarks (33→24 unified) | LIVE_STREAM for camera, VIDEO for uploads | -| `rtmpose_processor` | FK quaternions + root position | RTMW3D-X + YOLOX-M, depth-corrected z | -| `yolo_pose_2d_processor` | 2D landmarks (17 COCO keypoints) | YOLOv8-M Pose | -| `yolo_tcpformer_processor` | FK + world landmarks | YOLO 2D → TCPFormer temporal 3D lifting | -| `mediapipe_object_detector_processor` | Bounding boxes + labels | EfficientDet-Lite2 | -| `mediapipe_hand_gesture_processor` | Hand landmarks + gestures | Per-hand classification | -| `image_processor` | Preprocessed frame | Resize, flip, normalize | -| `data_processor` | Feature vectors | Temporal smoothing, windowing | - -**Utilities:** -- `kinetic.py` — Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24 joints) -- `filters.py` — MedianFilter, GaussianFilter for temporal smoothing -- `io.py` — Frame encoding/decoding -- `log_streamer.py` — SocketIOLogHandler for real-time log streaming to clients - -### Frontend - -**App.tsx** - Root layout: left sidebar (Controls), center (FunctionViewer), right sidebar (LogPanel). Animated background orbs, connection status indicator, error boundary. - -**Controls.tsx** - Function selector (radio buttons), source type toggle (camera/video), device picker, start/stop buttons. Camera permission requested on Start (not on mount). - -**FunctionViewer.tsx** - Routes to View2D, View3D, RoboticControlView (`voice`), or placeholder based on `functionDef.viewMode`. - -**View2D.tsx** - Canvas-based 2D viewer. Displays annotated frames from backend. Camera source mirroring via `scaleX(-1)`. - -**View3D.tsx** - 3D scene wrapper. Model selector dropdown (MediaPipe / YOLO+RTMPose). Avatar/skeleton renderer toggle. Manages video/canvas refs for Skeleton3DViewer. - -**Skeleton3DViewer.tsx** - Three.js Canvas with VideoPlane background + AvatarRenderer or StickBallRenderer. Orbit controls, grid, lighting, error boundary. - -**CameraCapture.tsx** - Camera/video source lifecycle. 10 FPS capture with backpressure (waits for backend result before sending next frame). - -**LogPanel.tsx** - Real-time backend log viewer. Severity-colored entries (DEBUG/INFO/WARNING/ERROR), auto-scroll, expand/collapse. - -**appStore.ts** (Zustand) - Centralized state: `activeFunction`, `sourceType`, `deviceId`, `videoFile`, `isStreamActive`, `isInitializing`, `initMessage`, `backendResult`, `rendererType` (avatar/stickball), `pose3dProcessorType` (mediapipe/rtmpose), sidebar collapse states, chat state (messages, streaming, session expiry). - -**Three.js renderers:** -- `AvatarRenderer` — Loads skeleton.glb, applies FK quaternions to Mixamo bones, T-pose caching, smooth interpolation -- `StickBallRenderer` — Procedural green spheres + lines using POSE_CONNECTIONS topology -- `VideoPlane` — Video texture on XY plane, camera source mirroring -- `boneMapping` — Mixamo bone names, joint→bone maps, conjugation rules, z-axis negation for legs - -## Architecture - -### Data Flow - -``` -Source (Camera 10 FPS / Video) → encode JPEG/Base64 → WebSocket - ↓ -WebSocket Handler → decode → ImageProcessor → DataProcessor → PoseProcessor - ↓ -encode JPEG/Base64 → WebSocket → Frontend - ↓ -View2D: Canvas render (annotated frame) -View3D: Three.js (VideoPlane + AvatarRenderer or StickBallRenderer) -``` - -### Stream Lifecycle - -**Initialize:** -``` -UI Start → requestPermission() → StreamInitService.initializeStream() -→ Backend creates processor pipeline → stream_initialized → Camera starts -``` - -**Process (with backpressure):** -``` -Camera captures → process_frame → Pipeline → pose_result -→ Update store → Render → next capture -``` - -**Cleanup:** -``` -Stop button → cleanup_processor → Backend releases processors -→ Reset permission → UI idle -``` - -## Deployment Architecture - -``` -GitHub Actions - │ - ├─ push to main ──────────────────────────────────────────────┐ - │ ├─ deploy_frontend.yml → build → rsync → nginx reload │ - │ │ Serves: https://robot.yingliu.site │ - │ └─ deploy_backend.yml → rsync → docker cp → restart │ - │ Serves: https://pose-backend.yingliu.site │ - │ │ - └─ push to staging ───────────────────────────────────────────┤ - ├─ deploy_frontend_staging.yml │ - │ Serves: https://staging.robot.yingliu.site │ - └─ deploy_backend_staging.yml │ - Serves: staging backend container │ - -VM1 (Frontend Edge) VM2 (GPU Backend) -┌─────────────────┐ ┌──────────────────────────┐ -│ Nginx │ │ Docker container │ -│ /var/www/frontend│ ──WS──► │ FastAPI + Socket.IO │ -│ Cloudflare TLS │ │ CUDA GPU acceleration │ -└─────────────────┘ └──────────────────────────┘ -``` - -## WebSocket Events - -### Client → Server - -| Event | Payload | -|-------|---------| -| `initialize_stream` | `{ stream_id, processor_type, processor_config, source_type }` | -| `process_frame` | `{ stream_id, frame (base64), timestamp_ms }` | -| `cleanup_processor` | `{ stream_id }` | -| `switch_model` | `{ stream_id, processor_type }` | -| `subscribe_logs` | `{}` | -| `unsubscribe_logs` | `{}` | -| `solve_ik` | `{ request_id, joints, root_position }` | - -### Server → Client - -| Event | Payload | -|-------|---------| -| `connection_status` | `{ status, sid }` | -| `stream_initialized` | `{ stream_id, status, message, processor_type }` | -| `stream_error` | `{ stream_id, message, code?, active_streams?, max_streams? }` | -| `stream_loading` | `{ stream_id, message }` | -| `pose_result` | `{ stream_id, frame (base64), pose_data, timestamp_ms }` | -| `model_switched` | `{ stream_id, processor_type, message }` | -| `log_batch` | `[{ level, message, timestamp, logger }]` | -| `error` | `{ message }` | -| `fk_result` | `{ request_id, fk_data, root_position, error? }` | - -## Configuration - -### Backend (config.py + config_template.json) -```python -HOST = os.getenv("POSE_STUDIO_HOST", "0.0.0.0") -PORT = int(os.getenv("POSE_STUDIO_PORT", 49101)) -POSE_WORKERS = min(cpu_count, 16) # env var override -MAX_CONCURRENT_STREAMS = 3 # env var override -``` - -### Frontend (environment files) -- `.env.local` → `VITE_BACKEND_URL=http://localhost:49101` -- `.env.production` → `VITE_BACKEND_URL=https://pose-backend.yingliu.site` -- `VITE_SECOND_BRAIN_URL` → SecondBrain guest chat API base URL - -## Tech Stack - -**Backend:** Python 3.13, FastAPI, Socket.IO, MediaPipe, rtmlib, Ultralytics (YOLOv8), PyTorch, OpenCV, NumPy - -**Frontend:** React 19, TypeScript, Three.js, React Three Fiber, Drei, Zustand, Socket.IO Client, Vite - -**Testing:** Playwright (E2E, production + staging configs) - -**CI/CD:** GitHub Actions, Cloudflare Tunnels (SSH via cloudflared + Access service tokens), rsync deployment - -**Infrastructure:** Nginx, Docker, NVIDIA CUDA, Cloudflare (TLS, DDoS, WAF) - -## Testing - -```bash -# From tests/ directory -npx playwright test # Run all tests -npx playwright test --config playwright.staging.config.ts # Staging tests -npx playwright test --headed # With visible browser -npx playwright show-report # View HTML report -``` - -## Debugging - -**Backend logs:** -```bash -tail -f logs/$(date +%Y-%m-%d).log -grep ERROR logs/*.log -``` - -**Real-time logs:** Open LogPanel in the right sidebar (frontend streams backend logs live) - -**Common issues:** -- Camera not starting → Check browser permissions -- No pose results → Check backend logs / LogPanel for initialization errors -- Low performance → Reduce FPS or resolution -- Stream limit reached → Max 3 concurrent streams (configurable) diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index d6cc45d..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(npx playwright install:*)", - "Bash(git status:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(git push:*)", - "Bash(GIT_EDITOR=true git rebase:*)", - "Bash(GIT_EDITOR=/usr/bin/true git rebase:*)" - ] - } -} diff --git a/.claude/skills/code-review/SKILL.md b/.claude/skills/code-review/SKILL.md deleted file mode 100644 index 7d70aa0..0000000 --- a/.claude/skills/code-review/SKILL.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -name: code-review ---- - -Review code changes **$ARGUMENTS** following the structured code review workflow below. - -## TODO List - -After identifying the scope of review (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each review phase (lint, quality, performance, testing) -- One task for each file or component under review - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This ensures thorough, systematic coverage of all review criteria. - -## Workflow Steps - -1. **Scope** -- Identify changes to review and establish review context -2. **Lint** -- Run linters and static analysis tools -3. **Code quality** -- Assess readability, structure, and conventions -4. **Performance** -- Analyze memory usage, speed, and algorithmic complexity -5. **Security** -- Check for vulnerabilities and sensitive data exposure -6. **Integration test** -- Validate changes work end-to-end in both directions -7. **Summary** -- Report findings and recommend actions - ---- - -## Step 1: Identify Review Scope - -Determine what code is under review: - -1. **Diff the changes** against the base branch: - ```bash - - git diff staging...HEAD --stat - git diff staging...HEAD - ``` -2. **List affected files** and categorize them (frontend, backend, config, docs) -3. **Read each changed file** in full to understand context around the diff -4. **Note the intent**: What is the change trying to accomplish? - -Document the following before proceeding: -- Files changed and lines affected -- Summary of what the changes do -- Which acceptance criteria (if any) apply - ---- - -## Step 2: Lint Check - -Run linters and static analysis for each affected area: - -### Frontend (TypeScript / React) -```bash -cd frontend -npx eslint src/ --ext .ts,.tsx -npx tsc --noEmit -``` - -### Backend (Python) -```bash -cd backend -python -m py_compile app.py -# Run any configured linters (flake8, ruff, mypy, etc.) -``` - -### Review for: -- Syntax errors or warnings -- Unused imports and variables -- Type errors or mismatches -- Formatting inconsistencies - -**If lint issues are found**: list each issue with file path and line number. - ---- - -## Step 3: Code Quality Review - -Assess the code for cleanliness and maintainability: - -### Readability -- Are variable and function names descriptive and consistent with existing conventions? -- Is the control flow easy to follow? -- Are complex sections commented where necessary (but not over-commented)? - -### Structure -- Does the code follow existing project patterns and architecture? -- Are responsibilities properly separated (e.g., no business logic in UI components)? -- Are new files placed in the correct directories per `PROJECT_STRUCTURE.md`? - -### Conventions -- Does the code match the style of surrounding code? -- Are TypeScript types properly defined (no unnecessary `any`)? -- Are Python type hints used consistently with the rest of the codebase? - -### Duplication -- Is there duplicated logic that should be extracted? -- Are there existing utilities or helpers that could be reused? - ---- - -## Step 4: Performance Review - -Analyze memory usage, speed, and algorithmic complexity: - -### Algorithmic complexity -- What is the Big O time and space complexity of new or modified logic? -- Are there unnecessary nested loops, redundant computations, or O(n^2) patterns that could be O(n)? - -### Memory -- Are large objects or arrays properly cleaned up? -- Are event listeners, intervals, or subscriptions properly disposed of? -- Are WebSocket connections and streams managed without leaks? - -### Rendering (Frontend) -- Are React components avoiding unnecessary re-renders? -- Are expensive computations memoized where appropriate? -- Are Three.js objects (geometries, materials, textures) properly disposed? - -### I/O (Backend) -- Are database queries or API calls efficient? -- Is caching used appropriately? -- Are WebSocket frame sizes and frequencies reasonable? - ---- - -## Step 5: Security Review - -Check for common vulnerabilities: - -- **No secrets in code**: No API keys, passwords, tokens, or credentials in committed files -- **Input validation**: User inputs are sanitized at system boundaries -- **Dependency safety**: No known vulnerable packages introduced -- **CORS / auth**: WebSocket and API endpoints have appropriate access controls -- **Data exposure**: No sensitive data logged or sent to the client unnecessarily - ---- - -## Step 6: Integration Testing - -Validate that changes work correctly end-to-end using two test configurations: - -### 6.1: Local frontend + Remote backend - -Test the frontend changes against the production backend: -1. Start the frontend locally: - ```bash - cd frontend - ./run_ui.sh - ``` -2. Configure the frontend to connect to the remote backend (`https://pose-backend.yingliu.site`) -3. Open `http://localhost:8585` in the browser -4. Validate: - - Does the UI render correctly? (function selector, source picker, log panel) - - Do all function modes work? (2D Pose, 3D Pose, Object Detection, Hand Gesture) - - Does 3D pose model switching (MediaPipe / YOLO+RTMPose) and avatar/skeleton toggle work? - - Are there console errors or broken WebSocket connections? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### 6.2: Remote frontend + Local backend - -Test the backend changes against the production frontend: -1. Start the backend locally: - ```bash - cd backend - ./run_server.sh - ``` -2. Access the remote frontend at `https://robot.yingliu.site` -3. Configure it to connect to the local backend (`http://localhost:49101`) -4. Validate: - - Does the backend serve pose/detection/gesture data correctly for all function modes? - - Are WebSocket streams stable? - - Does the log panel show backend logs in real time? - - Is the response time acceptable? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### Automated testing (optional) - -Use Playwright for automated validation. See `tests/README.md` for full setup. - -```bash -cd tests -npm test # run all tests -npm run test:headed # run with visible browser -``` - -Test specs are in `tests/specs/`: -- `pose-validation.spec.ts` — Main E2E suite (UI controls, video upload, pose detection) -- `staging-video-test.spec.ts` — Staging backend smoke test (uses `playwright.staging.config.ts`) - ---- - -## Step 7: Review Summary - -Compile findings into a structured report: - -### Report format - -```markdown -## Code Review Summary - -**Branch**: -**Files reviewed**: -**Overall assessment**: PASS | PASS WITH COMMENTS | NEEDS CHANGES - -### Lint -- [ ] All lint checks pass - -### Code Quality -- [ ] Naming and conventions consistent -- [ ] Structure follows project patterns -- [ ] No unnecessary duplication - -### Performance -- [ ] No algorithmic concerns -- [ ] Memory management is correct -- [ ] No rendering or I/O bottlenecks - -### Security -- [ ] No secrets or credentials exposed -- [ ] Input validation present at boundaries - -### Integration -- [ ] Local frontend + remote backend: PASS / FAIL -- [ ] Remote frontend + local backend: PASS / FAIL - -### Issues Found -| # | Severity | File | Line | Description | -|---|----------|------|------|-------------| -| 1 | High/Medium/Low | path/to/file | 42 | Description of issue | - -### Recommendations -- -``` - -### Severity levels -| Level | Meaning | Action | -|-------|---------|--------| -| **High** | Bug, security issue, or data loss risk | Must fix before merge | -| **Medium** | Performance concern or convention violation | Should fix before merge | -| **Low** | Style nit or minor improvement | Optional, can fix later | - ---- - -## After Review - -Present the review summary to the user and use `AskUserQuestion` to determine next steps: - -**Question:** "How would you like to proceed after the review?" - -**Options:** -- Fix issues and re-review (Recommended) -- Address findings and run the review again -- Proceed to commit -- Accept current state and commit changes -- Discuss findings -- Talk through specific issues before deciding diff --git a/.claude/skills/develop/SKILL.md b/.claude/skills/develop/SKILL.md deleted file mode 100644 index 62234ab..0000000 --- a/.claude/skills/develop/SKILL.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -name: develop ---- - -Accept user requirement **$ARGUMENTS** following the full development workflow. - -## TODO List - -After understanding the request (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each acceptance criterion to implement -- One task for each affected file or component to build - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This systematic approach ensures all acceptance criteria are met and nothing is missed. - -## Workflow Overview - -**CRITICAL: Steps 1–5 are MANDATORY and BLOCKING. You MUST complete each step before proceeding to the next. Do NOT skip Steps 4 or 5.** - -| Step | Action | Mandatory | Notes | -|------|--------|-----------|-------| -| 1 | **Understand** | YES | Analyze requirements, define acceptance criteria | -| 2 | **Branch** | YES | Create a properly named branch from the correct base | -| 3 | **Implement** | YES | Make changes following code conventions and lint standards | -| 4 | **Validate** | **YES — BLOCKING** | Run ALL test steps in test/SKILL.md — NO exceptions | -| 5 | **Document** | **YES — BLOCKING** | Update CHANGELOG.md, PROJECT_STRUCTURE.md, README.md | -| 6 | **Review** | Optional | Optionally run a developer code review | -| 7 | **Commit** | YES | Stage files and commit with conventional message format | -| 8 | **Push & PR** | YES | Push branch and create pull request | - ---- - -## Step 1: Understand the Requirement - -Thoroughly analyze the requirement by: -1. Breaking down the request into discrete components -2. Identifying acceptance criteria, dependencies, and any linked PRDs or designs -3. Using `AskUserQuestion` to clarify any ambiguities -4. Reading [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) to understand the project architecture - -Document the following: -- Summary of the requirement -- Detailed description -- Acceptance criteria (what defines "done") -- Dependencies on other work -- Links to PRDs or design documents - ---- - -## Step 2: Create Feature Branch - -### Prepare the base - -1. Identify which folder(s) will be affected by the changes (refer to [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md)) -2. Fetch latest from remote: - ```bash - git fetch origin - ``` -3. Determine the base branch: - - If one release branch exists, use it - - If multiple exist, show the latest two and ask the user which to target using `AskUserQuestion` (mark the latest as "Recommended") - -### Create the branch - -For new features targeting production: -```bash -git checkout staging -git pull origin staging -git checkout -b type/brief-description -``` - -**Git flow:** `feature branch` → PR to `staging` → auto-deploy & test → PR to `main` → production - -### Branch naming: `type/brief-description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat/` | New feature | `feat/avatar-creation` | -| `fix/` | Bug fix | `fix/pose-detection-issue` | -| `docs/` | Documentation | `docs/update-structure` | -| `wip/` | Work in progress | `wip/experiment-joint-rotation` | - -**Rules:** lowercase, hyphens (not underscores), include ticket key if applicable, keep it brief. - -### Branching from in-progress work - -When a ticket depends on another ticket still in code review, branch from that ticket's branch instead. After the base ticket is merged, rebase onto the target branch and force push with `--force-with-lease`. - ---- - -## Step 3: Implement Changes - -**Function modes and processors** are defined in [frontend/src/types/functions.ts](frontend/src/types/functions.ts): - -| Function | Default Processor | Backend File | -|----------|------------------|--------------| -| 2D Pose Estimation | `yolo_pose_2d` | `processors/yolo_pose_2d_processor.py` | -| 3D Pose Estimation | `mediapipe` (switchable to `rtmpose`) | `processors/mediapipe_processor.py`, `processors/rtmpose_processor.py` | -| Object Detection | `mediapipe_object_detection` | `processors/mediapipe_object_detector_processor.py` | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | `processors/mediapipe_hand_gesture_processor.py` | -| Avatar Voice Control | — (SecondBrain guest API) | `services/secondBrainService.ts` | - -State management: [frontend/src/stores/appStore.ts](frontend/src/stores/appStore.ts) (Zustand store). - -Follow these guidelines during implementation: - -1. **Code conventions** — Maintain consistent formatting and style with existing codebase -2. **Acceptance criteria** — Ensure all acceptance criteria from Step 1 are fully met -3. **Update TODO** — Mark tasks as `in_progress` when starting, `completed` when finished -4. **Test as you go** — Validate changes incrementally to catch issues early -5. **Optimize complexity** — Analyze and improve Big O time/space complexity where possible -6. **Lint check** — Run linters after implementation (see below) -7. **Deployment requiremnt** - Update deployment pipeline if there are dependancies - -### Frontend lint - -```bash -cd frontend && npx tsc --noEmit -``` - -### Backend lint - -Compile-check all modified Python files: -```bash -cd backend && python -m py_compile app.py -# Also check any modified processor or core files, e.g.: -# python -m py_compile processors/mediapipe_processor.py -# python -m py_compile core/websocket_handler.py -# python -m py_compile config.py -``` - ---- - -## Step 4: Validate - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST read and execute EVERY sub-step defined in [.claude/skills/test/SKILL.md](.claude/skills/test/SKILL.md) before proceeding to Step 5. -> **This is NOT optional.** Do NOT skip, abbreviate, summarize, or reorder any sub-step. -> The test skill defines four compulsory steps plus user-approval gates — ALL must be completed. -> **If you skip this step or any sub-step, the entire workflow is invalid.** - -Execute the test skill steps in order: - -| Test Step | What to do | Gate | -|-----------|-----------|------| -| **Step 0** Pre-flight | Kill all servers; verify ports 49101 and 8585 are free | — | -| **Step 1** Automated E2E | Run `cd tests && npm test`; verify screenshots in `tests/results/` | — | -| **Step 2** Manual testing | Start dev servers; test video file and camera with all three pose models | **STOP — Ask user for approval before continuing to Step 3** | -| **Step 3** Staging | Commit → PR to `staging` → wait for CI deploy → health check → run staging tests | **STOP — Ask user for approval before continuing to Step 4** | -| **Step 4** Remote GPU | SSH health check + tail logs on staging and production backends | — | - -**After ALL test steps pass and the user explicitly approves at each gate**, return here and continue to Step 5. Do NOT proceed to Step 5 without completing every test step. - ---- - -## Step 5: Update Documentation - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST update all applicable documentation files listed below before proceeding to Step 6. -> **This is NOT optional.** Do NOT skip this step. Do NOT proceed to commit without completing documentation updates. -> For each file below, either update it OR explicitly state to the user why no update is needed. - -### 1. [CHANGELOG.md](CHANGELOG.md) - -Add an entry at the top of the appropriate `CHANGELOG.md`: - -```markdown -## - - -- : -``` - -If the version section doesn't exist, create it. - -### 2. [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) - -If changes affect project architecture (new files, moved directories, new processors), update this file accordingly. - -### 3. [README.md](README.md) - -If changes affect setup, usage, or dependencies, update the relevant `README.md`. - -### 4. TODO status - -Mark all completed tasks in your TODO list as `completed` using `TodoWrite`. - -### 5. Skill update - -Update any `.claude/skills/*/SKILL.md` files that are affected by workflow changes discovered during this session. - ---- - -## Step 6: Developer Review (Optional) - -Use `AskUserQuestion` to ask whether the user wants to run a Developer Review before committing: - -**Question:** "Would you like to run a Developer Review before committing?" - -**Options:** -- Yes (Recommended) — Review changes against base branch to catch issues early -- No — Skip review and proceed to commit - -If the user selects "Yes", invoke the `code-review` skill using the `Skill` tool: -``` -skill: "code-review" -``` -This executes [.claude/skills/code-review/SKILL.md](.claude/skills/code-review/SKILL.md). - ---- - -## Step 7: Commit Changes - -### Commit message format: `type: description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat:` | New feature | `feat: add avatar joint rotation support` | -| `fix:` | Bug fix | `fix: resolve pose detection accuracy issue` | -| `docs:` | Documentation | `docs: update PROJECT_STRUCTURE.md` | -| `style:` | Formatting only | `style: fix indentation in pose module` | -| `refactor:` | Code restructuring | `refactor: extract avatar axis alignment logic` | -| `perf:` | Performance improvement | `perf: optimize skeleton rendering` | -| `test:` | Tests | `test: add unit tests for pose service` | -| `chore:` | Build/tooling | `chore: update Python dependencies` | - -### Commit rules - -1. **Stage specific files** — Use `git add <file>` for each file (avoid `git add -A` or `git add .`) -2. **Imperative mood** — "add" not "added", "fix" not "fixed" -3. **Under 72 characters** -4. **No trailing period** - ---- - -## Step 8: Push Branch and Create Pull Request - -### Rebase before push - -```bash -git fetch origin -git rebase origin/staging -``` - -Resolve any conflicts before pushing. - -### Push - -```bash -git push -u origin <branch-name> -``` - -### Create PR - -Use `gh pr create` CLI command. - -**PR title:** Clear, descriptive, under 70 characters. - -**PR description** must include: -1. **Summary of changes** — Brief overview of what was implemented -2. **Link to issue/ticket** — Reference the original requirement or issue number -3. **Testing notes** — How to test the changes, steps to reproduce -4. **Screenshots** — Include screenshots for UI changes -5. **Acceptance criteria** — Confirm all criteria from Step 1 are met - ---- - -## Pre-PR Checklist - -Before creating the pull request, verify **every item**: - -- [ ] All acceptance criteria from Step 1 are fully met -- [ ] All TODO tasks are marked as `completed` -- [ ] **Test Step 0** — Stale servers killed, ports verified free -- [ ] **Test Step 1** — Playwright E2E tests passed; screenshots reviewed -- [ ] **Test Step 2** — Manual testing completed; user approved to proceed -- [ ] **Test Step 3** — Staging deployed, health check OK, staging tests passed; user approved to proceed -- [ ] **Test Step 4** — Remote GPU health check and logs reviewed -- [ ] Lint checks pass (`tsc --noEmit`, `py_compile`) -- [ ] Commit messages follow `type: description` format -- [ ] Branch name follows `type/brief-description` convention -- [ ] PR title is clear and descriptive -- [ ] Documentation updated ([CHANGELOG.md](CHANGELOG.md), [README.md](README.md), [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) as needed) -- [ ] No secrets, credentials, or sensitive data in committed code -- [ ] Code follows project conventions and existing patterns -- [ ] No unnecessary files added (temp files, IDE configs, build artifacts) diff --git a/.claude/skills/ssh-servers/SKILL.md b/.claude/skills/ssh-servers/SKILL.md deleted file mode 100644 index 669d8f7..0000000 --- a/.claude/skills/ssh-servers/SKILL.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -name: ssh-servers ---- - -SSH into the backend and/or frontend remote servers to inspect, debug, or manage deployed code. - -## Prerequisites - -Before connecting, ensure the SSH agent has the deploy key loaded: - -```bash -ssh-add -l # Check if keys are loaded -ssh-add ~/.ssh/github_deploy # RSA key (frontend) -ssh-add ~/.ssh/github_deploy_ed25519 # Ed25519 key (backend) -``` - -If `github_deploy_ed25519` has wrong permissions: -```bash -chmod 600 ~/.ssh/github_deploy_ed25519 -``` - -## SSH Hosts - -Both servers use Cloudflare Tunnels (via `cloudflared access ssh`) as configured in `~/.ssh/config`. - -### Backend (GPU Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-backend` | -| Hostname | `pose-backend-ssh.yingliu.site` | -| User | `root` | -| Key | `~/.ssh/github_deploy_ed25519` | -| Production container | `pose-spatial-studio-backend` (port 49101) | -| Staging container | `pose-spatial-studio-backend-staging` (port 49102) | -| Code path | `/root/backend/` (inside container) | -| Logs | `/root/backend/logs/app.log` (inside container) | - -**Connect:** -```bash -ssh pose-backend -``` - -**Interactive shell inside the container:** -```bash -docker exec -it pose-spatial-studio-backend bash -``` -This drops you into the container at `/root/backend/` where you can run Python, inspect files, install packages, and debug directly. - -**Common commands (run on host, prefix container commands with `docker exec`):** - -```bash -# Check backend health -docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health - -# View recent logs -docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log - -# Inspect deployed code -docker exec pose-spatial-studio-backend cat /root/backend/app.py -docker exec pose-spatial-studio-backend cat /root/backend/config.py -docker exec pose-spatial-studio-backend ls -la /root/backend/processors/ - -# Check installed packages -docker exec pose-spatial-studio-backend pip list | grep -iE 'mediapipe|rtm|onnx|torch|opencv' - -# Check GPU status -docker exec pose-spatial-studio-backend python3 -c "import onnxruntime as ort; print(ort.get_available_providers())" - -# Restart the backend app -docker exec pose-spatial-studio-backend bash -c "pkill -f 'python.*app.py'" || true -sleep 2 -docker start pose-spatial-studio-backend 2>/dev/null || true -docker exec -d pose-spatial-studio-backend bash -c "cd /root/backend && python app.py >> logs/app.log 2>&1" -``` - -### Frontend (Nginx Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-frontend` | -| Hostname | `pose-frontend-ssh.yingliu.site` | -| User | `sophia` | -| Key | `~/.ssh/github_deploy` (RSA) | -| Code path | `/var/www/frontend/` | -| Serves | `https://robot.yingliu.site` | - -**Connect:** -```bash -ssh pose-frontend -``` - -**Common commands:** - -```bash -# List deployed files -ls -la /var/www/frontend/ -ls -la /var/www/frontend/assets/ - -# View deployed index.html -cat /var/www/frontend/index.html - -# Check nginx status -sudo systemctl status nginx - -# Reload nginx after manual changes -sudo systemctl reload nginx -``` - -## Debugging Workflow - -When a problem exists on remote but not locally: - -1. **Check logs** on the backend for errors: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend tail -80 /root/backend/logs/app.log" - ``` - -2. **Compare deployed code** with local code: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend cat /root/backend/<file>" | diff - backend/<file> - ``` - -3. **Compare package versions** (common source of discrepancies): - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend pip list" > /tmp/remote-packages.txt - diff <(cat /tmp/remote-packages.txt) <(cd backend && .venv.nosync/bin/pip list) - ``` - -4. **Check frontend build output** — the remote serves a production Vite build, not dev mode: - ```bash - ssh pose-frontend "cat /var/www/frontend/index.html" - ``` - -## Quick Health Check (No SSH Required) - -```bash -# Production -curl -s https://pose-backend.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://robot.yingliu.site - -# Staging -curl -s https://pose-backend-staging.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://staging.robot.yingliu.site -``` diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md deleted file mode 100644 index 3c3a11f..0000000 --- a/.claude/skills/test/SKILL.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -name: test ---- - -Run the validation suite for **$ARGUMENTS** (or full validation if no arguments given). - -**CRITICAL: Steps 0, 1, 2, 3, and 4 are ALL MANDATORY and BLOCKING — do NOT skip ANY of them.** -**Steps 2 and 3 have USER-APPROVAL GATES — you MUST stop and get explicit user approval before continuing past them.** -**If you skip any step or gate, the entire validation is invalid and must be restarted from Step 0.** - -**IMPORTANT — run from the correct project:** All tests MUST run from the **working project directory** (the project with your code changes), NOT from wherever this SKILL.md lives. If you have multiple project copies (e.g., `pose-spatial-studio` and `pose-spatial-studio-1`), always `cd` into the one with your active feature branch before running any commands. - -**IMPORTANT — Do not make any changes in Staging and produc server from local project. It can only be done through GitHub CI/CD pipeline** ---- - -## Step 0: Pre-flight checks (MANDATORY) - -Before running any tests, **kill all existing servers** to prevent stale/wrong backends from being reused: - -```bash -pkill -f "app.py" 2>/dev/null; pkill -f "uvicorn" 2>/dev/null; pkill -f "run_server.sh" 2>/dev/null -pkill -f "vite" 2>/dev/null; pkill -f "run_ui.sh" 2>/dev/null -sleep 2 -# Verify ports are free: -lsof -i :49101 2>&1; lsof -i :8585 2>&1 -``` - -This is critical because Playwright's `reuseExistingServer: true` (the local default) will silently reuse any backend already running on port 49101 — even if it's from a different project copy. This causes false-positive test results. - ---- - -## Step 1: Automated E2E tests (MANDATORY) - -Run Playwright tests from the **working project's** `tests/` directory. Playwright auto-starts both backend (port 49101) and frontend (port 8585) via `webServer` config if they aren't already running. - -**Run all tests:** -```bash -cd tests && npm test -``` - -**Run a single spec on Chromium (minimum bar):** -```bash -cd tests && npx playwright test specs/pose-validation.spec.ts --project=chromium -``` - -**Other commands:** -```bash -cd tests && npm run test:headed # Visible browser -cd tests && npm run test:debug # Debug with breakpoints -cd tests && npm run test:ui # Interactive Playwright UI -cd tests && npm run test:report # View HTML report after a run -``` - -**Test specs** live in `tests/specs/`: -| Spec | Purpose | Config | -|------|---------|--------| -| `pose-validation.spec.ts` | Main E2E suite — UI controls, video upload, pose detection | default | -| `staging-video-test.spec.ts` | Staging backend smoke test | `playwright.staging.config.ts` | - -If tests fail, diagnose and fix before continuing. - -**Visual verification** — after tests pass, examine screenshots in `tests/results/` using the Read tool and verify: -- `pose-validation-avatar.png` — **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips -- `pose-validation-skeleton.png` — **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally -- Both screenshots should show the camera feed with a 2D skeleton overlay drawn on the image - ---- - -## Step 2: Manual testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Manual testing is complete. May I proceed to Step 3 (Staging environment testing)?" Do NOT continue to Step 3 without explicit user approval.** - -1. **Start dev environment** (if not already running): - - python venv environment is in `backend/.venv.nosync` - - Backend: `cd backend && ./run_server.sh` (port 49101) - - Frontend: `cd frontend && ./run_ui.sh` (port 8585) - - Confirm frontend connects to the **local** backend - -2. **Test with video file** (no camera needed): - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector (left sidebar) - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching: use the model selector dropdown in the 3D view to switch between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** functions - -3. **Visual checks** (take screenshots and examine with Read tool): - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips. avatar is not jittering. the movement is like a human. - - Toggle between Avatar and Skeleton using the inline button in the 3D view toolbar - - Switch model via the dropdown → re-check all of the above - -4. **Test with live camera** (when relevant): - - Select source type **Camera** → choose a camera device → click **Start** - - Verify pose landmarks overlay and 3D skeleton/avatar respond in real time - ---- - -## Step 3: Staging environment testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Staging tests are complete. May I proceed to Step 4 (Remote GPU validation)?" Do NOT continue to Step 4 without explicit user approval.** - -Validate changes against the staging backend before production deployment. - -**Staging infrastructure:** -- Backend container: `pose-spatial-studio-backend-staging` on VM2 port 49102 -- Staging URL: `https://pose-backend-staging.yingliu.site` -- CI/CD: `deploy_backend_staging.yml` triggers on push to `staging` branch - -**Workflow:** - -1. Commit changes and create/merge PR to `staging` branch and approve merge -2. Wait for the **Deploy backend (staging)** GitHub Actions workflow to pass -3. Health check: - ```bash - curl -s https://pose-backend-staging.yingliu.site/health - ``` -4. Run frontend locally against staging backend and wait for user approval: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev - ``` -5. **Test with video file:** - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -6. run the automated staging test: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev & - cd tests && npx playwright test specs/staging-video-test.spec.ts --config=playwright.staging.config.ts --project=chromium - ``` -7. **Visual verification** — take a screenshot and examine with the Read tool: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - - Toggle Avatar/Skeleton via inline button → switch model via dropdown → re-check -8. Check staging backend logs: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -30 /root/backend/logs/app.log" - ``` -9. **STOP** — Use `AskUserQuestion` to ask the user whether staging tests passed and whether to proceed to Step 4 - ---- - -## Step 4: Remote GPU validation (MANDATORY) - -Verify the staging backend is healthy and GPU-accessible. This applies to **all changes** because the backend always runs on GPU infrastructure. - -```bash -# Staging backend -ssh pose-backend "docker exec pose-spatial-studio-backend-staging curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -50 /root/backend/logs/app.log" - -# Production backend -ssh pose-backend "docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log" -``` - -**Visual verification** — run frontend against the production backend, take a screenshot, and examine with the Read tool: -1. `cd frontend && VITE_BACKEND_URL=https://pose-backend.yingliu.site npm run dev` -2. Open `http://localhost:8585` → Select **3D Pose Estimation** → Source type **Video File** → upload `tests/test.mp4` -3. Click **Start** → verify stream initializes → switch models between **MediaPipe** and **YOLO+RTMPose** -4. Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -5. Verify via screenshot: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - -See `/ssh-servers` skill for full remote debugging commands. - ---- - -## Validation checklist - -Before marking validation as complete, confirm ALL of the following: - -- [ ] **Step 0** — Pre-flight checks completed (servers killed, ports free) -- [ ] **Step 1** — Automated Playwright tests passed; screenshots verified -- [ ] **Step 2** — Manual testing completed; user approved to proceed -- [ ] **Step 3** — Staging environment tested (deployed, health check OK, pose detection works); user approved to proceed -- [ ] **Step 4** — Remote GPU validation passed (health check + logs reviewed) -- [ ] Implementation meets all acceptance criteria - -If any step fails → fix the issue and re-run from that step. - -**After ALL steps pass, return to develop/SKILL.md and continue with Step 5 (Update Documentation). Do NOT skip documentation updates.** \ No newline at end of file diff --git a/.github/workflows/deploy_frontend_staging.yml b/.github/workflows/deploy_frontend_staging.yml index 47468ac..5891423 100644 --- a/.github/workflows/deploy_frontend_staging.yml +++ b/.github/workflows/deploy_frontend_staging.yml @@ -61,7 +61,7 @@ jobs: - name: Deploy via rsync run: | - ssh vm-ssh "mkdir -p /var/www/staging" + ssh vm-ssh "sudo mkdir -p /var/www/staging && sudo chown ${{ secrets.SSH_USER }}:www-data /var/www/staging" rsync -az --delete dist/ vm-ssh:/var/www/staging - name: Reload nginx diff --git a/frontend/src/components/Skeleton3DViewer.tsx b/frontend/src/components/Skeleton3DViewer.tsx index c057a74..8b5fac4 100644 --- a/frontend/src/components/Skeleton3DViewer.tsx +++ b/frontend/src/components/Skeleton3DViewer.tsx @@ -68,8 +68,8 @@ export function Skeleton3DViewer({ poseResult, videoElement, processedCanvas, re <div style={{ width: '100%', height: '100%', backgroundColor: '#000' }}> <ViewerErrorBoundary> <Canvas> - <PerspectiveCamera makeDefault position={[0, 0, 3]} /> - <OrbitControls enableDamping dampingFactor={0.05} /> + <PerspectiveCamera makeDefault position={[0, 1.2, 5]} /> + <OrbitControls enableDamping dampingFactor={0.05} target={[0, 0.6, 0]} /> <ambientLight intensity={0.5} /> <directionalLight position={[10, 10, 5]} intensity={1} /> From 3ba82ae89ba555235d6e033c9bff5dd303787f2b Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 22:40:28 +0100 Subject: [PATCH 03/17] fix: restore accidentally deleted .claude/ project files Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- .claude/PROJECT_STRUCTURE.md | 341 ++++++++++++++++++++++++++++ .claude/settings.local.json | 13 ++ .claude/skills/code-review/SKILL.md | 255 +++++++++++++++++++++ .claude/skills/develop/SKILL.md | 286 +++++++++++++++++++++++ .claude/skills/ssh-servers/SKILL.md | 145 ++++++++++++ .claude/skills/test/SKILL.md | 189 +++++++++++++++ 6 files changed, 1229 insertions(+) create mode 100644 .claude/PROJECT_STRUCTURE.md create mode 100644 .claude/settings.local.json create mode 100644 .claude/skills/code-review/SKILL.md create mode 100644 .claude/skills/develop/SKILL.md create mode 100644 .claude/skills/ssh-servers/SKILL.md create mode 100644 .claude/skills/test/SKILL.md diff --git a/.claude/PROJECT_STRUCTURE.md b/.claude/PROJECT_STRUCTURE.md new file mode 100644 index 0000000..f218406 --- /dev/null +++ b/.claude/PROJECT_STRUCTURE.md @@ -0,0 +1,341 @@ +# Project Structure + +## Directory Layout + +``` +pose-spatial-studio/ +├── backend/ +│ ├── app.py # FastAPI + Socket.IO server, health/info endpoints +│ ├── config.py # Settings (host, port, CORS, GPU detection, config merging) +│ ├── config_template.json # Default processor configuration template +│ ├── requirements.txt # Python deps (FastAPI, MediaPipe, torch, rtmlib, ultralytics) +│ ├── run_server.sh # Local dev startup script +│ ├── yolov8m-pose.pt # YOLOv8-M Pose model weights +│ ├── core/ +│ │ └── websocket_handler.py # Socket.IO stream management, processor pipeline, log streaming +│ ├── processors/ +│ │ ├── __init__.py +│ │ ├── base_processor.py # Abstract processor interface +│ │ ├── data_processor.py # Temporal smoothing, windowing, feature extraction +│ │ ├── image_processor.py # Frame preprocessing (resize, flip, normalize) +│ │ ├── mediapipe_processor.py # MediaPipe pose estimation (2D/3D landmarks) +│ │ ├── mediapipe_object_detector_processor.py # Object detection (EfficientDet) +│ │ ├── mediapipe_hand_gesture_processor.py # Hand gesture recognition +│ │ ├── rtmpose_processor.py # RTMPose3D via rtmlib (RTMW3D-X + YOLOX-M, FK output) +│ │ ├── yolo_pose_2d_processor.py # YOLOv8-Pose 2D detection +│ │ └── yolo_tcpformer_processor.py # YOLO 2D + TCPFormer 3D lifting (81-frame temporal) +│ ├── utils/ +│ │ ├── __init__.py +│ │ ├── cache.py # Model caching utilities +│ │ ├── filters.py # MedianFilter, GaussianFilter for temporal smoothing +│ │ ├── io.py # Frame I/O, encoding/decoding +│ │ ├── kinetic.py # Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24) +│ │ ├── locate_path.py # Project root detection +│ │ ├── log_streamer.py # SocketIOLogHandler for real-time log streaming +│ │ └── logger.py # Structured logging setup +│ └── models/ +│ ├── tcpformer/ # TCPFormer 2D→3D lifting model (AAAI 2025) +│ │ ├── model.py +│ │ └── *.pth.tr # Checkpoint (auto-downloaded) +│ ├── efficientdet_lite2.tflite # MediaPipe object detection model +│ └── pose_landmarker_full.task # MediaPipe pose landmarker model +│ +├── frontend/ +│ ├── src/ +│ │ ├── main.tsx # App entry point +│ │ ├── App.tsx # Root layout: sidebar + viewer + log panel +│ │ ├── App.css # Global styles +│ │ ├── components/ +│ │ │ ├── CameraCapture.tsx # Camera/video source lifecycle, 10 FPS capture, backpressure +│ │ │ ├── Controls.tsx # Function selector, source/device picker, start/stop +│ │ │ ├── FunctionViewer.tsx # Routes to View2D, View3D, RoboticControlView, or placeholder by viewMode +│ │ │ ├── View2D.tsx # 2D canvas viewer with camera mirroring +│ │ │ ├── View3D.tsx # 3D scene with model selector, renderer toggle +│ │ │ ├── Skeleton3DViewer.tsx # Three.js canvas: video plane + skeleton + controls +│ │ │ ├── LogPanel.tsx # Real-time backend log viewer (right sidebar) +│ │ │ ├── ChatPanel.tsx # Chat UI with text input, voice recognition, SSE streaming +│ │ │ ├── RoboticControlView.tsx # Avatar + ChatPanel side-by-side layout for voice control +│ │ │ └── DataAnalysis_.tsx # Placeholder (unused) +│ │ ├── hooks/ +│ │ │ ├── useCameraDevices.ts # Camera device enumeration + permission management +│ │ │ ├── useLogStream.ts # Backend log subscription (rolling 1000-entry buffer) +│ │ │ ├── useSessionTimer.ts # 60s guest session countdown timer +│ │ │ ├── useVoiceRecognition.ts # Web Speech API wrapper for voice input +│ │ │ └── useWebSocket.ts # Socket connection + stale result filtering +│ │ ├── services/ +│ │ │ ├── socketService.ts # Socket.IO singleton client (VITE_BACKEND_URL) +│ │ │ ├── streamInitService.ts # Async stream init with model switching, 60s timeout +│ │ │ ├── streamService.ts # Per-stream frame transmission +│ │ │ └── secondBrainService.ts # SSE streaming to SecondBrain guest chat endpoint +│ │ ├── stores/ +│ │ │ └── appStore.ts # Zustand store (function, source, stream, renderer state) +│ │ ├── three/ +│ │ │ ├── AvatarRenderer.tsx # Mixamo rigged avatar with FK quaternion animation +│ │ │ ├── StickBallRenderer.tsx # Procedural ball-and-stick skeleton +│ │ │ ├── VideoPlane.tsx # Video feed texture on XY plane, camera mirroring +│ │ │ ├── boneMapping.ts # Mixamo bone names, joint→bone maps, FK transforms +│ │ │ └── connections.ts # Skeleton bone connection topology +│ │ └── types/ +│ │ ├── functions.ts # Function definitions, processor types, view modes +│ │ ├── chat.ts # Chat message, session, and streaming types +│ │ └── pose.ts # Landmark, PoseData, DetectedObject, DetectedHand types +│ ├── public/ +│ │ └── avatars/skeleton.glb # Mixamo rigged skeleton model +│ ├── .env.local # Dev: VITE_BACKEND_URL=http://localhost:49101 +│ ├── .env.production # Prod: VITE_BACKEND_URL=https://pose-backend.yingliu.site +│ ├── package.json +│ ├── vite.config.ts # Vite config (port 8585, @ path alias) +│ └── run_ui.sh # Local dev startup script +│ +├── .github/workflows/ +│ ├── deploy_backend.yml # CI/CD: backend to production (main branch) +│ ├── deploy_backend_staging.yml # CI/CD: backend to staging (staging branch) +│ ├── deploy_frontend.yml # CI/CD: frontend to production (main branch) +│ └── deploy_frontend_staging.yml # CI/CD: frontend to staging (staging branch) +│ +├── .claude/ +│ ├── PROJECT_STRUCTURE.md # This file +│ └── skills/ +│ ├── develop/SKILL.md # Full development workflow +│ ├── code-review/SKILL.md # Code review workflow +│ ├── test/SKILL.md # Test workflow +│ └── ssh-servers/SKILL.md # Remote server access +│ +├── tests/ +│ ├── test_solve_ik.py # Integration test for solve_ik (local + Socket.IO) +│ ├── playwright.config.ts # Playwright config (production) +│ ├── playwright.staging.config.ts # Playwright config (staging) +│ └── specs/ +│ ├── pose-validation.spec.ts # E2E pose detection + 3D rendering tests +│ ├── avatar-voice-control.spec.ts # E2E avatar voice control tests (8 tests) +│ └── staging-video-test.spec.ts # Staging-specific video upload tests +│ +├── README.md # Project overview and setup guide +├── CHANGELOG.md # Version history +├── output/ # Processing output directory +├── logs/ # Backend log files (date-stamped) +└── .cache/ # Runtime model cache +``` + +## Function Modes + +The app operates in single-function mode with five available functions: + +| Function | Processor | View | Description | +|----------|-----------|------|-------------| +| 2D Pose Estimation | `yolo_pose_2d` | 2D | YOLOv8-Pose 2D keypoint detection | +| 3D Pose Estimation | `mediapipe` or `rtmpose` | 3D | Switchable 3D model with avatar/skeleton rendering | +| Object Detection | `mediapipe_object_detection` | 2D | EfficientDet bounding boxes + labels | +| Hand Gesture Recognition | `mediapipe_hand_gesture` | 2D | Per-hand landmarks + gesture classification | +| Avatar Voice Control | — (SecondBrain) | voice | Voice/text commands → SecondBrain → solve_ik → avatar | + +## Core Components + +### Backend + +**app.py** - FastAPI server with Socket.IO, CORS, endpoints: `/` (info), `/health` (stats), `/info` (features) + +**config.py** - Centralized settings with env var support: +- `POSE_STUDIO_HOST` (default `0.0.0.0`), `POSE_STUDIO_PORT` (default `49101`) +- `POSE_WORKERS` — thread pool size (default `min(cpu_count, 16)`) +- `MAX_CONCURRENT_STREAMS` — server-wide limit (default `3`) +- BLAS thread pinning (`OMP_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`) +- CORS origins: `localhost:8585`, `robot.yingliu.site`, `staging.robot.yingliu.site` +- GPU detection (ONNX Runtime CUDA, PyTorch CUDA) +- Config merging from `config_template.json` + +**websocket_handler.py** - Manages: +- Client connections/disconnections with per-client cleanup +- Stream initialization with multi-processor pipeline +- Concurrent frame processing via `ThreadPoolExecutor(max_workers=POSE_WORKERS)` +- Per-stream timing metrics exposed on `/health` +- Real-time log streaming (`subscribe_logs` / `unsubscribe_logs`) +- Model switching (`switch_model` event) +- IK solving (`solve_ik` event → FK quaternion result via Converter) +- Concurrent stream limit enforcement + +**Processors** (all inherit from `base_processor.py`): + +| Processor | Output | Notes | +|-----------|--------|-------| +| `mediapipe_processor` | 2D/3D landmarks (33→24 unified) | LIVE_STREAM for camera, VIDEO for uploads | +| `rtmpose_processor` | FK quaternions + root position | RTMW3D-X + YOLOX-M, depth-corrected z | +| `yolo_pose_2d_processor` | 2D landmarks (17 COCO keypoints) | YOLOv8-M Pose | +| `yolo_tcpformer_processor` | FK + world landmarks | YOLO 2D → TCPFormer temporal 3D lifting | +| `mediapipe_object_detector_processor` | Bounding boxes + labels | EfficientDet-Lite2 | +| `mediapipe_hand_gesture_processor` | Hand landmarks + gestures | Per-hand classification | +| `image_processor` | Preprocessed frame | Resize, flip, normalize | +| `data_processor` | Feature vectors | Temporal smoothing, windowing | + +**Utilities:** +- `kinetic.py` — Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24 joints) +- `filters.py` — MedianFilter, GaussianFilter for temporal smoothing +- `io.py` — Frame encoding/decoding +- `log_streamer.py` — SocketIOLogHandler for real-time log streaming to clients + +### Frontend + +**App.tsx** - Root layout: left sidebar (Controls), center (FunctionViewer), right sidebar (LogPanel). Animated background orbs, connection status indicator, error boundary. + +**Controls.tsx** - Function selector (radio buttons), source type toggle (camera/video), device picker, start/stop buttons. Camera permission requested on Start (not on mount). + +**FunctionViewer.tsx** - Routes to View2D, View3D, RoboticControlView (`voice`), or placeholder based on `functionDef.viewMode`. + +**View2D.tsx** - Canvas-based 2D viewer. Displays annotated frames from backend. Camera source mirroring via `scaleX(-1)`. + +**View3D.tsx** - 3D scene wrapper. Model selector dropdown (MediaPipe / YOLO+RTMPose). Avatar/skeleton renderer toggle. Manages video/canvas refs for Skeleton3DViewer. + +**Skeleton3DViewer.tsx** - Three.js Canvas with VideoPlane background + AvatarRenderer or StickBallRenderer. Orbit controls, grid, lighting, error boundary. + +**CameraCapture.tsx** - Camera/video source lifecycle. 10 FPS capture with backpressure (waits for backend result before sending next frame). + +**LogPanel.tsx** - Real-time backend log viewer. Severity-colored entries (DEBUG/INFO/WARNING/ERROR), auto-scroll, expand/collapse. + +**appStore.ts** (Zustand) - Centralized state: `activeFunction`, `sourceType`, `deviceId`, `videoFile`, `isStreamActive`, `isInitializing`, `initMessage`, `backendResult`, `rendererType` (avatar/stickball), `pose3dProcessorType` (mediapipe/rtmpose), sidebar collapse states, chat state (messages, streaming, session expiry). + +**Three.js renderers:** +- `AvatarRenderer` — Loads skeleton.glb, applies FK quaternions to Mixamo bones, T-pose caching, smooth interpolation +- `StickBallRenderer` — Procedural green spheres + lines using POSE_CONNECTIONS topology +- `VideoPlane` — Video texture on XY plane, camera source mirroring +- `boneMapping` — Mixamo bone names, joint→bone maps, conjugation rules, z-axis negation for legs + +## Architecture + +### Data Flow + +``` +Source (Camera 10 FPS / Video) → encode JPEG/Base64 → WebSocket + ↓ +WebSocket Handler → decode → ImageProcessor → DataProcessor → PoseProcessor + ↓ +encode JPEG/Base64 → WebSocket → Frontend + ↓ +View2D: Canvas render (annotated frame) +View3D: Three.js (VideoPlane + AvatarRenderer or StickBallRenderer) +``` + +### Stream Lifecycle + +**Initialize:** +``` +UI Start → requestPermission() → StreamInitService.initializeStream() +→ Backend creates processor pipeline → stream_initialized → Camera starts +``` + +**Process (with backpressure):** +``` +Camera captures → process_frame → Pipeline → pose_result +→ Update store → Render → next capture +``` + +**Cleanup:** +``` +Stop button → cleanup_processor → Backend releases processors +→ Reset permission → UI idle +``` + +## Deployment Architecture + +``` +GitHub Actions + │ + ├─ push to main ──────────────────────────────────────────────┐ + │ ├─ deploy_frontend.yml → build → rsync → nginx reload │ + │ │ Serves: https://robot.yingliu.site │ + │ └─ deploy_backend.yml → rsync → docker cp → restart │ + │ Serves: https://pose-backend.yingliu.site │ + │ │ + └─ push to staging ───────────────────────────────────────────┤ + ├─ deploy_frontend_staging.yml │ + │ Serves: https://staging.robot.yingliu.site │ + └─ deploy_backend_staging.yml │ + Serves: staging backend container │ + +VM1 (Frontend Edge) VM2 (GPU Backend) +┌─────────────────┐ ┌──────────────────────────┐ +│ Nginx │ │ Docker container │ +│ /var/www/frontend│ ──WS──► │ FastAPI + Socket.IO │ +│ Cloudflare TLS │ │ CUDA GPU acceleration │ +└─────────────────┘ └──────────────────────────┘ +``` + +## WebSocket Events + +### Client → Server + +| Event | Payload | +|-------|---------| +| `initialize_stream` | `{ stream_id, processor_type, processor_config, source_type }` | +| `process_frame` | `{ stream_id, frame (base64), timestamp_ms }` | +| `cleanup_processor` | `{ stream_id }` | +| `switch_model` | `{ stream_id, processor_type }` | +| `subscribe_logs` | `{}` | +| `unsubscribe_logs` | `{}` | +| `solve_ik` | `{ request_id, joints, root_position }` | + +### Server → Client + +| Event | Payload | +|-------|---------| +| `connection_status` | `{ status, sid }` | +| `stream_initialized` | `{ stream_id, status, message, processor_type }` | +| `stream_error` | `{ stream_id, message, code?, active_streams?, max_streams? }` | +| `stream_loading` | `{ stream_id, message }` | +| `pose_result` | `{ stream_id, frame (base64), pose_data, timestamp_ms }` | +| `model_switched` | `{ stream_id, processor_type, message }` | +| `log_batch` | `[{ level, message, timestamp, logger }]` | +| `error` | `{ message }` | +| `fk_result` | `{ request_id, fk_data, root_position, error? }` | + +## Configuration + +### Backend (config.py + config_template.json) +```python +HOST = os.getenv("POSE_STUDIO_HOST", "0.0.0.0") +PORT = int(os.getenv("POSE_STUDIO_PORT", 49101)) +POSE_WORKERS = min(cpu_count, 16) # env var override +MAX_CONCURRENT_STREAMS = 3 # env var override +``` + +### Frontend (environment files) +- `.env.local` → `VITE_BACKEND_URL=http://localhost:49101` +- `.env.production` → `VITE_BACKEND_URL=https://pose-backend.yingliu.site` +- `VITE_SECOND_BRAIN_URL` → SecondBrain guest chat API base URL + +## Tech Stack + +**Backend:** Python 3.13, FastAPI, Socket.IO, MediaPipe, rtmlib, Ultralytics (YOLOv8), PyTorch, OpenCV, NumPy + +**Frontend:** React 19, TypeScript, Three.js, React Three Fiber, Drei, Zustand, Socket.IO Client, Vite + +**Testing:** Playwright (E2E, production + staging configs) + +**CI/CD:** GitHub Actions, Cloudflare Tunnels (SSH via cloudflared + Access service tokens), rsync deployment + +**Infrastructure:** Nginx, Docker, NVIDIA CUDA, Cloudflare (TLS, DDoS, WAF) + +## Testing + +```bash +# From tests/ directory +npx playwright test # Run all tests +npx playwright test --config playwright.staging.config.ts # Staging tests +npx playwright test --headed # With visible browser +npx playwright show-report # View HTML report +``` + +## Debugging + +**Backend logs:** +```bash +tail -f logs/$(date +%Y-%m-%d).log +grep ERROR logs/*.log +``` + +**Real-time logs:** Open LogPanel in the right sidebar (frontend streams backend logs live) + +**Common issues:** +- Camera not starting → Check browser permissions +- No pose results → Check backend logs / LogPanel for initialization errors +- Low performance → Reduce FPS or resolution +- Stream limit reached → Max 3 concurrent streams (configurable) diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..d6cc45d --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,13 @@ +{ + "permissions": { + "allow": [ + "Bash(npx playwright install:*)", + "Bash(git status:*)", + "Bash(git add:*)", + "Bash(git commit:*)", + "Bash(git push:*)", + "Bash(GIT_EDITOR=true git rebase:*)", + "Bash(GIT_EDITOR=/usr/bin/true git rebase:*)" + ] + } +} diff --git a/.claude/skills/code-review/SKILL.md b/.claude/skills/code-review/SKILL.md new file mode 100644 index 0000000..7d70aa0 --- /dev/null +++ b/.claude/skills/code-review/SKILL.md @@ -0,0 +1,255 @@ +--- +name: code-review +--- + +Review code changes **$ARGUMENTS** following the structured code review workflow below. + +## TODO List + +After identifying the scope of review (Step 1), **create a TODO list** using the `TodoWrite` tool with: +- One task for each review phase (lint, quality, performance, testing) +- One task for each file or component under review + +Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This ensures thorough, systematic coverage of all review criteria. + +## Workflow Steps + +1. **Scope** -- Identify changes to review and establish review context +2. **Lint** -- Run linters and static analysis tools +3. **Code quality** -- Assess readability, structure, and conventions +4. **Performance** -- Analyze memory usage, speed, and algorithmic complexity +5. **Security** -- Check for vulnerabilities and sensitive data exposure +6. **Integration test** -- Validate changes work end-to-end in both directions +7. **Summary** -- Report findings and recommend actions + +--- + +## Step 1: Identify Review Scope + +Determine what code is under review: + +1. **Diff the changes** against the base branch: + ```bash + + git diff staging...HEAD --stat + git diff staging...HEAD + ``` +2. **List affected files** and categorize them (frontend, backend, config, docs) +3. **Read each changed file** in full to understand context around the diff +4. **Note the intent**: What is the change trying to accomplish? + +Document the following before proceeding: +- Files changed and lines affected +- Summary of what the changes do +- Which acceptance criteria (if any) apply + +--- + +## Step 2: Lint Check + +Run linters and static analysis for each affected area: + +### Frontend (TypeScript / React) +```bash +cd frontend +npx eslint src/ --ext .ts,.tsx +npx tsc --noEmit +``` + +### Backend (Python) +```bash +cd backend +python -m py_compile app.py +# Run any configured linters (flake8, ruff, mypy, etc.) +``` + +### Review for: +- Syntax errors or warnings +- Unused imports and variables +- Type errors or mismatches +- Formatting inconsistencies + +**If lint issues are found**: list each issue with file path and line number. + +--- + +## Step 3: Code Quality Review + +Assess the code for cleanliness and maintainability: + +### Readability +- Are variable and function names descriptive and consistent with existing conventions? +- Is the control flow easy to follow? +- Are complex sections commented where necessary (but not over-commented)? + +### Structure +- Does the code follow existing project patterns and architecture? +- Are responsibilities properly separated (e.g., no business logic in UI components)? +- Are new files placed in the correct directories per `PROJECT_STRUCTURE.md`? + +### Conventions +- Does the code match the style of surrounding code? +- Are TypeScript types properly defined (no unnecessary `any`)? +- Are Python type hints used consistently with the rest of the codebase? + +### Duplication +- Is there duplicated logic that should be extracted? +- Are there existing utilities or helpers that could be reused? + +--- + +## Step 4: Performance Review + +Analyze memory usage, speed, and algorithmic complexity: + +### Algorithmic complexity +- What is the Big O time and space complexity of new or modified logic? +- Are there unnecessary nested loops, redundant computations, or O(n^2) patterns that could be O(n)? + +### Memory +- Are large objects or arrays properly cleaned up? +- Are event listeners, intervals, or subscriptions properly disposed of? +- Are WebSocket connections and streams managed without leaks? + +### Rendering (Frontend) +- Are React components avoiding unnecessary re-renders? +- Are expensive computations memoized where appropriate? +- Are Three.js objects (geometries, materials, textures) properly disposed? + +### I/O (Backend) +- Are database queries or API calls efficient? +- Is caching used appropriately? +- Are WebSocket frame sizes and frequencies reasonable? + +--- + +## Step 5: Security Review + +Check for common vulnerabilities: + +- **No secrets in code**: No API keys, passwords, tokens, or credentials in committed files +- **Input validation**: User inputs are sanitized at system boundaries +- **Dependency safety**: No known vulnerable packages introduced +- **CORS / auth**: WebSocket and API endpoints have appropriate access controls +- **Data exposure**: No sensitive data logged or sent to the client unnecessarily + +--- + +## Step 6: Integration Testing + +Validate that changes work correctly end-to-end using two test configurations: + +### 6.1: Local frontend + Remote backend + +Test the frontend changes against the production backend: +1. Start the frontend locally: + ```bash + cd frontend + ./run_ui.sh + ``` +2. Configure the frontend to connect to the remote backend (`https://pose-backend.yingliu.site`) +3. Open `http://localhost:8585` in the browser +4. Validate: + - Does the UI render correctly? (function selector, source picker, log panel) + - Do all function modes work? (2D Pose, 3D Pose, Object Detection, Hand Gesture) + - Does 3D pose model switching (MediaPipe / YOLO+RTMPose) and avatar/skeleton toggle work? + - Are there console errors or broken WebSocket connections? + +**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. + +### 6.2: Remote frontend + Local backend + +Test the backend changes against the production frontend: +1. Start the backend locally: + ```bash + cd backend + ./run_server.sh + ``` +2. Access the remote frontend at `https://robot.yingliu.site` +3. Configure it to connect to the local backend (`http://localhost:49101`) +4. Validate: + - Does the backend serve pose/detection/gesture data correctly for all function modes? + - Are WebSocket streams stable? + - Does the log panel show backend logs in real time? + - Is the response time acceptable? + +**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. + +### Automated testing (optional) + +Use Playwright for automated validation. See `tests/README.md` for full setup. + +```bash +cd tests +npm test # run all tests +npm run test:headed # run with visible browser +``` + +Test specs are in `tests/specs/`: +- `pose-validation.spec.ts` — Main E2E suite (UI controls, video upload, pose detection) +- `staging-video-test.spec.ts` — Staging backend smoke test (uses `playwright.staging.config.ts`) + +--- + +## Step 7: Review Summary + +Compile findings into a structured report: + +### Report format + +```markdown +## Code Review Summary + +**Branch**: <branch-name> +**Files reviewed**: <count> +**Overall assessment**: PASS | PASS WITH COMMENTS | NEEDS CHANGES + +### Lint +- [ ] All lint checks pass + +### Code Quality +- [ ] Naming and conventions consistent +- [ ] Structure follows project patterns +- [ ] No unnecessary duplication + +### Performance +- [ ] No algorithmic concerns +- [ ] Memory management is correct +- [ ] No rendering or I/O bottlenecks + +### Security +- [ ] No secrets or credentials exposed +- [ ] Input validation present at boundaries + +### Integration +- [ ] Local frontend + remote backend: PASS / FAIL +- [ ] Remote frontend + local backend: PASS / FAIL + +### Issues Found +| # | Severity | File | Line | Description | +|---|----------|------|------|-------------| +| 1 | High/Medium/Low | path/to/file | 42 | Description of issue | + +### Recommendations +- <action items for the developer> +``` + +### Severity levels +| Level | Meaning | Action | +|-------|---------|--------| +| **High** | Bug, security issue, or data loss risk | Must fix before merge | +| **Medium** | Performance concern or convention violation | Should fix before merge | +| **Low** | Style nit or minor improvement | Optional, can fix later | + +--- + +## After Review + +Present the review summary to the user and use `AskUserQuestion` to determine next steps: + +**Question:** "How would you like to proceed after the review?" + +**Options:** +- Fix issues and re-review (Recommended) -- Address findings and run the review again +- Proceed to commit -- Accept current state and commit changes +- Discuss findings -- Talk through specific issues before deciding diff --git a/.claude/skills/develop/SKILL.md b/.claude/skills/develop/SKILL.md new file mode 100644 index 0000000..62234ab --- /dev/null +++ b/.claude/skills/develop/SKILL.md @@ -0,0 +1,286 @@ +--- +name: develop +--- + +Accept user requirement **$ARGUMENTS** following the full development workflow. + +## TODO List + +After understanding the request (Step 1), **create a TODO list** using the `TodoWrite` tool with: +- One task for each acceptance criterion to implement +- One task for each affected file or component to build + +Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This systematic approach ensures all acceptance criteria are met and nothing is missed. + +## Workflow Overview + +**CRITICAL: Steps 1–5 are MANDATORY and BLOCKING. You MUST complete each step before proceeding to the next. Do NOT skip Steps 4 or 5.** + +| Step | Action | Mandatory | Notes | +|------|--------|-----------|-------| +| 1 | **Understand** | YES | Analyze requirements, define acceptance criteria | +| 2 | **Branch** | YES | Create a properly named branch from the correct base | +| 3 | **Implement** | YES | Make changes following code conventions and lint standards | +| 4 | **Validate** | **YES — BLOCKING** | Run ALL test steps in test/SKILL.md — NO exceptions | +| 5 | **Document** | **YES — BLOCKING** | Update CHANGELOG.md, PROJECT_STRUCTURE.md, README.md | +| 6 | **Review** | Optional | Optionally run a developer code review | +| 7 | **Commit** | YES | Stage files and commit with conventional message format | +| 8 | **Push & PR** | YES | Push branch and create pull request | + +--- + +## Step 1: Understand the Requirement + +Thoroughly analyze the requirement by: +1. Breaking down the request into discrete components +2. Identifying acceptance criteria, dependencies, and any linked PRDs or designs +3. Using `AskUserQuestion` to clarify any ambiguities +4. Reading [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) to understand the project architecture + +Document the following: +- Summary of the requirement +- Detailed description +- Acceptance criteria (what defines "done") +- Dependencies on other work +- Links to PRDs or design documents + +--- + +## Step 2: Create Feature Branch + +### Prepare the base + +1. Identify which folder(s) will be affected by the changes (refer to [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md)) +2. Fetch latest from remote: + ```bash + git fetch origin + ``` +3. Determine the base branch: + - If one release branch exists, use it + - If multiple exist, show the latest two and ask the user which to target using `AskUserQuestion` (mark the latest as "Recommended") + +### Create the branch + +For new features targeting production: +```bash +git checkout staging +git pull origin staging +git checkout -b type/brief-description +``` + +**Git flow:** `feature branch` → PR to `staging` → auto-deploy & test → PR to `main` → production + +### Branch naming: `type/brief-description` + +| Type | Purpose | Example | +|------|---------|---------| +| `feat/` | New feature | `feat/avatar-creation` | +| `fix/` | Bug fix | `fix/pose-detection-issue` | +| `docs/` | Documentation | `docs/update-structure` | +| `wip/` | Work in progress | `wip/experiment-joint-rotation` | + +**Rules:** lowercase, hyphens (not underscores), include ticket key if applicable, keep it brief. + +### Branching from in-progress work + +When a ticket depends on another ticket still in code review, branch from that ticket's branch instead. After the base ticket is merged, rebase onto the target branch and force push with `--force-with-lease`. + +--- + +## Step 3: Implement Changes + +**Function modes and processors** are defined in [frontend/src/types/functions.ts](frontend/src/types/functions.ts): + +| Function | Default Processor | Backend File | +|----------|------------------|--------------| +| 2D Pose Estimation | `yolo_pose_2d` | `processors/yolo_pose_2d_processor.py` | +| 3D Pose Estimation | `mediapipe` (switchable to `rtmpose`) | `processors/mediapipe_processor.py`, `processors/rtmpose_processor.py` | +| Object Detection | `mediapipe_object_detection` | `processors/mediapipe_object_detector_processor.py` | +| Hand Gesture Recognition | `mediapipe_hand_gesture` | `processors/mediapipe_hand_gesture_processor.py` | +| Avatar Voice Control | — (SecondBrain guest API) | `services/secondBrainService.ts` | + +State management: [frontend/src/stores/appStore.ts](frontend/src/stores/appStore.ts) (Zustand store). + +Follow these guidelines during implementation: + +1. **Code conventions** — Maintain consistent formatting and style with existing codebase +2. **Acceptance criteria** — Ensure all acceptance criteria from Step 1 are fully met +3. **Update TODO** — Mark tasks as `in_progress` when starting, `completed` when finished +4. **Test as you go** — Validate changes incrementally to catch issues early +5. **Optimize complexity** — Analyze and improve Big O time/space complexity where possible +6. **Lint check** — Run linters after implementation (see below) +7. **Deployment requiremnt** - Update deployment pipeline if there are dependancies + +### Frontend lint + +```bash +cd frontend && npx tsc --noEmit +``` + +### Backend lint + +Compile-check all modified Python files: +```bash +cd backend && python -m py_compile app.py +# Also check any modified processor or core files, e.g.: +# python -m py_compile processors/mediapipe_processor.py +# python -m py_compile core/websocket_handler.py +# python -m py_compile config.py +``` + +--- + +## Step 4: Validate + +> **MANDATORY — BLOCKING STEP — DO NOT SKIP** +> +> You MUST read and execute EVERY sub-step defined in [.claude/skills/test/SKILL.md](.claude/skills/test/SKILL.md) before proceeding to Step 5. +> **This is NOT optional.** Do NOT skip, abbreviate, summarize, or reorder any sub-step. +> The test skill defines four compulsory steps plus user-approval gates — ALL must be completed. +> **If you skip this step or any sub-step, the entire workflow is invalid.** + +Execute the test skill steps in order: + +| Test Step | What to do | Gate | +|-----------|-----------|------| +| **Step 0** Pre-flight | Kill all servers; verify ports 49101 and 8585 are free | — | +| **Step 1** Automated E2E | Run `cd tests && npm test`; verify screenshots in `tests/results/` | — | +| **Step 2** Manual testing | Start dev servers; test video file and camera with all three pose models | **STOP — Ask user for approval before continuing to Step 3** | +| **Step 3** Staging | Commit → PR to `staging` → wait for CI deploy → health check → run staging tests | **STOP — Ask user for approval before continuing to Step 4** | +| **Step 4** Remote GPU | SSH health check + tail logs on staging and production backends | — | + +**After ALL test steps pass and the user explicitly approves at each gate**, return here and continue to Step 5. Do NOT proceed to Step 5 without completing every test step. + +--- + +## Step 5: Update Documentation + +> **MANDATORY — BLOCKING STEP — DO NOT SKIP** +> +> You MUST update all applicable documentation files listed below before proceeding to Step 6. +> **This is NOT optional.** Do NOT skip this step. Do NOT proceed to commit without completing documentation updates. +> For each file below, either update it OR explicitly state to the user why no update is needed. + +### 1. [CHANGELOG.md](CHANGELOG.md) + +Add an entry at the top of the appropriate `CHANGELOG.md`: + +```markdown +## <version> - <Day Month Year> + +- <brief feature summary (under 10 words)>: <Title> +``` + +If the version section doesn't exist, create it. + +### 2. [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) + +If changes affect project architecture (new files, moved directories, new processors), update this file accordingly. + +### 3. [README.md](README.md) + +If changes affect setup, usage, or dependencies, update the relevant `README.md`. + +### 4. TODO status + +Mark all completed tasks in your TODO list as `completed` using `TodoWrite`. + +### 5. Skill update + +Update any `.claude/skills/*/SKILL.md` files that are affected by workflow changes discovered during this session. + +--- + +## Step 6: Developer Review (Optional) + +Use `AskUserQuestion` to ask whether the user wants to run a Developer Review before committing: + +**Question:** "Would you like to run a Developer Review before committing?" + +**Options:** +- Yes (Recommended) — Review changes against base branch to catch issues early +- No — Skip review and proceed to commit + +If the user selects "Yes", invoke the `code-review` skill using the `Skill` tool: +``` +skill: "code-review" +``` +This executes [.claude/skills/code-review/SKILL.md](.claude/skills/code-review/SKILL.md). + +--- + +## Step 7: Commit Changes + +### Commit message format: `type: description` + +| Type | Purpose | Example | +|------|---------|---------| +| `feat:` | New feature | `feat: add avatar joint rotation support` | +| `fix:` | Bug fix | `fix: resolve pose detection accuracy issue` | +| `docs:` | Documentation | `docs: update PROJECT_STRUCTURE.md` | +| `style:` | Formatting only | `style: fix indentation in pose module` | +| `refactor:` | Code restructuring | `refactor: extract avatar axis alignment logic` | +| `perf:` | Performance improvement | `perf: optimize skeleton rendering` | +| `test:` | Tests | `test: add unit tests for pose service` | +| `chore:` | Build/tooling | `chore: update Python dependencies` | + +### Commit rules + +1. **Stage specific files** — Use `git add <file>` for each file (avoid `git add -A` or `git add .`) +2. **Imperative mood** — "add" not "added", "fix" not "fixed" +3. **Under 72 characters** +4. **No trailing period** + +--- + +## Step 8: Push Branch and Create Pull Request + +### Rebase before push + +```bash +git fetch origin +git rebase origin/staging +``` + +Resolve any conflicts before pushing. + +### Push + +```bash +git push -u origin <branch-name> +``` + +### Create PR + +Use `gh pr create` CLI command. + +**PR title:** Clear, descriptive, under 70 characters. + +**PR description** must include: +1. **Summary of changes** — Brief overview of what was implemented +2. **Link to issue/ticket** — Reference the original requirement or issue number +3. **Testing notes** — How to test the changes, steps to reproduce +4. **Screenshots** — Include screenshots for UI changes +5. **Acceptance criteria** — Confirm all criteria from Step 1 are met + +--- + +## Pre-PR Checklist + +Before creating the pull request, verify **every item**: + +- [ ] All acceptance criteria from Step 1 are fully met +- [ ] All TODO tasks are marked as `completed` +- [ ] **Test Step 0** — Stale servers killed, ports verified free +- [ ] **Test Step 1** — Playwright E2E tests passed; screenshots reviewed +- [ ] **Test Step 2** — Manual testing completed; user approved to proceed +- [ ] **Test Step 3** — Staging deployed, health check OK, staging tests passed; user approved to proceed +- [ ] **Test Step 4** — Remote GPU health check and logs reviewed +- [ ] Lint checks pass (`tsc --noEmit`, `py_compile`) +- [ ] Commit messages follow `type: description` format +- [ ] Branch name follows `type/brief-description` convention +- [ ] PR title is clear and descriptive +- [ ] Documentation updated ([CHANGELOG.md](CHANGELOG.md), [README.md](README.md), [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) as needed) +- [ ] No secrets, credentials, or sensitive data in committed code +- [ ] Code follows project conventions and existing patterns +- [ ] No unnecessary files added (temp files, IDE configs, build artifacts) diff --git a/.claude/skills/ssh-servers/SKILL.md b/.claude/skills/ssh-servers/SKILL.md new file mode 100644 index 0000000..669d8f7 --- /dev/null +++ b/.claude/skills/ssh-servers/SKILL.md @@ -0,0 +1,145 @@ +--- +name: ssh-servers +--- + +SSH into the backend and/or frontend remote servers to inspect, debug, or manage deployed code. + +## Prerequisites + +Before connecting, ensure the SSH agent has the deploy key loaded: + +```bash +ssh-add -l # Check if keys are loaded +ssh-add ~/.ssh/github_deploy # RSA key (frontend) +ssh-add ~/.ssh/github_deploy_ed25519 # Ed25519 key (backend) +``` + +If `github_deploy_ed25519` has wrong permissions: +```bash +chmod 600 ~/.ssh/github_deploy_ed25519 +``` + +## SSH Hosts + +Both servers use Cloudflare Tunnels (via `cloudflared access ssh`) as configured in `~/.ssh/config`. + +### Backend (GPU Server) + +| Field | Value | +|-------|-------| +| SSH alias | `pose-backend` | +| Hostname | `pose-backend-ssh.yingliu.site` | +| User | `root` | +| Key | `~/.ssh/github_deploy_ed25519` | +| Production container | `pose-spatial-studio-backend` (port 49101) | +| Staging container | `pose-spatial-studio-backend-staging` (port 49102) | +| Code path | `/root/backend/` (inside container) | +| Logs | `/root/backend/logs/app.log` (inside container) | + +**Connect:** +```bash +ssh pose-backend +``` + +**Interactive shell inside the container:** +```bash +docker exec -it pose-spatial-studio-backend bash +``` +This drops you into the container at `/root/backend/` where you can run Python, inspect files, install packages, and debug directly. + +**Common commands (run on host, prefix container commands with `docker exec`):** + +```bash +# Check backend health +docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health + +# View recent logs +docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log + +# Inspect deployed code +docker exec pose-spatial-studio-backend cat /root/backend/app.py +docker exec pose-spatial-studio-backend cat /root/backend/config.py +docker exec pose-spatial-studio-backend ls -la /root/backend/processors/ + +# Check installed packages +docker exec pose-spatial-studio-backend pip list | grep -iE 'mediapipe|rtm|onnx|torch|opencv' + +# Check GPU status +docker exec pose-spatial-studio-backend python3 -c "import onnxruntime as ort; print(ort.get_available_providers())" + +# Restart the backend app +docker exec pose-spatial-studio-backend bash -c "pkill -f 'python.*app.py'" || true +sleep 2 +docker start pose-spatial-studio-backend 2>/dev/null || true +docker exec -d pose-spatial-studio-backend bash -c "cd /root/backend && python app.py >> logs/app.log 2>&1" +``` + +### Frontend (Nginx Server) + +| Field | Value | +|-------|-------| +| SSH alias | `pose-frontend` | +| Hostname | `pose-frontend-ssh.yingliu.site` | +| User | `sophia` | +| Key | `~/.ssh/github_deploy` (RSA) | +| Code path | `/var/www/frontend/` | +| Serves | `https://robot.yingliu.site` | + +**Connect:** +```bash +ssh pose-frontend +``` + +**Common commands:** + +```bash +# List deployed files +ls -la /var/www/frontend/ +ls -la /var/www/frontend/assets/ + +# View deployed index.html +cat /var/www/frontend/index.html + +# Check nginx status +sudo systemctl status nginx + +# Reload nginx after manual changes +sudo systemctl reload nginx +``` + +## Debugging Workflow + +When a problem exists on remote but not locally: + +1. **Check logs** on the backend for errors: + ```bash + ssh pose-backend "docker exec pose-spatial-studio-backend tail -80 /root/backend/logs/app.log" + ``` + +2. **Compare deployed code** with local code: + ```bash + ssh pose-backend "docker exec pose-spatial-studio-backend cat /root/backend/<file>" | diff - backend/<file> + ``` + +3. **Compare package versions** (common source of discrepancies): + ```bash + ssh pose-backend "docker exec pose-spatial-studio-backend pip list" > /tmp/remote-packages.txt + diff <(cat /tmp/remote-packages.txt) <(cd backend && .venv.nosync/bin/pip list) + ``` + +4. **Check frontend build output** — the remote serves a production Vite build, not dev mode: + ```bash + ssh pose-frontend "cat /var/www/frontend/index.html" + ``` + +## Quick Health Check (No SSH Required) + +```bash +# Production +curl -s https://pose-backend.yingliu.site/health | python3 -m json.tool +curl -s -o /dev/null -w "%{http_code}" https://robot.yingliu.site + +# Staging +curl -s https://pose-backend-staging.yingliu.site/health | python3 -m json.tool +curl -s -o /dev/null -w "%{http_code}" https://staging.robot.yingliu.site +``` diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md new file mode 100644 index 0000000..3c3a11f --- /dev/null +++ b/.claude/skills/test/SKILL.md @@ -0,0 +1,189 @@ +--- +name: test +--- + +Run the validation suite for **$ARGUMENTS** (or full validation if no arguments given). + +**CRITICAL: Steps 0, 1, 2, 3, and 4 are ALL MANDATORY and BLOCKING — do NOT skip ANY of them.** +**Steps 2 and 3 have USER-APPROVAL GATES — you MUST stop and get explicit user approval before continuing past them.** +**If you skip any step or gate, the entire validation is invalid and must be restarted from Step 0.** + +**IMPORTANT — run from the correct project:** All tests MUST run from the **working project directory** (the project with your code changes), NOT from wherever this SKILL.md lives. If you have multiple project copies (e.g., `pose-spatial-studio` and `pose-spatial-studio-1`), always `cd` into the one with your active feature branch before running any commands. + +**IMPORTANT — Do not make any changes in Staging and produc server from local project. It can only be done through GitHub CI/CD pipeline** +--- + +## Step 0: Pre-flight checks (MANDATORY) + +Before running any tests, **kill all existing servers** to prevent stale/wrong backends from being reused: + +```bash +pkill -f "app.py" 2>/dev/null; pkill -f "uvicorn" 2>/dev/null; pkill -f "run_server.sh" 2>/dev/null +pkill -f "vite" 2>/dev/null; pkill -f "run_ui.sh" 2>/dev/null +sleep 2 +# Verify ports are free: +lsof -i :49101 2>&1; lsof -i :8585 2>&1 +``` + +This is critical because Playwright's `reuseExistingServer: true` (the local default) will silently reuse any backend already running on port 49101 — even if it's from a different project copy. This causes false-positive test results. + +--- + +## Step 1: Automated E2E tests (MANDATORY) + +Run Playwright tests from the **working project's** `tests/` directory. Playwright auto-starts both backend (port 49101) and frontend (port 8585) via `webServer` config if they aren't already running. + +**Run all tests:** +```bash +cd tests && npm test +``` + +**Run a single spec on Chromium (minimum bar):** +```bash +cd tests && npx playwright test specs/pose-validation.spec.ts --project=chromium +``` + +**Other commands:** +```bash +cd tests && npm run test:headed # Visible browser +cd tests && npm run test:debug # Debug with breakpoints +cd tests && npm run test:ui # Interactive Playwright UI +cd tests && npm run test:report # View HTML report after a run +``` + +**Test specs** live in `tests/specs/`: +| Spec | Purpose | Config | +|------|---------|--------| +| `pose-validation.spec.ts` | Main E2E suite — UI controls, video upload, pose detection | default | +| `staging-video-test.spec.ts` | Staging backend smoke test | `playwright.staging.config.ts` | + +If tests fail, diagnose and fix before continuing. + +**Visual verification** — after tests pass, examine screenshots in `tests/results/` using the Read tool and verify: +- `pose-validation-avatar.png` — **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips +- `pose-validation-skeleton.png` — **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally +- Both screenshots should show the camera feed with a 2D skeleton overlay drawn on the image + +--- + +## Step 2: Manual testing (MANDATORY) + +> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Manual testing is complete. May I proceed to Step 3 (Staging environment testing)?" Do NOT continue to Step 3 without explicit user approval.** + +1. **Start dev environment** (if not already running): + - python venv environment is in `backend/.venv.nosync` + - Backend: `cd backend && ./run_server.sh` (port 49101) + - Frontend: `cd frontend && ./run_ui.sh` (port 8585) + - Confirm frontend connects to the **local** backend + +2. **Test with video file** (no camera needed): + - Open `http://localhost:8585` + - Select **3D Pose Estimation** from the function selector (left sidebar) + - Choose source type **Video File** → upload `tests/test.mp4` + - Click **Start** → verify stream initializes and video plays + - Test model switching: use the model selector dropdown in the 3D view to switch between **MediaPipe** and **YOLO+RTMPose** + - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** functions + +3. **Visual checks** (take screenshots and examine with Read tool): + - 2D views: annotated frame shows skeleton overlay drawn on the video + - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally + - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips. avatar is not jittering. the movement is like a human. + - Toggle between Avatar and Skeleton using the inline button in the 3D view toolbar + - Switch model via the dropdown → re-check all of the above + +4. **Test with live camera** (when relevant): + - Select source type **Camera** → choose a camera device → click **Start** + - Verify pose landmarks overlay and 3D skeleton/avatar respond in real time + +--- + +## Step 3: Staging environment testing (MANDATORY) + +> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Staging tests are complete. May I proceed to Step 4 (Remote GPU validation)?" Do NOT continue to Step 4 without explicit user approval.** + +Validate changes against the staging backend before production deployment. + +**Staging infrastructure:** +- Backend container: `pose-spatial-studio-backend-staging` on VM2 port 49102 +- Staging URL: `https://pose-backend-staging.yingliu.site` +- CI/CD: `deploy_backend_staging.yml` triggers on push to `staging` branch + +**Workflow:** + +1. Commit changes and create/merge PR to `staging` branch and approve merge +2. Wait for the **Deploy backend (staging)** GitHub Actions workflow to pass +3. Health check: + ```bash + curl -s https://pose-backend-staging.yingliu.site/health + ``` +4. Run frontend locally against staging backend and wait for user approval: + ```bash + cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev + ``` +5. **Test with video file:** + - Open `http://localhost:8585` + - Select **3D Pose Estimation** from the function selector + - Choose source type **Video File** → upload `tests/test.mp4` + - Click **Start** → verify stream initializes and video plays + - Test model switching between **MediaPipe** and **YOLO+RTMPose** + - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** +6. run the automated staging test: + ```bash + cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev & + cd tests && npx playwright test specs/staging-video-test.spec.ts --config=playwright.staging.config.ts --project=chromium + ``` +7. **Visual verification** — take a screenshot and examine with the Read tool: + - 2D views: annotated frame shows skeleton overlay drawn on the video + - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally + - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips + - Toggle Avatar/Skeleton via inline button → switch model via dropdown → re-check +8. Check staging backend logs: + ```bash + ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -30 /root/backend/logs/app.log" + ``` +9. **STOP** — Use `AskUserQuestion` to ask the user whether staging tests passed and whether to proceed to Step 4 + +--- + +## Step 4: Remote GPU validation (MANDATORY) + +Verify the staging backend is healthy and GPU-accessible. This applies to **all changes** because the backend always runs on GPU infrastructure. + +```bash +# Staging backend +ssh pose-backend "docker exec pose-spatial-studio-backend-staging curl -s http://localhost:49101/health" +ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -50 /root/backend/logs/app.log" + +# Production backend +ssh pose-backend "docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health" +ssh pose-backend "docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log" +``` + +**Visual verification** — run frontend against the production backend, take a screenshot, and examine with the Read tool: +1. `cd frontend && VITE_BACKEND_URL=https://pose-backend.yingliu.site npm run dev` +2. Open `http://localhost:8585` → Select **3D Pose Estimation** → Source type **Video File** → upload `tests/test.mp4` +3. Click **Start** → verify stream initializes → switch models between **MediaPipe** and **YOLO+RTMPose** +4. Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** +5. Verify via screenshot: + - 2D views: annotated frame shows skeleton overlay drawn on the video + - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally + - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips + +See `/ssh-servers` skill for full remote debugging commands. + +--- + +## Validation checklist + +Before marking validation as complete, confirm ALL of the following: + +- [ ] **Step 0** — Pre-flight checks completed (servers killed, ports free) +- [ ] **Step 1** — Automated Playwright tests passed; screenshots verified +- [ ] **Step 2** — Manual testing completed; user approved to proceed +- [ ] **Step 3** — Staging environment tested (deployed, health check OK, pose detection works); user approved to proceed +- [ ] **Step 4** — Remote GPU validation passed (health check + logs reviewed) +- [ ] Implementation meets all acceptance criteria + +If any step fails → fix the issue and re-run from that step. + +**After ALL steps pass, return to develop/SKILL.md and continue with Step 5 (Update Documentation). Do NOT skip documentation updates.** \ No newline at end of file From d6c4dc570e7c17fe23fd6fe484a54cf6c0882242 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 22:41:52 +0100 Subject: [PATCH 04/17] Revert "fix: restore accidentally deleted .claude/ project files" This reverts commit 3ba82ae89ba555235d6e033c9bff5dd303787f2b. --- .claude/PROJECT_STRUCTURE.md | 341 ---------------------------- .claude/settings.local.json | 13 -- .claude/skills/code-review/SKILL.md | 255 --------------------- .claude/skills/develop/SKILL.md | 286 ----------------------- .claude/skills/ssh-servers/SKILL.md | 145 ------------ .claude/skills/test/SKILL.md | 189 --------------- 6 files changed, 1229 deletions(-) delete mode 100644 .claude/PROJECT_STRUCTURE.md delete mode 100644 .claude/settings.local.json delete mode 100644 .claude/skills/code-review/SKILL.md delete mode 100644 .claude/skills/develop/SKILL.md delete mode 100644 .claude/skills/ssh-servers/SKILL.md delete mode 100644 .claude/skills/test/SKILL.md diff --git a/.claude/PROJECT_STRUCTURE.md b/.claude/PROJECT_STRUCTURE.md deleted file mode 100644 index f218406..0000000 --- a/.claude/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,341 +0,0 @@ -# Project Structure - -## Directory Layout - -``` -pose-spatial-studio/ -├── backend/ -│ ├── app.py # FastAPI + Socket.IO server, health/info endpoints -│ ├── config.py # Settings (host, port, CORS, GPU detection, config merging) -│ ├── config_template.json # Default processor configuration template -│ ├── requirements.txt # Python deps (FastAPI, MediaPipe, torch, rtmlib, ultralytics) -│ ├── run_server.sh # Local dev startup script -│ ├── yolov8m-pose.pt # YOLOv8-M Pose model weights -│ ├── core/ -│ │ └── websocket_handler.py # Socket.IO stream management, processor pipeline, log streaming -│ ├── processors/ -│ │ ├── __init__.py -│ │ ├── base_processor.py # Abstract processor interface -│ │ ├── data_processor.py # Temporal smoothing, windowing, feature extraction -│ │ ├── image_processor.py # Frame preprocessing (resize, flip, normalize) -│ │ ├── mediapipe_processor.py # MediaPipe pose estimation (2D/3D landmarks) -│ │ ├── mediapipe_object_detector_processor.py # Object detection (EfficientDet) -│ │ ├── mediapipe_hand_gesture_processor.py # Hand gesture recognition -│ │ ├── rtmpose_processor.py # RTMPose3D via rtmlib (RTMW3D-X + YOLOX-M, FK output) -│ │ ├── yolo_pose_2d_processor.py # YOLOv8-Pose 2D detection -│ │ └── yolo_tcpformer_processor.py # YOLO 2D + TCPFormer 3D lifting (81-frame temporal) -│ ├── utils/ -│ │ ├── __init__.py -│ │ ├── cache.py # Model caching utilities -│ │ ├── filters.py # MedianFilter, GaussianFilter for temporal smoothing -│ │ ├── io.py # Frame I/O, encoding/decoding -│ │ ├── kinetic.py # Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24) -│ │ ├── locate_path.py # Project root detection -│ │ ├── log_streamer.py # SocketIOLogHandler for real-time log streaming -│ │ └── logger.py # Structured logging setup -│ └── models/ -│ ├── tcpformer/ # TCPFormer 2D→3D lifting model (AAAI 2025) -│ │ ├── model.py -│ │ └── *.pth.tr # Checkpoint (auto-downloaded) -│ ├── efficientdet_lite2.tflite # MediaPipe object detection model -│ └── pose_landmarker_full.task # MediaPipe pose landmarker model -│ -├── frontend/ -│ ├── src/ -│ │ ├── main.tsx # App entry point -│ │ ├── App.tsx # Root layout: sidebar + viewer + log panel -│ │ ├── App.css # Global styles -│ │ ├── components/ -│ │ │ ├── CameraCapture.tsx # Camera/video source lifecycle, 10 FPS capture, backpressure -│ │ │ ├── Controls.tsx # Function selector, source/device picker, start/stop -│ │ │ ├── FunctionViewer.tsx # Routes to View2D, View3D, RoboticControlView, or placeholder by viewMode -│ │ │ ├── View2D.tsx # 2D canvas viewer with camera mirroring -│ │ │ ├── View3D.tsx # 3D scene with model selector, renderer toggle -│ │ │ ├── Skeleton3DViewer.tsx # Three.js canvas: video plane + skeleton + controls -│ │ │ ├── LogPanel.tsx # Real-time backend log viewer (right sidebar) -│ │ │ ├── ChatPanel.tsx # Chat UI with text input, voice recognition, SSE streaming -│ │ │ ├── RoboticControlView.tsx # Avatar + ChatPanel side-by-side layout for voice control -│ │ │ └── DataAnalysis_.tsx # Placeholder (unused) -│ │ ├── hooks/ -│ │ │ ├── useCameraDevices.ts # Camera device enumeration + permission management -│ │ │ ├── useLogStream.ts # Backend log subscription (rolling 1000-entry buffer) -│ │ │ ├── useSessionTimer.ts # 60s guest session countdown timer -│ │ │ ├── useVoiceRecognition.ts # Web Speech API wrapper for voice input -│ │ │ └── useWebSocket.ts # Socket connection + stale result filtering -│ │ ├── services/ -│ │ │ ├── socketService.ts # Socket.IO singleton client (VITE_BACKEND_URL) -│ │ │ ├── streamInitService.ts # Async stream init with model switching, 60s timeout -│ │ │ ├── streamService.ts # Per-stream frame transmission -│ │ │ └── secondBrainService.ts # SSE streaming to SecondBrain guest chat endpoint -│ │ ├── stores/ -│ │ │ └── appStore.ts # Zustand store (function, source, stream, renderer state) -│ │ ├── three/ -│ │ │ ├── AvatarRenderer.tsx # Mixamo rigged avatar with FK quaternion animation -│ │ │ ├── StickBallRenderer.tsx # Procedural ball-and-stick skeleton -│ │ │ ├── VideoPlane.tsx # Video feed texture on XY plane, camera mirroring -│ │ │ ├── boneMapping.ts # Mixamo bone names, joint→bone maps, FK transforms -│ │ │ └── connections.ts # Skeleton bone connection topology -│ │ └── types/ -│ │ ├── functions.ts # Function definitions, processor types, view modes -│ │ ├── chat.ts # Chat message, session, and streaming types -│ │ └── pose.ts # Landmark, PoseData, DetectedObject, DetectedHand types -│ ├── public/ -│ │ └── avatars/skeleton.glb # Mixamo rigged skeleton model -│ ├── .env.local # Dev: VITE_BACKEND_URL=http://localhost:49101 -│ ├── .env.production # Prod: VITE_BACKEND_URL=https://pose-backend.yingliu.site -│ ├── package.json -│ ├── vite.config.ts # Vite config (port 8585, @ path alias) -│ └── run_ui.sh # Local dev startup script -│ -├── .github/workflows/ -│ ├── deploy_backend.yml # CI/CD: backend to production (main branch) -│ ├── deploy_backend_staging.yml # CI/CD: backend to staging (staging branch) -│ ├── deploy_frontend.yml # CI/CD: frontend to production (main branch) -│ └── deploy_frontend_staging.yml # CI/CD: frontend to staging (staging branch) -│ -├── .claude/ -│ ├── PROJECT_STRUCTURE.md # This file -│ └── skills/ -│ ├── develop/SKILL.md # Full development workflow -│ ├── code-review/SKILL.md # Code review workflow -│ ├── test/SKILL.md # Test workflow -│ └── ssh-servers/SKILL.md # Remote server access -│ -├── tests/ -│ ├── test_solve_ik.py # Integration test for solve_ik (local + Socket.IO) -│ ├── playwright.config.ts # Playwright config (production) -│ ├── playwright.staging.config.ts # Playwright config (staging) -│ └── specs/ -│ ├── pose-validation.spec.ts # E2E pose detection + 3D rendering tests -│ ├── avatar-voice-control.spec.ts # E2E avatar voice control tests (8 tests) -│ └── staging-video-test.spec.ts # Staging-specific video upload tests -│ -├── README.md # Project overview and setup guide -├── CHANGELOG.md # Version history -├── output/ # Processing output directory -├── logs/ # Backend log files (date-stamped) -└── .cache/ # Runtime model cache -``` - -## Function Modes - -The app operates in single-function mode with five available functions: - -| Function | Processor | View | Description | -|----------|-----------|------|-------------| -| 2D Pose Estimation | `yolo_pose_2d` | 2D | YOLOv8-Pose 2D keypoint detection | -| 3D Pose Estimation | `mediapipe` or `rtmpose` | 3D | Switchable 3D model with avatar/skeleton rendering | -| Object Detection | `mediapipe_object_detection` | 2D | EfficientDet bounding boxes + labels | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | 2D | Per-hand landmarks + gesture classification | -| Avatar Voice Control | — (SecondBrain) | voice | Voice/text commands → SecondBrain → solve_ik → avatar | - -## Core Components - -### Backend - -**app.py** - FastAPI server with Socket.IO, CORS, endpoints: `/` (info), `/health` (stats), `/info` (features) - -**config.py** - Centralized settings with env var support: -- `POSE_STUDIO_HOST` (default `0.0.0.0`), `POSE_STUDIO_PORT` (default `49101`) -- `POSE_WORKERS` — thread pool size (default `min(cpu_count, 16)`) -- `MAX_CONCURRENT_STREAMS` — server-wide limit (default `3`) -- BLAS thread pinning (`OMP_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`) -- CORS origins: `localhost:8585`, `robot.yingliu.site`, `staging.robot.yingliu.site` -- GPU detection (ONNX Runtime CUDA, PyTorch CUDA) -- Config merging from `config_template.json` - -**websocket_handler.py** - Manages: -- Client connections/disconnections with per-client cleanup -- Stream initialization with multi-processor pipeline -- Concurrent frame processing via `ThreadPoolExecutor(max_workers=POSE_WORKERS)` -- Per-stream timing metrics exposed on `/health` -- Real-time log streaming (`subscribe_logs` / `unsubscribe_logs`) -- Model switching (`switch_model` event) -- IK solving (`solve_ik` event → FK quaternion result via Converter) -- Concurrent stream limit enforcement - -**Processors** (all inherit from `base_processor.py`): - -| Processor | Output | Notes | -|-----------|--------|-------| -| `mediapipe_processor` | 2D/3D landmarks (33→24 unified) | LIVE_STREAM for camera, VIDEO for uploads | -| `rtmpose_processor` | FK quaternions + root position | RTMW3D-X + YOLOX-M, depth-corrected z | -| `yolo_pose_2d_processor` | 2D landmarks (17 COCO keypoints) | YOLOv8-M Pose | -| `yolo_tcpformer_processor` | FK + world landmarks | YOLO 2D → TCPFormer temporal 3D lifting | -| `mediapipe_object_detector_processor` | Bounding boxes + labels | EfficientDet-Lite2 | -| `mediapipe_hand_gesture_processor` | Hand landmarks + gestures | Per-hand classification | -| `image_processor` | Preprocessed frame | Resize, flip, normalize | -| `data_processor` | Feature vectors | Temporal smoothing, windowing | - -**Utilities:** -- `kinetic.py` — Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24 joints) -- `filters.py` — MedianFilter, GaussianFilter for temporal smoothing -- `io.py` — Frame encoding/decoding -- `log_streamer.py` — SocketIOLogHandler for real-time log streaming to clients - -### Frontend - -**App.tsx** - Root layout: left sidebar (Controls), center (FunctionViewer), right sidebar (LogPanel). Animated background orbs, connection status indicator, error boundary. - -**Controls.tsx** - Function selector (radio buttons), source type toggle (camera/video), device picker, start/stop buttons. Camera permission requested on Start (not on mount). - -**FunctionViewer.tsx** - Routes to View2D, View3D, RoboticControlView (`voice`), or placeholder based on `functionDef.viewMode`. - -**View2D.tsx** - Canvas-based 2D viewer. Displays annotated frames from backend. Camera source mirroring via `scaleX(-1)`. - -**View3D.tsx** - 3D scene wrapper. Model selector dropdown (MediaPipe / YOLO+RTMPose). Avatar/skeleton renderer toggle. Manages video/canvas refs for Skeleton3DViewer. - -**Skeleton3DViewer.tsx** - Three.js Canvas with VideoPlane background + AvatarRenderer or StickBallRenderer. Orbit controls, grid, lighting, error boundary. - -**CameraCapture.tsx** - Camera/video source lifecycle. 10 FPS capture with backpressure (waits for backend result before sending next frame). - -**LogPanel.tsx** - Real-time backend log viewer. Severity-colored entries (DEBUG/INFO/WARNING/ERROR), auto-scroll, expand/collapse. - -**appStore.ts** (Zustand) - Centralized state: `activeFunction`, `sourceType`, `deviceId`, `videoFile`, `isStreamActive`, `isInitializing`, `initMessage`, `backendResult`, `rendererType` (avatar/stickball), `pose3dProcessorType` (mediapipe/rtmpose), sidebar collapse states, chat state (messages, streaming, session expiry). - -**Three.js renderers:** -- `AvatarRenderer` — Loads skeleton.glb, applies FK quaternions to Mixamo bones, T-pose caching, smooth interpolation -- `StickBallRenderer` — Procedural green spheres + lines using POSE_CONNECTIONS topology -- `VideoPlane` — Video texture on XY plane, camera source mirroring -- `boneMapping` — Mixamo bone names, joint→bone maps, conjugation rules, z-axis negation for legs - -## Architecture - -### Data Flow - -``` -Source (Camera 10 FPS / Video) → encode JPEG/Base64 → WebSocket - ↓ -WebSocket Handler → decode → ImageProcessor → DataProcessor → PoseProcessor - ↓ -encode JPEG/Base64 → WebSocket → Frontend - ↓ -View2D: Canvas render (annotated frame) -View3D: Three.js (VideoPlane + AvatarRenderer or StickBallRenderer) -``` - -### Stream Lifecycle - -**Initialize:** -``` -UI Start → requestPermission() → StreamInitService.initializeStream() -→ Backend creates processor pipeline → stream_initialized → Camera starts -``` - -**Process (with backpressure):** -``` -Camera captures → process_frame → Pipeline → pose_result -→ Update store → Render → next capture -``` - -**Cleanup:** -``` -Stop button → cleanup_processor → Backend releases processors -→ Reset permission → UI idle -``` - -## Deployment Architecture - -``` -GitHub Actions - │ - ├─ push to main ──────────────────────────────────────────────┐ - │ ├─ deploy_frontend.yml → build → rsync → nginx reload │ - │ │ Serves: https://robot.yingliu.site │ - │ └─ deploy_backend.yml → rsync → docker cp → restart │ - │ Serves: https://pose-backend.yingliu.site │ - │ │ - └─ push to staging ───────────────────────────────────────────┤ - ├─ deploy_frontend_staging.yml │ - │ Serves: https://staging.robot.yingliu.site │ - └─ deploy_backend_staging.yml │ - Serves: staging backend container │ - -VM1 (Frontend Edge) VM2 (GPU Backend) -┌─────────────────┐ ┌──────────────────────────┐ -│ Nginx │ │ Docker container │ -│ /var/www/frontend│ ──WS──► │ FastAPI + Socket.IO │ -│ Cloudflare TLS │ │ CUDA GPU acceleration │ -└─────────────────┘ └──────────────────────────┘ -``` - -## WebSocket Events - -### Client → Server - -| Event | Payload | -|-------|---------| -| `initialize_stream` | `{ stream_id, processor_type, processor_config, source_type }` | -| `process_frame` | `{ stream_id, frame (base64), timestamp_ms }` | -| `cleanup_processor` | `{ stream_id }` | -| `switch_model` | `{ stream_id, processor_type }` | -| `subscribe_logs` | `{}` | -| `unsubscribe_logs` | `{}` | -| `solve_ik` | `{ request_id, joints, root_position }` | - -### Server → Client - -| Event | Payload | -|-------|---------| -| `connection_status` | `{ status, sid }` | -| `stream_initialized` | `{ stream_id, status, message, processor_type }` | -| `stream_error` | `{ stream_id, message, code?, active_streams?, max_streams? }` | -| `stream_loading` | `{ stream_id, message }` | -| `pose_result` | `{ stream_id, frame (base64), pose_data, timestamp_ms }` | -| `model_switched` | `{ stream_id, processor_type, message }` | -| `log_batch` | `[{ level, message, timestamp, logger }]` | -| `error` | `{ message }` | -| `fk_result` | `{ request_id, fk_data, root_position, error? }` | - -## Configuration - -### Backend (config.py + config_template.json) -```python -HOST = os.getenv("POSE_STUDIO_HOST", "0.0.0.0") -PORT = int(os.getenv("POSE_STUDIO_PORT", 49101)) -POSE_WORKERS = min(cpu_count, 16) # env var override -MAX_CONCURRENT_STREAMS = 3 # env var override -``` - -### Frontend (environment files) -- `.env.local` → `VITE_BACKEND_URL=http://localhost:49101` -- `.env.production` → `VITE_BACKEND_URL=https://pose-backend.yingliu.site` -- `VITE_SECOND_BRAIN_URL` → SecondBrain guest chat API base URL - -## Tech Stack - -**Backend:** Python 3.13, FastAPI, Socket.IO, MediaPipe, rtmlib, Ultralytics (YOLOv8), PyTorch, OpenCV, NumPy - -**Frontend:** React 19, TypeScript, Three.js, React Three Fiber, Drei, Zustand, Socket.IO Client, Vite - -**Testing:** Playwright (E2E, production + staging configs) - -**CI/CD:** GitHub Actions, Cloudflare Tunnels (SSH via cloudflared + Access service tokens), rsync deployment - -**Infrastructure:** Nginx, Docker, NVIDIA CUDA, Cloudflare (TLS, DDoS, WAF) - -## Testing - -```bash -# From tests/ directory -npx playwright test # Run all tests -npx playwright test --config playwright.staging.config.ts # Staging tests -npx playwright test --headed # With visible browser -npx playwright show-report # View HTML report -``` - -## Debugging - -**Backend logs:** -```bash -tail -f logs/$(date +%Y-%m-%d).log -grep ERROR logs/*.log -``` - -**Real-time logs:** Open LogPanel in the right sidebar (frontend streams backend logs live) - -**Common issues:** -- Camera not starting → Check browser permissions -- No pose results → Check backend logs / LogPanel for initialization errors -- Low performance → Reduce FPS or resolution -- Stream limit reached → Max 3 concurrent streams (configurable) diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index d6cc45d..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(npx playwright install:*)", - "Bash(git status:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(git push:*)", - "Bash(GIT_EDITOR=true git rebase:*)", - "Bash(GIT_EDITOR=/usr/bin/true git rebase:*)" - ] - } -} diff --git a/.claude/skills/code-review/SKILL.md b/.claude/skills/code-review/SKILL.md deleted file mode 100644 index 7d70aa0..0000000 --- a/.claude/skills/code-review/SKILL.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -name: code-review ---- - -Review code changes **$ARGUMENTS** following the structured code review workflow below. - -## TODO List - -After identifying the scope of review (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each review phase (lint, quality, performance, testing) -- One task for each file or component under review - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This ensures thorough, systematic coverage of all review criteria. - -## Workflow Steps - -1. **Scope** -- Identify changes to review and establish review context -2. **Lint** -- Run linters and static analysis tools -3. **Code quality** -- Assess readability, structure, and conventions -4. **Performance** -- Analyze memory usage, speed, and algorithmic complexity -5. **Security** -- Check for vulnerabilities and sensitive data exposure -6. **Integration test** -- Validate changes work end-to-end in both directions -7. **Summary** -- Report findings and recommend actions - ---- - -## Step 1: Identify Review Scope - -Determine what code is under review: - -1. **Diff the changes** against the base branch: - ```bash - - git diff staging...HEAD --stat - git diff staging...HEAD - ``` -2. **List affected files** and categorize them (frontend, backend, config, docs) -3. **Read each changed file** in full to understand context around the diff -4. **Note the intent**: What is the change trying to accomplish? - -Document the following before proceeding: -- Files changed and lines affected -- Summary of what the changes do -- Which acceptance criteria (if any) apply - ---- - -## Step 2: Lint Check - -Run linters and static analysis for each affected area: - -### Frontend (TypeScript / React) -```bash -cd frontend -npx eslint src/ --ext .ts,.tsx -npx tsc --noEmit -``` - -### Backend (Python) -```bash -cd backend -python -m py_compile app.py -# Run any configured linters (flake8, ruff, mypy, etc.) -``` - -### Review for: -- Syntax errors or warnings -- Unused imports and variables -- Type errors or mismatches -- Formatting inconsistencies - -**If lint issues are found**: list each issue with file path and line number. - ---- - -## Step 3: Code Quality Review - -Assess the code for cleanliness and maintainability: - -### Readability -- Are variable and function names descriptive and consistent with existing conventions? -- Is the control flow easy to follow? -- Are complex sections commented where necessary (but not over-commented)? - -### Structure -- Does the code follow existing project patterns and architecture? -- Are responsibilities properly separated (e.g., no business logic in UI components)? -- Are new files placed in the correct directories per `PROJECT_STRUCTURE.md`? - -### Conventions -- Does the code match the style of surrounding code? -- Are TypeScript types properly defined (no unnecessary `any`)? -- Are Python type hints used consistently with the rest of the codebase? - -### Duplication -- Is there duplicated logic that should be extracted? -- Are there existing utilities or helpers that could be reused? - ---- - -## Step 4: Performance Review - -Analyze memory usage, speed, and algorithmic complexity: - -### Algorithmic complexity -- What is the Big O time and space complexity of new or modified logic? -- Are there unnecessary nested loops, redundant computations, or O(n^2) patterns that could be O(n)? - -### Memory -- Are large objects or arrays properly cleaned up? -- Are event listeners, intervals, or subscriptions properly disposed of? -- Are WebSocket connections and streams managed without leaks? - -### Rendering (Frontend) -- Are React components avoiding unnecessary re-renders? -- Are expensive computations memoized where appropriate? -- Are Three.js objects (geometries, materials, textures) properly disposed? - -### I/O (Backend) -- Are database queries or API calls efficient? -- Is caching used appropriately? -- Are WebSocket frame sizes and frequencies reasonable? - ---- - -## Step 5: Security Review - -Check for common vulnerabilities: - -- **No secrets in code**: No API keys, passwords, tokens, or credentials in committed files -- **Input validation**: User inputs are sanitized at system boundaries -- **Dependency safety**: No known vulnerable packages introduced -- **CORS / auth**: WebSocket and API endpoints have appropriate access controls -- **Data exposure**: No sensitive data logged or sent to the client unnecessarily - ---- - -## Step 6: Integration Testing - -Validate that changes work correctly end-to-end using two test configurations: - -### 6.1: Local frontend + Remote backend - -Test the frontend changes against the production backend: -1. Start the frontend locally: - ```bash - cd frontend - ./run_ui.sh - ``` -2. Configure the frontend to connect to the remote backend (`https://pose-backend.yingliu.site`) -3. Open `http://localhost:8585` in the browser -4. Validate: - - Does the UI render correctly? (function selector, source picker, log panel) - - Do all function modes work? (2D Pose, 3D Pose, Object Detection, Hand Gesture) - - Does 3D pose model switching (MediaPipe / YOLO+RTMPose) and avatar/skeleton toggle work? - - Are there console errors or broken WebSocket connections? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### 6.2: Remote frontend + Local backend - -Test the backend changes against the production frontend: -1. Start the backend locally: - ```bash - cd backend - ./run_server.sh - ``` -2. Access the remote frontend at `https://robot.yingliu.site` -3. Configure it to connect to the local backend (`http://localhost:49101`) -4. Validate: - - Does the backend serve pose/detection/gesture data correctly for all function modes? - - Are WebSocket streams stable? - - Does the log panel show backend logs in real time? - - Is the response time acceptable? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### Automated testing (optional) - -Use Playwright for automated validation. See `tests/README.md` for full setup. - -```bash -cd tests -npm test # run all tests -npm run test:headed # run with visible browser -``` - -Test specs are in `tests/specs/`: -- `pose-validation.spec.ts` — Main E2E suite (UI controls, video upload, pose detection) -- `staging-video-test.spec.ts` — Staging backend smoke test (uses `playwright.staging.config.ts`) - ---- - -## Step 7: Review Summary - -Compile findings into a structured report: - -### Report format - -```markdown -## Code Review Summary - -**Branch**: <branch-name> -**Files reviewed**: <count> -**Overall assessment**: PASS | PASS WITH COMMENTS | NEEDS CHANGES - -### Lint -- [ ] All lint checks pass - -### Code Quality -- [ ] Naming and conventions consistent -- [ ] Structure follows project patterns -- [ ] No unnecessary duplication - -### Performance -- [ ] No algorithmic concerns -- [ ] Memory management is correct -- [ ] No rendering or I/O bottlenecks - -### Security -- [ ] No secrets or credentials exposed -- [ ] Input validation present at boundaries - -### Integration -- [ ] Local frontend + remote backend: PASS / FAIL -- [ ] Remote frontend + local backend: PASS / FAIL - -### Issues Found -| # | Severity | File | Line | Description | -|---|----------|------|------|-------------| -| 1 | High/Medium/Low | path/to/file | 42 | Description of issue | - -### Recommendations -- <action items for the developer> -``` - -### Severity levels -| Level | Meaning | Action | -|-------|---------|--------| -| **High** | Bug, security issue, or data loss risk | Must fix before merge | -| **Medium** | Performance concern or convention violation | Should fix before merge | -| **Low** | Style nit or minor improvement | Optional, can fix later | - ---- - -## After Review - -Present the review summary to the user and use `AskUserQuestion` to determine next steps: - -**Question:** "How would you like to proceed after the review?" - -**Options:** -- Fix issues and re-review (Recommended) -- Address findings and run the review again -- Proceed to commit -- Accept current state and commit changes -- Discuss findings -- Talk through specific issues before deciding diff --git a/.claude/skills/develop/SKILL.md b/.claude/skills/develop/SKILL.md deleted file mode 100644 index 62234ab..0000000 --- a/.claude/skills/develop/SKILL.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -name: develop ---- - -Accept user requirement **$ARGUMENTS** following the full development workflow. - -## TODO List - -After understanding the request (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each acceptance criterion to implement -- One task for each affected file or component to build - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This systematic approach ensures all acceptance criteria are met and nothing is missed. - -## Workflow Overview - -**CRITICAL: Steps 1–5 are MANDATORY and BLOCKING. You MUST complete each step before proceeding to the next. Do NOT skip Steps 4 or 5.** - -| Step | Action | Mandatory | Notes | -|------|--------|-----------|-------| -| 1 | **Understand** | YES | Analyze requirements, define acceptance criteria | -| 2 | **Branch** | YES | Create a properly named branch from the correct base | -| 3 | **Implement** | YES | Make changes following code conventions and lint standards | -| 4 | **Validate** | **YES — BLOCKING** | Run ALL test steps in test/SKILL.md — NO exceptions | -| 5 | **Document** | **YES — BLOCKING** | Update CHANGELOG.md, PROJECT_STRUCTURE.md, README.md | -| 6 | **Review** | Optional | Optionally run a developer code review | -| 7 | **Commit** | YES | Stage files and commit with conventional message format | -| 8 | **Push & PR** | YES | Push branch and create pull request | - ---- - -## Step 1: Understand the Requirement - -Thoroughly analyze the requirement by: -1. Breaking down the request into discrete components -2. Identifying acceptance criteria, dependencies, and any linked PRDs or designs -3. Using `AskUserQuestion` to clarify any ambiguities -4. Reading [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) to understand the project architecture - -Document the following: -- Summary of the requirement -- Detailed description -- Acceptance criteria (what defines "done") -- Dependencies on other work -- Links to PRDs or design documents - ---- - -## Step 2: Create Feature Branch - -### Prepare the base - -1. Identify which folder(s) will be affected by the changes (refer to [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md)) -2. Fetch latest from remote: - ```bash - git fetch origin - ``` -3. Determine the base branch: - - If one release branch exists, use it - - If multiple exist, show the latest two and ask the user which to target using `AskUserQuestion` (mark the latest as "Recommended") - -### Create the branch - -For new features targeting production: -```bash -git checkout staging -git pull origin staging -git checkout -b type/brief-description -``` - -**Git flow:** `feature branch` → PR to `staging` → auto-deploy & test → PR to `main` → production - -### Branch naming: `type/brief-description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat/` | New feature | `feat/avatar-creation` | -| `fix/` | Bug fix | `fix/pose-detection-issue` | -| `docs/` | Documentation | `docs/update-structure` | -| `wip/` | Work in progress | `wip/experiment-joint-rotation` | - -**Rules:** lowercase, hyphens (not underscores), include ticket key if applicable, keep it brief. - -### Branching from in-progress work - -When a ticket depends on another ticket still in code review, branch from that ticket's branch instead. After the base ticket is merged, rebase onto the target branch and force push with `--force-with-lease`. - ---- - -## Step 3: Implement Changes - -**Function modes and processors** are defined in [frontend/src/types/functions.ts](frontend/src/types/functions.ts): - -| Function | Default Processor | Backend File | -|----------|------------------|--------------| -| 2D Pose Estimation | `yolo_pose_2d` | `processors/yolo_pose_2d_processor.py` | -| 3D Pose Estimation | `mediapipe` (switchable to `rtmpose`) | `processors/mediapipe_processor.py`, `processors/rtmpose_processor.py` | -| Object Detection | `mediapipe_object_detection` | `processors/mediapipe_object_detector_processor.py` | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | `processors/mediapipe_hand_gesture_processor.py` | -| Avatar Voice Control | — (SecondBrain guest API) | `services/secondBrainService.ts` | - -State management: [frontend/src/stores/appStore.ts](frontend/src/stores/appStore.ts) (Zustand store). - -Follow these guidelines during implementation: - -1. **Code conventions** — Maintain consistent formatting and style with existing codebase -2. **Acceptance criteria** — Ensure all acceptance criteria from Step 1 are fully met -3. **Update TODO** — Mark tasks as `in_progress` when starting, `completed` when finished -4. **Test as you go** — Validate changes incrementally to catch issues early -5. **Optimize complexity** — Analyze and improve Big O time/space complexity where possible -6. **Lint check** — Run linters after implementation (see below) -7. **Deployment requiremnt** - Update deployment pipeline if there are dependancies - -### Frontend lint - -```bash -cd frontend && npx tsc --noEmit -``` - -### Backend lint - -Compile-check all modified Python files: -```bash -cd backend && python -m py_compile app.py -# Also check any modified processor or core files, e.g.: -# python -m py_compile processors/mediapipe_processor.py -# python -m py_compile core/websocket_handler.py -# python -m py_compile config.py -``` - ---- - -## Step 4: Validate - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST read and execute EVERY sub-step defined in [.claude/skills/test/SKILL.md](.claude/skills/test/SKILL.md) before proceeding to Step 5. -> **This is NOT optional.** Do NOT skip, abbreviate, summarize, or reorder any sub-step. -> The test skill defines four compulsory steps plus user-approval gates — ALL must be completed. -> **If you skip this step or any sub-step, the entire workflow is invalid.** - -Execute the test skill steps in order: - -| Test Step | What to do | Gate | -|-----------|-----------|------| -| **Step 0** Pre-flight | Kill all servers; verify ports 49101 and 8585 are free | — | -| **Step 1** Automated E2E | Run `cd tests && npm test`; verify screenshots in `tests/results/` | — | -| **Step 2** Manual testing | Start dev servers; test video file and camera with all three pose models | **STOP — Ask user for approval before continuing to Step 3** | -| **Step 3** Staging | Commit → PR to `staging` → wait for CI deploy → health check → run staging tests | **STOP — Ask user for approval before continuing to Step 4** | -| **Step 4** Remote GPU | SSH health check + tail logs on staging and production backends | — | - -**After ALL test steps pass and the user explicitly approves at each gate**, return here and continue to Step 5. Do NOT proceed to Step 5 without completing every test step. - ---- - -## Step 5: Update Documentation - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST update all applicable documentation files listed below before proceeding to Step 6. -> **This is NOT optional.** Do NOT skip this step. Do NOT proceed to commit without completing documentation updates. -> For each file below, either update it OR explicitly state to the user why no update is needed. - -### 1. [CHANGELOG.md](CHANGELOG.md) - -Add an entry at the top of the appropriate `CHANGELOG.md`: - -```markdown -## <version> - <Day Month Year> - -- <brief feature summary (under 10 words)>: <Title> -``` - -If the version section doesn't exist, create it. - -### 2. [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) - -If changes affect project architecture (new files, moved directories, new processors), update this file accordingly. - -### 3. [README.md](README.md) - -If changes affect setup, usage, or dependencies, update the relevant `README.md`. - -### 4. TODO status - -Mark all completed tasks in your TODO list as `completed` using `TodoWrite`. - -### 5. Skill update - -Update any `.claude/skills/*/SKILL.md` files that are affected by workflow changes discovered during this session. - ---- - -## Step 6: Developer Review (Optional) - -Use `AskUserQuestion` to ask whether the user wants to run a Developer Review before committing: - -**Question:** "Would you like to run a Developer Review before committing?" - -**Options:** -- Yes (Recommended) — Review changes against base branch to catch issues early -- No — Skip review and proceed to commit - -If the user selects "Yes", invoke the `code-review` skill using the `Skill` tool: -``` -skill: "code-review" -``` -This executes [.claude/skills/code-review/SKILL.md](.claude/skills/code-review/SKILL.md). - ---- - -## Step 7: Commit Changes - -### Commit message format: `type: description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat:` | New feature | `feat: add avatar joint rotation support` | -| `fix:` | Bug fix | `fix: resolve pose detection accuracy issue` | -| `docs:` | Documentation | `docs: update PROJECT_STRUCTURE.md` | -| `style:` | Formatting only | `style: fix indentation in pose module` | -| `refactor:` | Code restructuring | `refactor: extract avatar axis alignment logic` | -| `perf:` | Performance improvement | `perf: optimize skeleton rendering` | -| `test:` | Tests | `test: add unit tests for pose service` | -| `chore:` | Build/tooling | `chore: update Python dependencies` | - -### Commit rules - -1. **Stage specific files** — Use `git add <file>` for each file (avoid `git add -A` or `git add .`) -2. **Imperative mood** — "add" not "added", "fix" not "fixed" -3. **Under 72 characters** -4. **No trailing period** - ---- - -## Step 8: Push Branch and Create Pull Request - -### Rebase before push - -```bash -git fetch origin -git rebase origin/staging -``` - -Resolve any conflicts before pushing. - -### Push - -```bash -git push -u origin <branch-name> -``` - -### Create PR - -Use `gh pr create` CLI command. - -**PR title:** Clear, descriptive, under 70 characters. - -**PR description** must include: -1. **Summary of changes** — Brief overview of what was implemented -2. **Link to issue/ticket** — Reference the original requirement or issue number -3. **Testing notes** — How to test the changes, steps to reproduce -4. **Screenshots** — Include screenshots for UI changes -5. **Acceptance criteria** — Confirm all criteria from Step 1 are met - ---- - -## Pre-PR Checklist - -Before creating the pull request, verify **every item**: - -- [ ] All acceptance criteria from Step 1 are fully met -- [ ] All TODO tasks are marked as `completed` -- [ ] **Test Step 0** — Stale servers killed, ports verified free -- [ ] **Test Step 1** — Playwright E2E tests passed; screenshots reviewed -- [ ] **Test Step 2** — Manual testing completed; user approved to proceed -- [ ] **Test Step 3** — Staging deployed, health check OK, staging tests passed; user approved to proceed -- [ ] **Test Step 4** — Remote GPU health check and logs reviewed -- [ ] Lint checks pass (`tsc --noEmit`, `py_compile`) -- [ ] Commit messages follow `type: description` format -- [ ] Branch name follows `type/brief-description` convention -- [ ] PR title is clear and descriptive -- [ ] Documentation updated ([CHANGELOG.md](CHANGELOG.md), [README.md](README.md), [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) as needed) -- [ ] No secrets, credentials, or sensitive data in committed code -- [ ] Code follows project conventions and existing patterns -- [ ] No unnecessary files added (temp files, IDE configs, build artifacts) diff --git a/.claude/skills/ssh-servers/SKILL.md b/.claude/skills/ssh-servers/SKILL.md deleted file mode 100644 index 669d8f7..0000000 --- a/.claude/skills/ssh-servers/SKILL.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -name: ssh-servers ---- - -SSH into the backend and/or frontend remote servers to inspect, debug, or manage deployed code. - -## Prerequisites - -Before connecting, ensure the SSH agent has the deploy key loaded: - -```bash -ssh-add -l # Check if keys are loaded -ssh-add ~/.ssh/github_deploy # RSA key (frontend) -ssh-add ~/.ssh/github_deploy_ed25519 # Ed25519 key (backend) -``` - -If `github_deploy_ed25519` has wrong permissions: -```bash -chmod 600 ~/.ssh/github_deploy_ed25519 -``` - -## SSH Hosts - -Both servers use Cloudflare Tunnels (via `cloudflared access ssh`) as configured in `~/.ssh/config`. - -### Backend (GPU Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-backend` | -| Hostname | `pose-backend-ssh.yingliu.site` | -| User | `root` | -| Key | `~/.ssh/github_deploy_ed25519` | -| Production container | `pose-spatial-studio-backend` (port 49101) | -| Staging container | `pose-spatial-studio-backend-staging` (port 49102) | -| Code path | `/root/backend/` (inside container) | -| Logs | `/root/backend/logs/app.log` (inside container) | - -**Connect:** -```bash -ssh pose-backend -``` - -**Interactive shell inside the container:** -```bash -docker exec -it pose-spatial-studio-backend bash -``` -This drops you into the container at `/root/backend/` where you can run Python, inspect files, install packages, and debug directly. - -**Common commands (run on host, prefix container commands with `docker exec`):** - -```bash -# Check backend health -docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health - -# View recent logs -docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log - -# Inspect deployed code -docker exec pose-spatial-studio-backend cat /root/backend/app.py -docker exec pose-spatial-studio-backend cat /root/backend/config.py -docker exec pose-spatial-studio-backend ls -la /root/backend/processors/ - -# Check installed packages -docker exec pose-spatial-studio-backend pip list | grep -iE 'mediapipe|rtm|onnx|torch|opencv' - -# Check GPU status -docker exec pose-spatial-studio-backend python3 -c "import onnxruntime as ort; print(ort.get_available_providers())" - -# Restart the backend app -docker exec pose-spatial-studio-backend bash -c "pkill -f 'python.*app.py'" || true -sleep 2 -docker start pose-spatial-studio-backend 2>/dev/null || true -docker exec -d pose-spatial-studio-backend bash -c "cd /root/backend && python app.py >> logs/app.log 2>&1" -``` - -### Frontend (Nginx Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-frontend` | -| Hostname | `pose-frontend-ssh.yingliu.site` | -| User | `sophia` | -| Key | `~/.ssh/github_deploy` (RSA) | -| Code path | `/var/www/frontend/` | -| Serves | `https://robot.yingliu.site` | - -**Connect:** -```bash -ssh pose-frontend -``` - -**Common commands:** - -```bash -# List deployed files -ls -la /var/www/frontend/ -ls -la /var/www/frontend/assets/ - -# View deployed index.html -cat /var/www/frontend/index.html - -# Check nginx status -sudo systemctl status nginx - -# Reload nginx after manual changes -sudo systemctl reload nginx -``` - -## Debugging Workflow - -When a problem exists on remote but not locally: - -1. **Check logs** on the backend for errors: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend tail -80 /root/backend/logs/app.log" - ``` - -2. **Compare deployed code** with local code: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend cat /root/backend/<file>" | diff - backend/<file> - ``` - -3. **Compare package versions** (common source of discrepancies): - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend pip list" > /tmp/remote-packages.txt - diff <(cat /tmp/remote-packages.txt) <(cd backend && .venv.nosync/bin/pip list) - ``` - -4. **Check frontend build output** — the remote serves a production Vite build, not dev mode: - ```bash - ssh pose-frontend "cat /var/www/frontend/index.html" - ``` - -## Quick Health Check (No SSH Required) - -```bash -# Production -curl -s https://pose-backend.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://robot.yingliu.site - -# Staging -curl -s https://pose-backend-staging.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://staging.robot.yingliu.site -``` diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md deleted file mode 100644 index 3c3a11f..0000000 --- a/.claude/skills/test/SKILL.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -name: test ---- - -Run the validation suite for **$ARGUMENTS** (or full validation if no arguments given). - -**CRITICAL: Steps 0, 1, 2, 3, and 4 are ALL MANDATORY and BLOCKING — do NOT skip ANY of them.** -**Steps 2 and 3 have USER-APPROVAL GATES — you MUST stop and get explicit user approval before continuing past them.** -**If you skip any step or gate, the entire validation is invalid and must be restarted from Step 0.** - -**IMPORTANT — run from the correct project:** All tests MUST run from the **working project directory** (the project with your code changes), NOT from wherever this SKILL.md lives. If you have multiple project copies (e.g., `pose-spatial-studio` and `pose-spatial-studio-1`), always `cd` into the one with your active feature branch before running any commands. - -**IMPORTANT — Do not make any changes in Staging and produc server from local project. It can only be done through GitHub CI/CD pipeline** ---- - -## Step 0: Pre-flight checks (MANDATORY) - -Before running any tests, **kill all existing servers** to prevent stale/wrong backends from being reused: - -```bash -pkill -f "app.py" 2>/dev/null; pkill -f "uvicorn" 2>/dev/null; pkill -f "run_server.sh" 2>/dev/null -pkill -f "vite" 2>/dev/null; pkill -f "run_ui.sh" 2>/dev/null -sleep 2 -# Verify ports are free: -lsof -i :49101 2>&1; lsof -i :8585 2>&1 -``` - -This is critical because Playwright's `reuseExistingServer: true` (the local default) will silently reuse any backend already running on port 49101 — even if it's from a different project copy. This causes false-positive test results. - ---- - -## Step 1: Automated E2E tests (MANDATORY) - -Run Playwright tests from the **working project's** `tests/` directory. Playwright auto-starts both backend (port 49101) and frontend (port 8585) via `webServer` config if they aren't already running. - -**Run all tests:** -```bash -cd tests && npm test -``` - -**Run a single spec on Chromium (minimum bar):** -```bash -cd tests && npx playwright test specs/pose-validation.spec.ts --project=chromium -``` - -**Other commands:** -```bash -cd tests && npm run test:headed # Visible browser -cd tests && npm run test:debug # Debug with breakpoints -cd tests && npm run test:ui # Interactive Playwright UI -cd tests && npm run test:report # View HTML report after a run -``` - -**Test specs** live in `tests/specs/`: -| Spec | Purpose | Config | -|------|---------|--------| -| `pose-validation.spec.ts` | Main E2E suite — UI controls, video upload, pose detection | default | -| `staging-video-test.spec.ts` | Staging backend smoke test | `playwright.staging.config.ts` | - -If tests fail, diagnose and fix before continuing. - -**Visual verification** — after tests pass, examine screenshots in `tests/results/` using the Read tool and verify: -- `pose-validation-avatar.png` — **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips -- `pose-validation-skeleton.png` — **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally -- Both screenshots should show the camera feed with a 2D skeleton overlay drawn on the image - ---- - -## Step 2: Manual testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Manual testing is complete. May I proceed to Step 3 (Staging environment testing)?" Do NOT continue to Step 3 without explicit user approval.** - -1. **Start dev environment** (if not already running): - - python venv environment is in `backend/.venv.nosync` - - Backend: `cd backend && ./run_server.sh` (port 49101) - - Frontend: `cd frontend && ./run_ui.sh` (port 8585) - - Confirm frontend connects to the **local** backend - -2. **Test with video file** (no camera needed): - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector (left sidebar) - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching: use the model selector dropdown in the 3D view to switch between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** functions - -3. **Visual checks** (take screenshots and examine with Read tool): - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips. avatar is not jittering. the movement is like a human. - - Toggle between Avatar and Skeleton using the inline button in the 3D view toolbar - - Switch model via the dropdown → re-check all of the above - -4. **Test with live camera** (when relevant): - - Select source type **Camera** → choose a camera device → click **Start** - - Verify pose landmarks overlay and 3D skeleton/avatar respond in real time - ---- - -## Step 3: Staging environment testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Staging tests are complete. May I proceed to Step 4 (Remote GPU validation)?" Do NOT continue to Step 4 without explicit user approval.** - -Validate changes against the staging backend before production deployment. - -**Staging infrastructure:** -- Backend container: `pose-spatial-studio-backend-staging` on VM2 port 49102 -- Staging URL: `https://pose-backend-staging.yingliu.site` -- CI/CD: `deploy_backend_staging.yml` triggers on push to `staging` branch - -**Workflow:** - -1. Commit changes and create/merge PR to `staging` branch and approve merge -2. Wait for the **Deploy backend (staging)** GitHub Actions workflow to pass -3. Health check: - ```bash - curl -s https://pose-backend-staging.yingliu.site/health - ``` -4. Run frontend locally against staging backend and wait for user approval: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev - ``` -5. **Test with video file:** - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -6. run the automated staging test: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev & - cd tests && npx playwright test specs/staging-video-test.spec.ts --config=playwright.staging.config.ts --project=chromium - ``` -7. **Visual verification** — take a screenshot and examine with the Read tool: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - - Toggle Avatar/Skeleton via inline button → switch model via dropdown → re-check -8. Check staging backend logs: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -30 /root/backend/logs/app.log" - ``` -9. **STOP** — Use `AskUserQuestion` to ask the user whether staging tests passed and whether to proceed to Step 4 - ---- - -## Step 4: Remote GPU validation (MANDATORY) - -Verify the staging backend is healthy and GPU-accessible. This applies to **all changes** because the backend always runs on GPU infrastructure. - -```bash -# Staging backend -ssh pose-backend "docker exec pose-spatial-studio-backend-staging curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -50 /root/backend/logs/app.log" - -# Production backend -ssh pose-backend "docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log" -``` - -**Visual verification** — run frontend against the production backend, take a screenshot, and examine with the Read tool: -1. `cd frontend && VITE_BACKEND_URL=https://pose-backend.yingliu.site npm run dev` -2. Open `http://localhost:8585` → Select **3D Pose Estimation** → Source type **Video File** → upload `tests/test.mp4` -3. Click **Start** → verify stream initializes → switch models between **MediaPipe** and **YOLO+RTMPose** -4. Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -5. Verify via screenshot: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - -See `/ssh-servers` skill for full remote debugging commands. - ---- - -## Validation checklist - -Before marking validation as complete, confirm ALL of the following: - -- [ ] **Step 0** — Pre-flight checks completed (servers killed, ports free) -- [ ] **Step 1** — Automated Playwright tests passed; screenshots verified -- [ ] **Step 2** — Manual testing completed; user approved to proceed -- [ ] **Step 3** — Staging environment tested (deployed, health check OK, pose detection works); user approved to proceed -- [ ] **Step 4** — Remote GPU validation passed (health check + logs reviewed) -- [ ] Implementation meets all acceptance criteria - -If any step fails → fix the issue and re-run from that step. - -**After ALL steps pass, return to develop/SKILL.md and continue with Step 5 (Update Documentation). Do NOT skip documentation updates.** \ No newline at end of file From 63d5432da820a73c73c5e568e923e704bebedd15 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 22:44:16 +0100 Subject: [PATCH 05/17] fix: rm frontend staging pipeline --- .github/workflows/deploy_frontend_staging.yml | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 .github/workflows/deploy_frontend_staging.yml diff --git a/.github/workflows/deploy_frontend_staging.yml b/.github/workflows/deploy_frontend_staging.yml deleted file mode 100644 index 5891423..0000000 --- a/.github/workflows/deploy_frontend_staging.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Deploy frontend (staging) - -on: - workflow_dispatch: - push: - branches: - - staging - paths: - - 'frontend/**' - - '.github/workflows/deploy_frontend_staging.yml' - -jobs: - deploy: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: frontend - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install deps - run: npm install - - - name: Build - run: npm run build - env: - VITE_BACKEND_URL: ${{ secrets.VITE_STAGING_BACKEND_URL }} - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Install cloudflared - run: | - curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - sudo mv cloudflared /usr/local/bin/cloudflared - sudo chmod +x /usr/local/bin/cloudflared - - - name: Configure SSH for Cloudflare - run: | - mkdir -p ~/.ssh - cat << 'EOF' >> ~/.ssh/config - Host vm-ssh - HostName ${{ secrets.SSH_HOST }} - User ${{ secrets.SSH_USER }} - ProxyCommand cloudflared access ssh --hostname %h --id ${{ secrets.CF_ACCESS_CLIENT_ID }} --secret ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - EOF - chmod 600 ~/.ssh/config - - - name: Test SSH via Cloudflare - run: ssh -o StrictHostKeyChecking=accept-new vm-ssh "echo 'SSH OK via Cloudflare'" - - - name: Deploy via rsync - run: | - ssh vm-ssh "sudo mkdir -p /var/www/staging && sudo chown ${{ secrets.SSH_USER }}:www-data /var/www/staging" - rsync -az --delete dist/ vm-ssh:/var/www/staging - - - name: Reload nginx - run: | - ssh vm-ssh "sudo systemctl reload nginx" From b386c9ac3bd715849e5c9e4da275cf8f01e80f71 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 23:34:42 +0100 Subject: [PATCH 06/17] fix: switch MediaPipe processors from LIVE_STREAM to synchronous VIDEO mode Eliminates segfaults in libmediapipe.so caused by async detect_async() race conditions in the native C++ layer (google/mediapipe#6085, #5466). - mediapipe_processor.py: remove ObjectDetector (now separate processor), switch PoseLandmarker to synchronous detect_for_video() - mediapipe_object_detector_processor.py: switch to detect_for_video() - mediapipe_hand_gesture_processor.py: switch to recognize_for_video() - View2D: lock container to 4:3 aspect ratio for consistent sizing - Skeleton3DViewer: zoom out camera to show full avatar + video Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- .../mediapipe_hand_gesture_processor.py | 33 ++------ .../mediapipe_object_detector_processor.py | 32 ++----- backend/processors/mediapipe_processor.py | 83 +------------------ frontend/src/components/View2D.tsx | 2 +- 4 files changed, 15 insertions(+), 135 deletions(-) diff --git a/backend/processors/mediapipe_hand_gesture_processor.py b/backend/processors/mediapipe_hand_gesture_processor.py index 55b02f9..37db1f9 100644 --- a/backend/processors/mediapipe_hand_gesture_processor.py +++ b/backend/processors/mediapipe_hand_gesture_processor.py @@ -3,7 +3,7 @@ Uses MediaPipe GestureRecognizer for combined hand landmark detection and gesture classification. Draws hand bounding boxes, 21-point hand skeleton with connections, and gesture/handedness labels on the frame. -Supports LIVE_STREAM (camera) and VIDEO (file) running modes. +Uses synchronous VIDEO running mode to avoid LIVE_STREAM segfaults. """ import cv2 @@ -11,7 +11,6 @@ import numpy as np from typing import Dict, Any, Optional import logging -import threading import subprocess import os @@ -67,18 +66,9 @@ def __init__(self, processor_id: str, 'min_tracking_confidence', 0.5) self.num_hands = int(pose_cfg.get('num_hands', 2)) self.device = pose_cfg.get('device', 'cpu') - self.source_type = pose_cfg.get('source_type', 'camera') self.gesture_recognizer = None self.last_timestamp = 0 - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_gesture_result = None - - def _gesture_result_callback(self, result, output_image, timestamp_ms): - with self.result_lock: - self.latest_gesture_result = result - def _get_delegate(self): if self.device == 'cuda': logger.info("HandGesture: using GPU delegate") @@ -96,9 +86,7 @@ def _try_initialize_with_delegate(self, delegate) -> bool: ["wget", "-O", self.model_path, _GESTURE_MODEL_URL], check=True) - use_live = self.source_type == 'camera' - running_mode = (mp.tasks.vision.RunningMode.LIVE_STREAM if use_live - else mp.tasks.vision.RunningMode.VIDEO) + running_mode = mp.tasks.vision.RunningMode.VIDEO logger.info(f"HandGesture running mode: {running_mode.name}") gr_kwargs = dict( @@ -111,8 +99,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: min_hand_presence_confidence=self.min_detection_confidence, min_tracking_confidence=self.min_tracking_confidence, ) - if use_live: - gr_kwargs['result_callback'] = self._gesture_result_callback self.gesture_recognizer = ( mp.tasks.vision.GestureRecognizer.create_from_options( mp.tasks.vision.GestureRecognizerOptions(**gr_kwargs))) @@ -145,9 +131,6 @@ def cleanup(self): if self.gesture_recognizer: self.gesture_recognizer.close() self.gesture_recognizer = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_gesture_result = None self._is_initialized = False logger.info( f"HandGesture processor {self.processor_id} cleaned up") @@ -180,15 +163,9 @@ def process_frame(self, frame: np.ndarray, image_format=mp.ImageFormat.SRGB, data=annotated.copy()) - # Run gesture recognition - if self.source_type == 'camera': - self.gesture_recognizer.recognize_async(mp_image, timestamp_ms) - with self.result_lock: - gesture_result = self.latest_gesture_result - else: - gesture_result = ( - self.gesture_recognizer.recognize_for_video( - mp_image, timestamp_ms)) + # Run gesture recognition (synchronous VIDEO mode) + gesture_result = self.gesture_recognizer.recognize_for_video( + mp_image, timestamp_ms) hands = [] if (gesture_result and gesture_result.hand_landmarks diff --git a/backend/processors/mediapipe_object_detector_processor.py b/backend/processors/mediapipe_object_detector_processor.py index 523315c..44bccc5 100644 --- a/backend/processors/mediapipe_object_detector_processor.py +++ b/backend/processors/mediapipe_object_detector_processor.py @@ -2,7 +2,7 @@ Detects all object categories using MediaPipe ObjectDetector (EfficientDet). Draws colored bounding boxes and category labels on the video frame. -Supports both LIVE_STREAM (camera) and VIDEO (file) running modes. +Uses synchronous VIDEO running mode to avoid LIVE_STREAM segfaults. """ import cv2 @@ -10,7 +10,6 @@ import numpy as np from typing import Dict, Any, Optional import logging -import threading import subprocess import os @@ -54,18 +53,9 @@ def __init__(self, processor_id: str, self.min_detection_confidence = pose_cfg.get('min_detection_confidence', 0.5) self.max_results = int(pose_cfg.get('max_results', 10)) self.device = pose_cfg.get('device', 'cpu') - self.source_type = pose_cfg.get('source_type', 'camera') self.object_detector = None self.last_timestamp = 0 - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_object_result = None - - def _detection_result_callback(self, result, output_image, timestamp_ms): - with self.result_lock: - self.latest_object_result = result - def _get_delegate(self): if self.device == 'cuda': logger.info("ObjectDetector: using GPU delegate") @@ -82,9 +72,7 @@ def _try_initialize_with_delegate(self, delegate) -> bool: subprocess.run( ["wget", "-O", self.model_path, model_url], check=True) - use_live = self.source_type == 'camera' - running_mode = (mp.tasks.vision.RunningMode.LIVE_STREAM if use_live - else mp.tasks.vision.RunningMode.VIDEO) + running_mode = mp.tasks.vision.RunningMode.VIDEO logger.info(f"ObjectDetector running mode: {running_mode.name}") od_kwargs = dict( @@ -94,8 +82,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: running_mode=running_mode, max_results=self.max_results, score_threshold=self.min_detection_confidence) - if use_live: - od_kwargs['result_callback'] = self._detection_result_callback self.object_detector = ( mp.tasks.vision.ObjectDetector.create_from_options( mp.tasks.vision.ObjectDetectorOptions(**od_kwargs))) @@ -128,9 +114,6 @@ def cleanup(self): if self.object_detector: self.object_detector.close() self.object_detector = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_object_result = None self._is_initialized = False logger.info( f"ObjectDetector processor {self.processor_id} cleaned up") @@ -163,14 +146,9 @@ def process_frame(self, frame: np.ndarray, data=cv2.resize(frame_rgb, (self.frame_width, self.frame_height))) - # Run detection - if self.source_type == 'camera': - self.object_detector.detect_async(mp_image, timestamp_ms) - with self.result_lock: - object_result = self.latest_object_result - else: - object_result = self.object_detector.detect_for_video( - mp_image, timestamp_ms) + # Run detection (synchronous VIDEO mode) + object_result = self.object_detector.detect_for_video( + mp_image, timestamp_ms) objects = [] if object_result and object_result.detections: diff --git a/backend/processors/mediapipe_processor.py b/backend/processors/mediapipe_processor.py index a8966fb..b3860f3 100644 --- a/backend/processors/mediapipe_processor.py +++ b/backend/processors/mediapipe_processor.py @@ -3,11 +3,8 @@ import numpy as np from typing import Dict, Any, Optional, List import logging -import threading from processors.base_processor import BaseProcessor import config -import subprocess -import os from utils.kinetic import Converter @@ -76,33 +73,15 @@ def __init__(self, processor_id: str, config_dict: Optional[Dict[str, Any]] = No self.config = config.merge_configs(config_dict) pose_processor_config = self.config['pose_processor'] self.pose_landmarker_model_path = pose_processor_config['pose_landmarker_model_name'] - self.object_detector_model_path = pose_processor_config['object_detector_model_name'] self.min_detection_confidence = pose_processor_config['min_detection_confidence'] self.min_tracking_confidence = pose_processor_config['min_tracking_confidence'] self.min_presence_confidence = pose_processor_config['min_presence_confidence'] self.pose_landmarker_frame_width = pose_processor_config['pose_landmarker_frame_width'] self.pose_landmarker_frame_height = pose_processor_config['pose_landmarker_frame_height'] self.num_poses = pose_processor_config['num_poses'] - self.object_detector_frame_width = pose_processor_config['object_detector_frame_width'] - self.object_detector_frame_height = pose_processor_config['object_detector_frame_height'] self.device = pose_processor_config.get('device', 'cpu') - self.source_type = pose_processor_config.get('source_type', 'camera') self.landmarker = None - self.object_detector = None self.last_timestamp = 0 - # LIVE_STREAM mode (camera) needs callbacks and a lock for async results - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_pose_result = None - self.latest_object_result = None - - def _pose_result_callback(self, result: mp.tasks.vision.PoseLandmarkerResult, output_image: mp.Image, timestamp_ms: int): - with self.result_lock: - self.latest_pose_result = result - - def _detection_result_callback(self, result: mp.tasks.vision.ObjectDetectorResult, output_image: mp.Image, timestamp_ms: int): - with self.result_lock: - self.latest_object_result = result def _get_delegate(self) -> 'mp.tasks.BaseOptions.Delegate': """Return GPU delegate if device is cuda, otherwise CPU.""" @@ -113,27 +92,9 @@ def _get_delegate(self) -> 'mp.tasks.BaseOptions.Delegate': return mp.tasks.BaseOptions.Delegate.CPU def _try_initialize_with_delegate(self, delegate) -> bool: - """Attempt to create MediaPipe tasks with the given delegate.""" - if not os.path.exists(self.object_detector_model_path): - model_url = MODEL_LINK.get(os.path.basename(self.object_detector_model_path)) - if model_url is None: - raise ValueError(f"No download URL found for model {self.object_detector_model_path}") - subprocess.run(["wget", "-O", self.object_detector_model_path, model_url], check=True) - - use_live = self.source_type == 'camera' - running_mode = mp.tasks.vision.RunningMode.LIVE_STREAM if use_live else mp.tasks.vision.RunningMode.VIDEO - logger.info(f"MediaPipe running mode: {running_mode.name} (source_type={self.source_type})") - - od_kwargs = dict( - base_options=mp.tasks.BaseOptions( - model_asset_path=self.object_detector_model_path, - delegate=delegate), - running_mode=running_mode, - max_results=5) - if use_live: - od_kwargs['result_callback'] = self._detection_result_callback - self.object_detector = mp.tasks.vision.ObjectDetector.create_from_options( - mp.tasks.vision.ObjectDetectorOptions(**od_kwargs)) + """Attempt to create MediaPipe PoseLandmarker with the given delegate.""" + running_mode = mp.tasks.vision.RunningMode.VIDEO + logger.info(f"MediaPipe running mode: {running_mode.name}") lm_kwargs = dict( base_options=mp.tasks.BaseOptions( @@ -145,8 +106,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: min_pose_presence_confidence=self.min_presence_confidence, min_tracking_confidence=self.min_tracking_confidence, output_segmentation_masks=False) - if use_live: - lm_kwargs['result_callback'] = self._pose_result_callback self.landmarker = mp.tasks.vision.PoseLandmarker.create_from_options( mp.tasks.vision.PoseLandmarkerOptions(**lm_kwargs)) return True @@ -196,37 +155,10 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: annotated_frame = cv2.resize(frame, (self.pose_landmarker_frame_width, self.pose_landmarker_frame_height)).copy() mp_pose_image = self._mp_image_from_frame(frame, self.pose_landmarker_frame_width, self.pose_landmarker_frame_height) - mp_object_image = self._mp_image_from_frame(frame, self.object_detector_frame_width, self.object_detector_frame_height) - if self.source_type == 'camera': - # LIVE_STREAM: async detection, read previous result - self.landmarker.detect_async(mp_pose_image, timestamp_ms) - self.object_detector.detect_async(mp_object_image, timestamp_ms) - with self.result_lock: - pose_result = self.latest_pose_result - object_result = self.latest_object_result - else: - # VIDEO: synchronous detection, returns current frame result - pose_result = self.landmarker.detect_for_video(mp_pose_image, timestamp_ms) - object_result = self.object_detector.detect_for_video(mp_object_image, timestamp_ms) + pose_result = self.landmarker.detect_for_video(mp_pose_image, timestamp_ms) landmarks, world_landmarks = [], [] - - if object_result and object_result.detections: - for obj in object_result.detections: - if not obj.categories or obj.categories[0].category_name != "person": - continue - bounding_box = obj.bounding_box - - scale_x = self.pose_landmarker_frame_width / self.object_detector_frame_width - scale_y = self.pose_landmarker_frame_height / self.object_detector_frame_height - - x1 = int(bounding_box.origin_x * scale_x) - y1 = int(bounding_box.origin_y * scale_y) - x2 = int((bounding_box.origin_x + bounding_box.width) * scale_x) - y2 = int((bounding_box.origin_y + bounding_box.height) * scale_y) - - cv2.rectangle(annotated_frame, (x1, y1), (x2, y2), (0, 255, 0), 2) if pose_result and pose_result.pose_landmarks: h, w, _ = annotated_frame.shape @@ -318,13 +250,6 @@ def cleanup(self): if self.landmarker: self.landmarker.close() self.landmarker = None - if self.object_detector: - self.object_detector.close() - self.object_detector = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_pose_result = None - self.latest_object_result = None self._is_initialized = False logger.info(f"MediaPipe processor {self.processor_id} cleaned up") diff --git a/frontend/src/components/View2D.tsx b/frontend/src/components/View2D.tsx index 839ad4c..3884d6c 100644 --- a/frontend/src/components/View2D.tsx +++ b/frontend/src/components/View2D.tsx @@ -38,7 +38,7 @@ export function View2D({ socket }: View2DProps) { }, [isStreamActive]); return ( - <div className="view-container" style={{ position: 'relative', width: '100%', height: '100%', minHeight: 400 }}> + <div className="view-container" style={{ position: 'relative', width: '100%', maxHeight: '100%', aspectRatio: '4 / 3' }}> {isStreamActive && <CameraCapture socket={socket} />} {!isStreamActive && !backendResult ? ( From 9f18e76e18ec8917d2b073e839c32c48892b38a0 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 23:54:08 +0100 Subject: [PATCH 07/17] fix: unify 2D view sizing by removing objectFit and centering container Lock container to 4:3 aspect ratio with margin auto centering. Remove objectFit: contain which caused variable black bar sizing across functions. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- frontend/src/components/View2D.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/components/View2D.tsx b/frontend/src/components/View2D.tsx index 3884d6c..e01abb4 100644 --- a/frontend/src/components/View2D.tsx +++ b/frontend/src/components/View2D.tsx @@ -38,7 +38,7 @@ export function View2D({ socket }: View2DProps) { }, [isStreamActive]); return ( - <div className="view-container" style={{ position: 'relative', width: '100%', maxHeight: '100%', aspectRatio: '4 / 3' }}> + <div className="view-container" style={{ position: 'relative', width: '100%', aspectRatio: '4 / 3', margin: '0 auto' }}> {isStreamActive && <CameraCapture socket={socket} />} {!isStreamActive && !backendResult ? ( @@ -61,7 +61,6 @@ export function View2D({ socket }: View2DProps) { style={{ width: '100%', height: '100%', - objectFit: 'contain', display: 'block', backgroundColor: '#000', borderRadius: 16, From cbafa80c6d9bd3d72088f31af8e81c11f90d1788 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sat, 4 Apr 2026 23:59:41 +0100 Subject: [PATCH 08/17] fix: normalize RTMPose world landmarks to match MediaPipe coordinate scale Apply fixed 0.25 scale factor to RTMPose world landmark coordinates so the 3D skeleton matches MediaPipe's normalized coordinate space. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/rtmpose_processor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 3e04ddb..2787c00 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -47,6 +47,7 @@ _Z_RANGE = 2.1744869 # dataset statistic: max root-relative depth in meters _Z_INPUT_HALF = 144.0 # codec input_size[2] / 2 = 288 / 2 (z dimension) _TORSO_LEG_HEIGHT = 1.35 # approximate shoulder-to-ankle height in meters +_MEDIAPIPE_BODY_SCALE = 0.85 # typical MediaPipe neck-to-ankle span in world units class RTMPoseProcessor(BaseProcessor): @@ -219,6 +220,14 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])), } + + scale = 0.25 + for jd in landmark_dict.values(): + if isinstance(jd, dict) and 'x' in jd: + jd['x'] *= scale + jd['y'] *= scale + jd['z'] *= scale + result.append(landmark_dict) return result From 9b53628af169b0205a5bb282a6b869336a6e23be Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:18:25 +0100 Subject: [PATCH 09/17] fix: optimise rtmpose speed --- backend/processors/rtmpose_processor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 2787c00..9c76038 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -1,5 +1,5 @@ import cv2 -from rtmlib import Wholebody3d, draw_skeleton +from rtmlib import PoseTracker, Wholebody3d, draw_skeleton from typing import Optional, List, Dict, Any from processors.base_processor import BaseProcessor from utils.kinetic import Converter @@ -59,7 +59,10 @@ def __init__(self, processor_id: str, config_dict: Optional[Dict[str, Any]] = No def initialize(self) -> bool: self._is_initialized = True - self.wholebody = Wholebody3d( + self.pose_tracker = PoseTracker( + Wholebody3d, + det_frequency=7, + tracking=False, mode='balanced', to_openpose=False, backend=self.backend, @@ -77,7 +80,7 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: return None frame = np.ascontiguousarray(frame) - keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.wholebody(frame) + keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.pose_tracker(frame) # Fix rtmlib z-depth bug: rtmlib decodes z using image height (384/2=192) # but the model codec uses z_input_size (288/2=144). Re-decode from raw From 0967c3abc1f921f83cd554c6500d8bf5a11347e7 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:26:01 +0100 Subject: [PATCH 10/17] fix: darken hand gesture bounding box colors for readability Change hand gesture colors from bright cyan/magenta to dark teal/magenta so white label text is readable against the bounding box backgrounds. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/mediapipe_hand_gesture_processor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/processors/mediapipe_hand_gesture_processor.py b/backend/processors/mediapipe_hand_gesture_processor.py index 37db1f9..23e4753 100644 --- a/backend/processors/mediapipe_hand_gesture_processor.py +++ b/backend/processors/mediapipe_hand_gesture_processor.py @@ -43,10 +43,10 @@ ]) # Colors (BGR) -_LEFT_HAND_COLOR = (255, 255, 0) # cyan -_RIGHT_HAND_COLOR = (255, 0, 255) # magenta -_JOINT_COLOR = (0, 255, 0) # green -_BONE_COLOR = (255, 128, 0) # light blue +_LEFT_HAND_COLOR = (180, 120, 0) # dark teal +_RIGHT_HAND_COLOR = (140, 0, 140) # dark magenta +_JOINT_COLOR = (0, 180, 0) # dark green +_BONE_COLOR = (160, 80, 0) # dark blue class MediaPipeHandGestureProcessor(BaseProcessor): From ff7e503c3f2697be00d3ed3d90abd15e4c51f420 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:32:26 +0100 Subject: [PATCH 11/17] fix: ensure onnxruntime-gpu in deploy pipelines, darken hand gesture colors Add step to uninstall CPU onnxruntime and reinstall onnxruntime-gpu after pip install requirements.txt, which pulls in the CPU variant via rtmlib dependency. Fixes RTMPose running on CPU instead of GPU after deploys. Also darken hand gesture bounding box colors for readability. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- .github/workflows/deploy_backend.yml | 3 +++ .github/workflows/deploy_backend_staging.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/deploy_backend.yml b/.github/workflows/deploy_backend.yml index 5f3b217..55efcbe 100644 --- a/.github/workflows/deploy_backend.yml +++ b/.github/workflows/deploy_backend.yml @@ -89,6 +89,9 @@ jobs: echo "=== Installing dependencies ===" docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" + echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime -y --break-system-packages 2>/dev/null; pip install --break-system-packages onnxruntime-gpu -q" + echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR if ! docker exec $CONTAINER test -f "$TCPFORMER_DIR/$TCPFORMER_FILE"; then diff --git a/.github/workflows/deploy_backend_staging.yml b/.github/workflows/deploy_backend_staging.yml index fde7976..a02b7a1 100644 --- a/.github/workflows/deploy_backend_staging.yml +++ b/.github/workflows/deploy_backend_staging.yml @@ -80,6 +80,9 @@ jobs: echo "=== Installing dependencies ===" docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" + echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime -y --break-system-packages 2>/dev/null; pip install --break-system-packages onnxruntime-gpu -q" + echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR if ! docker exec $CONTAINER test -f "$TCPFORMER_DIR/$TCPFORMER_FILE"; then From 4823ceb5e505121c6053300e51e9f61444571d99 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:35:04 +0100 Subject: [PATCH 12/17] fix: change rtmpose scale --- backend/processors/rtmpose_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 9c76038..6cf7436 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -224,7 +224,7 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, "presence": float(np.mean([person_scores[i] for i in valid])), } - scale = 0.25 + scale = 0.3 for jd in landmark_dict.values(): if isinstance(jd, dict) and 'x' in jd: jd['x'] *= scale From b2c8bfde778d00831615b094f0196dc6d33ded84 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:38:50 +0100 Subject: [PATCH 13/17] fix: uninstall both onnxruntime packages before force-reinstalling GPU variant Uninstalling only the CPU onnxruntime removes shared module files that onnxruntime-gpu also uses, breaking the installation. Now uninstall both packages and force-reinstall onnxruntime-gpu to ensure a clean state. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- .github/workflows/deploy_backend.yml | 2 +- .github/workflows/deploy_backend_staging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_backend.yml b/.github/workflows/deploy_backend.yml index 55efcbe..9d9b7d5 100644 --- a/.github/workflows/deploy_backend.yml +++ b/.github/workflows/deploy_backend.yml @@ -90,7 +90,7 @@ jobs: docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" - docker exec $CONTAINER bash -c "pip uninstall onnxruntime -y --break-system-packages 2>/dev/null; pip install --break-system-packages onnxruntime-gpu -q" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime onnxruntime-gpu -y --break-system-packages 2>/dev/null; pip install --break-system-packages --force-reinstall onnxruntime-gpu -q" echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR diff --git a/.github/workflows/deploy_backend_staging.yml b/.github/workflows/deploy_backend_staging.yml index a02b7a1..710125c 100644 --- a/.github/workflows/deploy_backend_staging.yml +++ b/.github/workflows/deploy_backend_staging.yml @@ -81,7 +81,7 @@ jobs: docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" - docker exec $CONTAINER bash -c "pip uninstall onnxruntime -y --break-system-packages 2>/dev/null; pip install --break-system-packages onnxruntime-gpu -q" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime onnxruntime-gpu -y --break-system-packages 2>/dev/null; pip install --break-system-packages --force-reinstall onnxruntime-gpu -q" echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR From 9d2ed9b19308359ea2daf032813ad1e82f07ab90 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:50:50 +0100 Subject: [PATCH 14/17] fix: use model root-relative 3D coords for stable skeleton scale Replace perspective unprojection (which caused skeleton to shrink/grow with arm movement due to body_height_px dependency) with direct use of RTMPose3D's root-relative keypoints_3d. Perspective unprojection now only computes root_position for scene placement. Removes scale hack. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/rtmpose_processor.py | 80 +++++++++++-------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 6cf7436..65f7ed1 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -47,7 +47,6 @@ _Z_RANGE = 2.1744869 # dataset statistic: max root-relative depth in meters _Z_INPUT_HALF = 144.0 # codec input_size[2] / 2 = 288 / 2 (z dimension) _TORSO_LEG_HEIGHT = 1.35 # approximate shoulder-to-ankle height in meters -_MEDIAPIPE_BODY_SCALE = 0.85 # typical MediaPipe neck-to-ankle span in world units class RTMPoseProcessor(BaseProcessor): @@ -69,6 +68,7 @@ def initialize(self) -> bool: device=self.device ) self._z_root_filter = MedianFilter(window_size=5) + self._root_positions = [] logger.info(f"RTMpose 3D processor {self.processor_id} initialized") return True @@ -114,14 +114,13 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: fk_data = self._fk_processing(world_landmarks) root_position = None - if world_landmarks: - hip_data = world_landmarks[0].get("hipCentre", {}) - if hip_data: - root_position = { - "x": float(hip_data.get("x", 0)), - "y": float(-hip_data.get("y", 0)), - "z": float(-hip_data.get("z", 0)) - } + if self._root_positions: + rp = self._root_positions[0] + root_position = { + "x": float(rp["x"]), + "y": float(-rp["y"]), + "z": float(-rp["z"]) + } return { "processed_frame": annotated_frame, @@ -164,21 +163,25 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, keypoints_2d: np.ndarray, scores: np.ndarray, w: int, h: int) -> List[Dict]: - """Build 3D world landmarks combining 2D image-space x,y with 3D z. + """Build 3D world landmarks using model's root-relative 3D coordinates. - Uses keypoints_2d (inverse-affine-transformed to image space by rtmlib) - for x,y with per-joint depth-corrected perspective unprojection. - Each joint's absolute depth (z_root + z_relative) is used instead of - a constant z_root, producing consistent 3D positions for accurate - FK bone direction computation. z comes from keypoints_3d (already - corrected for rtmlib's codec mismatch). + Uses keypoints_3d directly for skeleton shape (consistent scale + regardless of pose). Root position is computed separately via + perspective unprojection for scene placement. """ - f_est = float(max(w, h)) # estimated focal length (~53° VFOV) + f_est = float(max(w, h)) + self._root_positions = [] result = [] for person_3d, person_2d, person_scores in zip( keypoints_3d, keypoints_2d, scores): - # Estimate root depth from visible body extent (indices 5-16) + # Compute hip center in 3D model space (root reference) + hip_indices = [11, 12] + hip_3d_x = float(np.mean([person_3d[i][0] for i in hip_indices])) + hip_3d_y = float(np.mean([person_3d[i][1] for i in hip_indices])) + hip_3d_z = float(np.mean([person_3d[i][2] for i in hip_indices])) + + # Compute root position via perspective unprojection (for scene placement) visible_ys = [person_2d[i][1] for i in range(5, 17) if i < len(person_2d) and person_scores[i] > 0.3] if len(visible_ys) >= 2: @@ -186,19 +189,21 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, z_root = _TORSO_LEG_HEIGHT * f_est / max(body_height_px, 50.0) else: z_root = 3.0 - - # Smooth z_root across frames to reduce scale jitter z_root = float(self._z_root_filter.filter(np.array([z_root]))[0]) - # Person center in image space (average of hip keypoints) - hip_indices = [11, 12] - valid_hips = [i for i in hip_indices if i < len(person_2d)] - cx = float(np.mean([person_2d[i][0] for i in valid_hips])) - cy = float(np.mean([person_2d[i][1] for i in valid_hips])) + valid_hips_2d = [i for i in hip_indices if i < len(person_2d)] + cx = float(np.mean([person_2d[i][0] for i in valid_hips_2d])) + cy = float(np.mean([person_2d[i][1] for i in valid_hips_2d])) + self._root_positions.append({ + "x": (cx - w / 2) * z_root / f_est, + "y": (cy - h / 2) * z_root / f_est, + "z": z_root + }) + # Build root-relative landmarks from model's 3D output (stable scale) landmark_dict = {} for joint_name, indices in COCO133_TO_OUTPUT_JOINTS.items(): - valid = [i for i in indices if i < len(person_2d)] + valid = [i for i in indices if i < len(person_3d)] if not valid: landmark_dict[joint_name] = { "x": 0.0, "y": 0.0, "z": 0.0, @@ -206,31 +211,14 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, } continue - # Per-joint depth-corrected perspective unprojection: - # use each joint's absolute depth instead of constant z_root - xs, ys, zs = [], [], [] - for i in valid: - z_rel = person_3d[i][2] - z_abs = max(z_root + z_rel, 0.5) - xs.append((person_2d[i][0] - cx) * z_abs / f_est) - ys.append((person_2d[i][1] - cy) * z_abs / f_est) - zs.append(z_rel) - landmark_dict[joint_name] = { - "x": float(np.mean(xs)), - "y": float(np.mean(ys)), - "z": float(np.mean(zs)), + "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])), + "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])), + "z": float(np.mean([person_3d[i][2] - hip_3d_z for i in valid])), "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])), } - scale = 0.3 - for jd in landmark_dict.values(): - if isinstance(jd, dict) and 'x' in jd: - jd['x'] *= scale - jd['y'] *= scale - jd['z'] *= scale - result.append(landmark_dict) return result From a349cf52c3c60851f5b239f6b80ec2c9138d4288 Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:53:25 +0100 Subject: [PATCH 15/17] fix: scale RTMPose x,y from model-input pixels to meters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model keypoints_3d x,y are in model-input-space (~288x384 pixels) while z is already in meters. Apply _XY_SCALE (z_range/z_input_half ≈ 0.015) to convert x,y to meters, matching the z coordinate space. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/rtmpose_processor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 65f7ed1..079d4ea 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -47,6 +47,9 @@ _Z_RANGE = 2.1744869 # dataset statistic: max root-relative depth in meters _Z_INPUT_HALF = 144.0 # codec input_size[2] / 2 = 288 / 2 (z dimension) _TORSO_LEG_HEIGHT = 1.35 # approximate shoulder-to-ankle height in meters +# Scale factor to convert model-input-space x,y (pixels) to meters. +# Model input is 288x384, z_range covers 288 pixels → 2.17m, so ~0.015 m/pixel. +_XY_SCALE = _Z_RANGE / _Z_INPUT_HALF # ≈ 0.015 m/pixel class RTMPoseProcessor(BaseProcessor): @@ -212,8 +215,8 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, continue landmark_dict[joint_name] = { - "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])), - "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])), + "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])) * _XY_SCALE, + "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])) * _XY_SCALE, "z": float(np.mean([person_3d[i][2] - hip_3d_z for i in valid])), "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])), From d4a6f76722578ebe8edb72fb113ac01753d9fb7c Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:54:23 +0100 Subject: [PATCH 16/17] fix: decode all 3 axes from simcc for bbox-independent skeleton coordinates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously only z was decoded from simcc while x,y came from keypoints_3d which depends on the bbox affine crop — causing skeleton size to change when the detection bbox shifts. Now all three axes are decoded from the raw simcc codec space (input_size 288x384x288), making coordinates consistent regardless of bbox changes. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/rtmpose_processor.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 079d4ea..61ca871 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -47,9 +47,6 @@ _Z_RANGE = 2.1744869 # dataset statistic: max root-relative depth in meters _Z_INPUT_HALF = 144.0 # codec input_size[2] / 2 = 288 / 2 (z dimension) _TORSO_LEG_HEIGHT = 1.35 # approximate shoulder-to-ankle height in meters -# Scale factor to convert model-input-space x,y (pixels) to meters. -# Model input is 288x384, z_range covers 288 pixels → 2.17m, so ~0.015 m/pixel. -_XY_SCALE = _Z_RANGE / _Z_INPUT_HALF # ≈ 0.015 m/pixel class RTMPoseProcessor(BaseProcessor): @@ -85,12 +82,15 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: frame = np.ascontiguousarray(frame) keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.pose_tracker(frame) - # Fix rtmlib z-depth bug: rtmlib decodes z using image height (384/2=192) - # but the model codec uses z_input_size (288/2=144). Re-decode from raw - # simcc pixel values to get correct root-relative depth in meters. + # Decode all 3 axes from raw simcc values for consistent coordinates. + # keypoints_simcc is in codec pixel space: input_size=(288, 384, 288). + # Convert to centered, scaled metric coordinates. if keypoints_3d is not None and len(keypoints_3d) > 0: - z_pixel = keypoints_simcc[..., 2] - keypoints_3d[..., 2] = (z_pixel / _Z_INPUT_HALF - 1.0) * _Z_RANGE + _X_HALF = 144.0 # 288 / 2 + _Y_HALF = 192.0 # 384 / 2 + keypoints_3d[..., 0] = (keypoints_simcc[..., 0] / _X_HALF - 1.0) * _Z_RANGE + keypoints_3d[..., 1] = (keypoints_simcc[..., 1] / _Y_HALF - 1.0) * _Z_RANGE + keypoints_3d[..., 2] = (keypoints_simcc[..., 2] / _Z_INPUT_HALF - 1.0) * _Z_RANGE annotated_frame = draw_skeleton(frame, keypoints_2d, scores, kpt_thr=0.5) annotated_frame = cv2.resize(annotated_frame, (640, 480)) @@ -215,8 +215,8 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, continue landmark_dict[joint_name] = { - "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])) * _XY_SCALE, - "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])) * _XY_SCALE, + "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])), + "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])), "z": float(np.mean([person_3d[i][2] - hip_3d_z for i in valid])), "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])), From 8d34d4511428681ab5c21aee15fb30e52323743d Mon Sep 17 00:00:00 2001 From: Ying <ying.liu@yingliu.site> Date: Sun, 5 Apr 2026 00:59:20 +0100 Subject: [PATCH 17/17] fix: use 2D keypoints for stable x,y with shared z_root, simcc for z only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simcc x,y are bbox-crop-dependent — they scale when the detection bbox changes size. Instead, use keypoints_2d (stable image-space pixels) for x,y via perspective unprojection with a shared z_root for all joints. Skeleton proportions now come from stable 2D pixel ratios. Z depth still from corrected simcc (root-relative, bbox-independent). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- backend/processors/rtmpose_processor.py | 48 +++++++++++++------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 61ca871..d351756 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -82,14 +82,10 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: frame = np.ascontiguousarray(frame) keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.pose_tracker(frame) - # Decode all 3 axes from raw simcc values for consistent coordinates. - # keypoints_simcc is in codec pixel space: input_size=(288, 384, 288). - # Convert to centered, scaled metric coordinates. + # Fix z-depth: rtmlib decodes z using image height (384/2=192) instead of + # the codec z input size (288/2=144). Re-decode z from raw simcc values. + # x,y are kept from keypoints_2d (stable image-space pixels) in _build_world_landmarks. if keypoints_3d is not None and len(keypoints_3d) > 0: - _X_HALF = 144.0 # 288 / 2 - _Y_HALF = 192.0 # 384 / 2 - keypoints_3d[..., 0] = (keypoints_simcc[..., 0] / _X_HALF - 1.0) * _Z_RANGE - keypoints_3d[..., 1] = (keypoints_simcc[..., 1] / _Y_HALF - 1.0) * _Z_RANGE keypoints_3d[..., 2] = (keypoints_simcc[..., 2] / _Z_INPUT_HALF - 1.0) * _Z_RANGE annotated_frame = draw_skeleton(frame, keypoints_2d, scores, kpt_thr=0.5) @@ -166,11 +162,13 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, keypoints_2d: np.ndarray, scores: np.ndarray, w: int, h: int) -> List[Dict]: - """Build 3D world landmarks using model's root-relative 3D coordinates. + """Build 3D world landmarks using 2D keypoints for x,y and simcc z for depth. - Uses keypoints_3d directly for skeleton shape (consistent scale - regardless of pose). Root position is computed separately via - perspective unprojection for scene placement. + x,y: from keypoints_2d (stable image-space pixels) with perspective + unprojection using a shared z_root for all joints — skeleton + proportions come from stable 2D pixel ratios. + z: from keypoints_3d (corrected simcc z) — root-relative depth. + Root position computed separately for scene placement. """ f_est = float(max(w, h)) self._root_positions = [] @@ -178,13 +176,9 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, result = [] for person_3d, person_2d, person_scores in zip( keypoints_3d, keypoints_2d, scores): - # Compute hip center in 3D model space (root reference) hip_indices = [11, 12] - hip_3d_x = float(np.mean([person_3d[i][0] for i in hip_indices])) - hip_3d_y = float(np.mean([person_3d[i][1] for i in hip_indices])) - hip_3d_z = float(np.mean([person_3d[i][2] for i in hip_indices])) - # Compute root position via perspective unprojection (for scene placement) + # Estimate z_root from visible body extent (smoothed) visible_ys = [person_2d[i][1] for i in range(5, 17) if i < len(person_2d) and person_scores[i] > 0.3] if len(visible_ys) >= 2: @@ -194,19 +188,25 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, z_root = 3.0 z_root = float(self._z_root_filter.filter(np.array([z_root]))[0]) - valid_hips_2d = [i for i in hip_indices if i < len(person_2d)] - cx = float(np.mean([person_2d[i][0] for i in valid_hips_2d])) - cy = float(np.mean([person_2d[i][1] for i in valid_hips_2d])) + # Hip center in image space + valid_hips = [i for i in hip_indices if i < len(person_2d)] + cx = float(np.mean([person_2d[i][0] for i in valid_hips])) + cy = float(np.mean([person_2d[i][1] for i in valid_hips])) + + # Root position for scene placement self._root_positions.append({ "x": (cx - w / 2) * z_root / f_est, "y": (cy - h / 2) * z_root / f_est, "z": z_root }) - # Build root-relative landmarks from model's 3D output (stable scale) + # Hip center z from corrected simcc (for root-relative depth) + hip_3d_z = float(np.mean([person_3d[i][2] for i in hip_indices])) + + # Build landmarks: x,y from 2D pixels (shared z_root), z from simcc landmark_dict = {} for joint_name, indices in COCO133_TO_OUTPUT_JOINTS.items(): - valid = [i for i in indices if i < len(person_3d)] + valid = [i for i in indices if i < len(person_2d)] if not valid: landmark_dict[joint_name] = { "x": 0.0, "y": 0.0, "z": 0.0, @@ -214,9 +214,11 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, } continue + # x,y: perspective unprojection with shared z_root (stable proportions) + # z: root-relative depth from corrected simcc landmark_dict[joint_name] = { - "x": float(np.mean([person_3d[i][0] - hip_3d_x for i in valid])), - "y": float(np.mean([person_3d[i][1] - hip_3d_y for i in valid])), + "x": float(np.mean([(person_2d[i][0] - cx) * z_root / f_est for i in valid])), + "y": float(np.mean([(person_2d[i][1] - cy) * z_root / f_est for i in valid])), "z": float(np.mean([person_3d[i][2] - hip_3d_z for i in valid])), "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])),