diff --git a/.claude/PROJECT_STRUCTURE.md b/.claude/PROJECT_STRUCTURE.md deleted file mode 100644 index f218406..0000000 --- a/.claude/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,341 +0,0 @@ -# Project Structure - -## Directory Layout - -``` -pose-spatial-studio/ -├── backend/ -│ ├── app.py # FastAPI + Socket.IO server, health/info endpoints -│ ├── config.py # Settings (host, port, CORS, GPU detection, config merging) -│ ├── config_template.json # Default processor configuration template -│ ├── requirements.txt # Python deps (FastAPI, MediaPipe, torch, rtmlib, ultralytics) -│ ├── run_server.sh # Local dev startup script -│ ├── yolov8m-pose.pt # YOLOv8-M Pose model weights -│ ├── core/ -│ │ └── websocket_handler.py # Socket.IO stream management, processor pipeline, log streaming -│ ├── processors/ -│ │ ├── __init__.py -│ │ ├── base_processor.py # Abstract processor interface -│ │ ├── data_processor.py # Temporal smoothing, windowing, feature extraction -│ │ ├── image_processor.py # Frame preprocessing (resize, flip, normalize) -│ │ ├── mediapipe_processor.py # MediaPipe pose estimation (2D/3D landmarks) -│ │ ├── mediapipe_object_detector_processor.py # Object detection (EfficientDet) -│ │ ├── mediapipe_hand_gesture_processor.py # Hand gesture recognition -│ │ ├── rtmpose_processor.py # RTMPose3D via rtmlib (RTMW3D-X + YOLOX-M, FK output) -│ │ ├── yolo_pose_2d_processor.py # YOLOv8-Pose 2D detection -│ │ └── yolo_tcpformer_processor.py # YOLO 2D + TCPFormer 3D lifting (81-frame temporal) -│ ├── utils/ -│ │ ├── __init__.py -│ │ ├── cache.py # Model caching utilities -│ │ ├── filters.py # MedianFilter, GaussianFilter for temporal smoothing -│ │ ├── io.py # Frame I/O, encoding/decoding -│ │ ├── kinetic.py # Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24) -│ │ ├── locate_path.py # Project root detection -│ │ ├── log_streamer.py # SocketIOLogHandler for real-time log streaming -│ │ └── logger.py # Structured logging setup -│ └── models/ -│ ├── tcpformer/ # TCPFormer 2D→3D lifting model (AAAI 2025) -│ │ ├── model.py -│ │ └── *.pth.tr # Checkpoint (auto-downloaded) -│ ├── efficientdet_lite2.tflite # MediaPipe object detection model -│ └── pose_landmarker_full.task # MediaPipe pose landmarker model -│ -├── frontend/ -│ ├── src/ -│ │ ├── main.tsx # App entry point -│ │ ├── App.tsx # Root layout: sidebar + viewer + log panel -│ │ ├── App.css # Global styles -│ │ ├── components/ -│ │ │ ├── CameraCapture.tsx # Camera/video source lifecycle, 10 FPS capture, backpressure -│ │ │ ├── Controls.tsx # Function selector, source/device picker, start/stop -│ │ │ ├── FunctionViewer.tsx # Routes to View2D, View3D, RoboticControlView, or placeholder by viewMode -│ │ │ ├── View2D.tsx # 2D canvas viewer with camera mirroring -│ │ │ ├── View3D.tsx # 3D scene with model selector, renderer toggle -│ │ │ ├── Skeleton3DViewer.tsx # Three.js canvas: video plane + skeleton + controls -│ │ │ ├── LogPanel.tsx # Real-time backend log viewer (right sidebar) -│ │ │ ├── ChatPanel.tsx # Chat UI with text input, voice recognition, SSE streaming -│ │ │ ├── RoboticControlView.tsx # Avatar + ChatPanel side-by-side layout for voice control -│ │ │ └── DataAnalysis_.tsx # Placeholder (unused) -│ │ ├── hooks/ -│ │ │ ├── useCameraDevices.ts # Camera device enumeration + permission management -│ │ │ ├── useLogStream.ts # Backend log subscription (rolling 1000-entry buffer) -│ │ │ ├── useSessionTimer.ts # 60s guest session countdown timer -│ │ │ ├── useVoiceRecognition.ts # Web Speech API wrapper for voice input -│ │ │ └── useWebSocket.ts # Socket connection + stale result filtering -│ │ ├── services/ -│ │ │ ├── socketService.ts # Socket.IO singleton client (VITE_BACKEND_URL) -│ │ │ ├── streamInitService.ts # Async stream init with model switching, 60s timeout -│ │ │ ├── streamService.ts # Per-stream frame transmission -│ │ │ └── secondBrainService.ts # SSE streaming to SecondBrain guest chat endpoint -│ │ ├── stores/ -│ │ │ └── appStore.ts # Zustand store (function, source, stream, renderer state) -│ │ ├── three/ -│ │ │ ├── AvatarRenderer.tsx # Mixamo rigged avatar with FK quaternion animation -│ │ │ ├── StickBallRenderer.tsx # Procedural ball-and-stick skeleton -│ │ │ ├── VideoPlane.tsx # Video feed texture on XY plane, camera mirroring -│ │ │ ├── boneMapping.ts # Mixamo bone names, joint→bone maps, FK transforms -│ │ │ └── connections.ts # Skeleton bone connection topology -│ │ └── types/ -│ │ ├── functions.ts # Function definitions, processor types, view modes -│ │ ├── chat.ts # Chat message, session, and streaming types -│ │ └── pose.ts # Landmark, PoseData, DetectedObject, DetectedHand types -│ ├── public/ -│ │ └── avatars/skeleton.glb # Mixamo rigged skeleton model -│ ├── .env.local # Dev: VITE_BACKEND_URL=http://localhost:49101 -│ ├── .env.production # Prod: VITE_BACKEND_URL=https://pose-backend.yingliu.site -│ ├── package.json -│ ├── vite.config.ts # Vite config (port 8585, @ path alias) -│ └── run_ui.sh # Local dev startup script -│ -├── .github/workflows/ -│ ├── deploy_backend.yml # CI/CD: backend to production (main branch) -│ ├── deploy_backend_staging.yml # CI/CD: backend to staging (staging branch) -│ ├── deploy_frontend.yml # CI/CD: frontend to production (main branch) -│ └── deploy_frontend_staging.yml # CI/CD: frontend to staging (staging branch) -│ -├── .claude/ -│ ├── PROJECT_STRUCTURE.md # This file -│ └── skills/ -│ ├── develop/SKILL.md # Full development workflow -│ ├── code-review/SKILL.md # Code review workflow -│ ├── test/SKILL.md # Test workflow -│ └── ssh-servers/SKILL.md # Remote server access -│ -├── tests/ -│ ├── test_solve_ik.py # Integration test for solve_ik (local + Socket.IO) -│ ├── playwright.config.ts # Playwright config (production) -│ ├── playwright.staging.config.ts # Playwright config (staging) -│ └── specs/ -│ ├── pose-validation.spec.ts # E2E pose detection + 3D rendering tests -│ ├── avatar-voice-control.spec.ts # E2E avatar voice control tests (8 tests) -│ └── staging-video-test.spec.ts # Staging-specific video upload tests -│ -├── README.md # Project overview and setup guide -├── CHANGELOG.md # Version history -├── output/ # Processing output directory -├── logs/ # Backend log files (date-stamped) -└── .cache/ # Runtime model cache -``` - -## Function Modes - -The app operates in single-function mode with five available functions: - -| Function | Processor | View | Description | -|----------|-----------|------|-------------| -| 2D Pose Estimation | `yolo_pose_2d` | 2D | YOLOv8-Pose 2D keypoint detection | -| 3D Pose Estimation | `mediapipe` or `rtmpose` | 3D | Switchable 3D model with avatar/skeleton rendering | -| Object Detection | `mediapipe_object_detection` | 2D | EfficientDet bounding boxes + labels | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | 2D | Per-hand landmarks + gesture classification | -| Avatar Voice Control | — (SecondBrain) | voice | Voice/text commands → SecondBrain → solve_ik → avatar | - -## Core Components - -### Backend - -**app.py** - FastAPI server with Socket.IO, CORS, endpoints: `/` (info), `/health` (stats), `/info` (features) - -**config.py** - Centralized settings with env var support: -- `POSE_STUDIO_HOST` (default `0.0.0.0`), `POSE_STUDIO_PORT` (default `49101`) -- `POSE_WORKERS` — thread pool size (default `min(cpu_count, 16)`) -- `MAX_CONCURRENT_STREAMS` — server-wide limit (default `3`) -- BLAS thread pinning (`OMP_NUM_THREADS=1`, `MKL_NUM_THREADS=1`, `OPENBLAS_NUM_THREADS=1`) -- CORS origins: `localhost:8585`, `robot.yingliu.site`, `staging.robot.yingliu.site` -- GPU detection (ONNX Runtime CUDA, PyTorch CUDA) -- Config merging from `config_template.json` - -**websocket_handler.py** - Manages: -- Client connections/disconnections with per-client cleanup -- Stream initialization with multi-processor pipeline -- Concurrent frame processing via `ThreadPoolExecutor(max_workers=POSE_WORKERS)` -- Per-stream timing metrics exposed on `/health` -- Real-time log streaming (`subscribe_logs` / `unsubscribe_logs`) -- Model switching (`switch_model` event) -- IK solving (`solve_ik` event → FK quaternion result via Converter) -- Concurrent stream limit enforcement - -**Processors** (all inherit from `base_processor.py`): - -| Processor | Output | Notes | -|-----------|--------|-------| -| `mediapipe_processor` | 2D/3D landmarks (33→24 unified) | LIVE_STREAM for camera, VIDEO for uploads | -| `rtmpose_processor` | FK quaternions + root position | RTMW3D-X + YOLOX-M, depth-corrected z | -| `yolo_pose_2d_processor` | 2D landmarks (17 COCO keypoints) | YOLOv8-M Pose | -| `yolo_tcpformer_processor` | FK + world landmarks | YOLO 2D → TCPFormer temporal 3D lifting | -| `mediapipe_object_detector_processor` | Bounding boxes + labels | EfficientDet-Lite2 | -| `mediapipe_hand_gesture_processor` | Hand landmarks + gestures | Per-hand classification | -| `image_processor` | Preprocessed frame | Resize, flip, normalize | -| `data_processor` | Feature vectors | Temporal smoothing, windowing | - -**Utilities:** -- `kinetic.py` — Landmark format conversion (MediaPipe 33 / COCO 17 → unified 24 joints) -- `filters.py` — MedianFilter, GaussianFilter for temporal smoothing -- `io.py` — Frame encoding/decoding -- `log_streamer.py` — SocketIOLogHandler for real-time log streaming to clients - -### Frontend - -**App.tsx** - Root layout: left sidebar (Controls), center (FunctionViewer), right sidebar (LogPanel). Animated background orbs, connection status indicator, error boundary. - -**Controls.tsx** - Function selector (radio buttons), source type toggle (camera/video), device picker, start/stop buttons. Camera permission requested on Start (not on mount). - -**FunctionViewer.tsx** - Routes to View2D, View3D, RoboticControlView (`voice`), or placeholder based on `functionDef.viewMode`. - -**View2D.tsx** - Canvas-based 2D viewer. Displays annotated frames from backend. Camera source mirroring via `scaleX(-1)`. - -**View3D.tsx** - 3D scene wrapper. Model selector dropdown (MediaPipe / YOLO+RTMPose). Avatar/skeleton renderer toggle. Manages video/canvas refs for Skeleton3DViewer. - -**Skeleton3DViewer.tsx** - Three.js Canvas with VideoPlane background + AvatarRenderer or StickBallRenderer. Orbit controls, grid, lighting, error boundary. - -**CameraCapture.tsx** - Camera/video source lifecycle. 10 FPS capture with backpressure (waits for backend result before sending next frame). - -**LogPanel.tsx** - Real-time backend log viewer. Severity-colored entries (DEBUG/INFO/WARNING/ERROR), auto-scroll, expand/collapse. - -**appStore.ts** (Zustand) - Centralized state: `activeFunction`, `sourceType`, `deviceId`, `videoFile`, `isStreamActive`, `isInitializing`, `initMessage`, `backendResult`, `rendererType` (avatar/stickball), `pose3dProcessorType` (mediapipe/rtmpose), sidebar collapse states, chat state (messages, streaming, session expiry). - -**Three.js renderers:** -- `AvatarRenderer` — Loads skeleton.glb, applies FK quaternions to Mixamo bones, T-pose caching, smooth interpolation -- `StickBallRenderer` — Procedural green spheres + lines using POSE_CONNECTIONS topology -- `VideoPlane` — Video texture on XY plane, camera source mirroring -- `boneMapping` — Mixamo bone names, joint→bone maps, conjugation rules, z-axis negation for legs - -## Architecture - -### Data Flow - -``` -Source (Camera 10 FPS / Video) → encode JPEG/Base64 → WebSocket - ↓ -WebSocket Handler → decode → ImageProcessor → DataProcessor → PoseProcessor - ↓ -encode JPEG/Base64 → WebSocket → Frontend - ↓ -View2D: Canvas render (annotated frame) -View3D: Three.js (VideoPlane + AvatarRenderer or StickBallRenderer) -``` - -### Stream Lifecycle - -**Initialize:** -``` -UI Start → requestPermission() → StreamInitService.initializeStream() -→ Backend creates processor pipeline → stream_initialized → Camera starts -``` - -**Process (with backpressure):** -``` -Camera captures → process_frame → Pipeline → pose_result -→ Update store → Render → next capture -``` - -**Cleanup:** -``` -Stop button → cleanup_processor → Backend releases processors -→ Reset permission → UI idle -``` - -## Deployment Architecture - -``` -GitHub Actions - │ - ├─ push to main ──────────────────────────────────────────────┐ - │ ├─ deploy_frontend.yml → build → rsync → nginx reload │ - │ │ Serves: https://robot.yingliu.site │ - │ └─ deploy_backend.yml → rsync → docker cp → restart │ - │ Serves: https://pose-backend.yingliu.site │ - │ │ - └─ push to staging ───────────────────────────────────────────┤ - ├─ deploy_frontend_staging.yml │ - │ Serves: https://staging.robot.yingliu.site │ - └─ deploy_backend_staging.yml │ - Serves: staging backend container │ - -VM1 (Frontend Edge) VM2 (GPU Backend) -┌─────────────────┐ ┌──────────────────────────┐ -│ Nginx │ │ Docker container │ -│ /var/www/frontend│ ──WS──► │ FastAPI + Socket.IO │ -│ Cloudflare TLS │ │ CUDA GPU acceleration │ -└─────────────────┘ └──────────────────────────┘ -``` - -## WebSocket Events - -### Client → Server - -| Event | Payload | -|-------|---------| -| `initialize_stream` | `{ stream_id, processor_type, processor_config, source_type }` | -| `process_frame` | `{ stream_id, frame (base64), timestamp_ms }` | -| `cleanup_processor` | `{ stream_id }` | -| `switch_model` | `{ stream_id, processor_type }` | -| `subscribe_logs` | `{}` | -| `unsubscribe_logs` | `{}` | -| `solve_ik` | `{ request_id, joints, root_position }` | - -### Server → Client - -| Event | Payload | -|-------|---------| -| `connection_status` | `{ status, sid }` | -| `stream_initialized` | `{ stream_id, status, message, processor_type }` | -| `stream_error` | `{ stream_id, message, code?, active_streams?, max_streams? }` | -| `stream_loading` | `{ stream_id, message }` | -| `pose_result` | `{ stream_id, frame (base64), pose_data, timestamp_ms }` | -| `model_switched` | `{ stream_id, processor_type, message }` | -| `log_batch` | `[{ level, message, timestamp, logger }]` | -| `error` | `{ message }` | -| `fk_result` | `{ request_id, fk_data, root_position, error? }` | - -## Configuration - -### Backend (config.py + config_template.json) -```python -HOST = os.getenv("POSE_STUDIO_HOST", "0.0.0.0") -PORT = int(os.getenv("POSE_STUDIO_PORT", 49101)) -POSE_WORKERS = min(cpu_count, 16) # env var override -MAX_CONCURRENT_STREAMS = 3 # env var override -``` - -### Frontend (environment files) -- `.env.local` → `VITE_BACKEND_URL=http://localhost:49101` -- `.env.production` → `VITE_BACKEND_URL=https://pose-backend.yingliu.site` -- `VITE_SECOND_BRAIN_URL` → SecondBrain guest chat API base URL - -## Tech Stack - -**Backend:** Python 3.13, FastAPI, Socket.IO, MediaPipe, rtmlib, Ultralytics (YOLOv8), PyTorch, OpenCV, NumPy - -**Frontend:** React 19, TypeScript, Three.js, React Three Fiber, Drei, Zustand, Socket.IO Client, Vite - -**Testing:** Playwright (E2E, production + staging configs) - -**CI/CD:** GitHub Actions, Cloudflare Tunnels (SSH via cloudflared + Access service tokens), rsync deployment - -**Infrastructure:** Nginx, Docker, NVIDIA CUDA, Cloudflare (TLS, DDoS, WAF) - -## Testing - -```bash -# From tests/ directory -npx playwright test # Run all tests -npx playwright test --config playwright.staging.config.ts # Staging tests -npx playwright test --headed # With visible browser -npx playwright show-report # View HTML report -``` - -## Debugging - -**Backend logs:** -```bash -tail -f logs/$(date +%Y-%m-%d).log -grep ERROR logs/*.log -``` - -**Real-time logs:** Open LogPanel in the right sidebar (frontend streams backend logs live) - -**Common issues:** -- Camera not starting → Check browser permissions -- No pose results → Check backend logs / LogPanel for initialization errors -- Low performance → Reduce FPS or resolution -- Stream limit reached → Max 3 concurrent streams (configurable) diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index d6cc45d..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(npx playwright install:*)", - "Bash(git status:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(git push:*)", - "Bash(GIT_EDITOR=true git rebase:*)", - "Bash(GIT_EDITOR=/usr/bin/true git rebase:*)" - ] - } -} diff --git a/.claude/skills/code-review/SKILL.md b/.claude/skills/code-review/SKILL.md deleted file mode 100644 index 7d70aa0..0000000 --- a/.claude/skills/code-review/SKILL.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -name: code-review ---- - -Review code changes **$ARGUMENTS** following the structured code review workflow below. - -## TODO List - -After identifying the scope of review (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each review phase (lint, quality, performance, testing) -- One task for each file or component under review - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This ensures thorough, systematic coverage of all review criteria. - -## Workflow Steps - -1. **Scope** -- Identify changes to review and establish review context -2. **Lint** -- Run linters and static analysis tools -3. **Code quality** -- Assess readability, structure, and conventions -4. **Performance** -- Analyze memory usage, speed, and algorithmic complexity -5. **Security** -- Check for vulnerabilities and sensitive data exposure -6. **Integration test** -- Validate changes work end-to-end in both directions -7. **Summary** -- Report findings and recommend actions - ---- - -## Step 1: Identify Review Scope - -Determine what code is under review: - -1. **Diff the changes** against the base branch: - ```bash - - git diff staging...HEAD --stat - git diff staging...HEAD - ``` -2. **List affected files** and categorize them (frontend, backend, config, docs) -3. **Read each changed file** in full to understand context around the diff -4. **Note the intent**: What is the change trying to accomplish? - -Document the following before proceeding: -- Files changed and lines affected -- Summary of what the changes do -- Which acceptance criteria (if any) apply - ---- - -## Step 2: Lint Check - -Run linters and static analysis for each affected area: - -### Frontend (TypeScript / React) -```bash -cd frontend -npx eslint src/ --ext .ts,.tsx -npx tsc --noEmit -``` - -### Backend (Python) -```bash -cd backend -python -m py_compile app.py -# Run any configured linters (flake8, ruff, mypy, etc.) -``` - -### Review for: -- Syntax errors or warnings -- Unused imports and variables -- Type errors or mismatches -- Formatting inconsistencies - -**If lint issues are found**: list each issue with file path and line number. - ---- - -## Step 3: Code Quality Review - -Assess the code for cleanliness and maintainability: - -### Readability -- Are variable and function names descriptive and consistent with existing conventions? -- Is the control flow easy to follow? -- Are complex sections commented where necessary (but not over-commented)? - -### Structure -- Does the code follow existing project patterns and architecture? -- Are responsibilities properly separated (e.g., no business logic in UI components)? -- Are new files placed in the correct directories per `PROJECT_STRUCTURE.md`? - -### Conventions -- Does the code match the style of surrounding code? -- Are TypeScript types properly defined (no unnecessary `any`)? -- Are Python type hints used consistently with the rest of the codebase? - -### Duplication -- Is there duplicated logic that should be extracted? -- Are there existing utilities or helpers that could be reused? - ---- - -## Step 4: Performance Review - -Analyze memory usage, speed, and algorithmic complexity: - -### Algorithmic complexity -- What is the Big O time and space complexity of new or modified logic? -- Are there unnecessary nested loops, redundant computations, or O(n^2) patterns that could be O(n)? - -### Memory -- Are large objects or arrays properly cleaned up? -- Are event listeners, intervals, or subscriptions properly disposed of? -- Are WebSocket connections and streams managed without leaks? - -### Rendering (Frontend) -- Are React components avoiding unnecessary re-renders? -- Are expensive computations memoized where appropriate? -- Are Three.js objects (geometries, materials, textures) properly disposed? - -### I/O (Backend) -- Are database queries or API calls efficient? -- Is caching used appropriately? -- Are WebSocket frame sizes and frequencies reasonable? - ---- - -## Step 5: Security Review - -Check for common vulnerabilities: - -- **No secrets in code**: No API keys, passwords, tokens, or credentials in committed files -- **Input validation**: User inputs are sanitized at system boundaries -- **Dependency safety**: No known vulnerable packages introduced -- **CORS / auth**: WebSocket and API endpoints have appropriate access controls -- **Data exposure**: No sensitive data logged or sent to the client unnecessarily - ---- - -## Step 6: Integration Testing - -Validate that changes work correctly end-to-end using two test configurations: - -### 6.1: Local frontend + Remote backend - -Test the frontend changes against the production backend: -1. Start the frontend locally: - ```bash - cd frontend - ./run_ui.sh - ``` -2. Configure the frontend to connect to the remote backend (`https://pose-backend.yingliu.site`) -3. Open `http://localhost:8585` in the browser -4. Validate: - - Does the UI render correctly? (function selector, source picker, log panel) - - Do all function modes work? (2D Pose, 3D Pose, Object Detection, Hand Gesture) - - Does 3D pose model switching (MediaPipe / YOLO+RTMPose) and avatar/skeleton toggle work? - - Are there console errors or broken WebSocket connections? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### 6.2: Remote frontend + Local backend - -Test the backend changes against the production frontend: -1. Start the backend locally: - ```bash - cd backend - ./run_server.sh - ``` -2. Access the remote frontend at `https://robot.yingliu.site` -3. Configure it to connect to the local backend (`http://localhost:49101`) -4. Validate: - - Does the backend serve pose/detection/gesture data correctly for all function modes? - - Are WebSocket streams stable? - - Does the log panel show backend logs in real time? - - Is the response time acceptable? - -**If issues are found**: return to development using the develop skill (`/develop`), then re-run this review. - -### Automated testing (optional) - -Use Playwright for automated validation. See `tests/README.md` for full setup. - -```bash -cd tests -npm test # run all tests -npm run test:headed # run with visible browser -``` - -Test specs are in `tests/specs/`: -- `pose-validation.spec.ts` — Main E2E suite (UI controls, video upload, pose detection) -- `staging-video-test.spec.ts` — Staging backend smoke test (uses `playwright.staging.config.ts`) - ---- - -## Step 7: Review Summary - -Compile findings into a structured report: - -### Report format - -```markdown -## Code Review Summary - -**Branch**: -**Files reviewed**: -**Overall assessment**: PASS | PASS WITH COMMENTS | NEEDS CHANGES - -### Lint -- [ ] All lint checks pass - -### Code Quality -- [ ] Naming and conventions consistent -- [ ] Structure follows project patterns -- [ ] No unnecessary duplication - -### Performance -- [ ] No algorithmic concerns -- [ ] Memory management is correct -- [ ] No rendering or I/O bottlenecks - -### Security -- [ ] No secrets or credentials exposed -- [ ] Input validation present at boundaries - -### Integration -- [ ] Local frontend + remote backend: PASS / FAIL -- [ ] Remote frontend + local backend: PASS / FAIL - -### Issues Found -| # | Severity | File | Line | Description | -|---|----------|------|------|-------------| -| 1 | High/Medium/Low | path/to/file | 42 | Description of issue | - -### Recommendations -- -``` - -### Severity levels -| Level | Meaning | Action | -|-------|---------|--------| -| **High** | Bug, security issue, or data loss risk | Must fix before merge | -| **Medium** | Performance concern or convention violation | Should fix before merge | -| **Low** | Style nit or minor improvement | Optional, can fix later | - ---- - -## After Review - -Present the review summary to the user and use `AskUserQuestion` to determine next steps: - -**Question:** "How would you like to proceed after the review?" - -**Options:** -- Fix issues and re-review (Recommended) -- Address findings and run the review again -- Proceed to commit -- Accept current state and commit changes -- Discuss findings -- Talk through specific issues before deciding diff --git a/.claude/skills/develop/SKILL.md b/.claude/skills/develop/SKILL.md deleted file mode 100644 index 62234ab..0000000 --- a/.claude/skills/develop/SKILL.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -name: develop ---- - -Accept user requirement **$ARGUMENTS** following the full development workflow. - -## TODO List - -After understanding the request (Step 1), **create a TODO list** using the `TodoWrite` tool with: -- One task for each acceptance criterion to implement -- One task for each affected file or component to build - -Mark each task as `in_progress` when starting work on it, and `completed` immediately when finished. This systematic approach ensures all acceptance criteria are met and nothing is missed. - -## Workflow Overview - -**CRITICAL: Steps 1–5 are MANDATORY and BLOCKING. You MUST complete each step before proceeding to the next. Do NOT skip Steps 4 or 5.** - -| Step | Action | Mandatory | Notes | -|------|--------|-----------|-------| -| 1 | **Understand** | YES | Analyze requirements, define acceptance criteria | -| 2 | **Branch** | YES | Create a properly named branch from the correct base | -| 3 | **Implement** | YES | Make changes following code conventions and lint standards | -| 4 | **Validate** | **YES — BLOCKING** | Run ALL test steps in test/SKILL.md — NO exceptions | -| 5 | **Document** | **YES — BLOCKING** | Update CHANGELOG.md, PROJECT_STRUCTURE.md, README.md | -| 6 | **Review** | Optional | Optionally run a developer code review | -| 7 | **Commit** | YES | Stage files and commit with conventional message format | -| 8 | **Push & PR** | YES | Push branch and create pull request | - ---- - -## Step 1: Understand the Requirement - -Thoroughly analyze the requirement by: -1. Breaking down the request into discrete components -2. Identifying acceptance criteria, dependencies, and any linked PRDs or designs -3. Using `AskUserQuestion` to clarify any ambiguities -4. Reading [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) to understand the project architecture - -Document the following: -- Summary of the requirement -- Detailed description -- Acceptance criteria (what defines "done") -- Dependencies on other work -- Links to PRDs or design documents - ---- - -## Step 2: Create Feature Branch - -### Prepare the base - -1. Identify which folder(s) will be affected by the changes (refer to [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md)) -2. Fetch latest from remote: - ```bash - git fetch origin - ``` -3. Determine the base branch: - - If one release branch exists, use it - - If multiple exist, show the latest two and ask the user which to target using `AskUserQuestion` (mark the latest as "Recommended") - -### Create the branch - -For new features targeting production: -```bash -git checkout staging -git pull origin staging -git checkout -b type/brief-description -``` - -**Git flow:** `feature branch` → PR to `staging` → auto-deploy & test → PR to `main` → production - -### Branch naming: `type/brief-description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat/` | New feature | `feat/avatar-creation` | -| `fix/` | Bug fix | `fix/pose-detection-issue` | -| `docs/` | Documentation | `docs/update-structure` | -| `wip/` | Work in progress | `wip/experiment-joint-rotation` | - -**Rules:** lowercase, hyphens (not underscores), include ticket key if applicable, keep it brief. - -### Branching from in-progress work - -When a ticket depends on another ticket still in code review, branch from that ticket's branch instead. After the base ticket is merged, rebase onto the target branch and force push with `--force-with-lease`. - ---- - -## Step 3: Implement Changes - -**Function modes and processors** are defined in [frontend/src/types/functions.ts](frontend/src/types/functions.ts): - -| Function | Default Processor | Backend File | -|----------|------------------|--------------| -| 2D Pose Estimation | `yolo_pose_2d` | `processors/yolo_pose_2d_processor.py` | -| 3D Pose Estimation | `mediapipe` (switchable to `rtmpose`) | `processors/mediapipe_processor.py`, `processors/rtmpose_processor.py` | -| Object Detection | `mediapipe_object_detection` | `processors/mediapipe_object_detector_processor.py` | -| Hand Gesture Recognition | `mediapipe_hand_gesture` | `processors/mediapipe_hand_gesture_processor.py` | -| Avatar Voice Control | — (SecondBrain guest API) | `services/secondBrainService.ts` | - -State management: [frontend/src/stores/appStore.ts](frontend/src/stores/appStore.ts) (Zustand store). - -Follow these guidelines during implementation: - -1. **Code conventions** — Maintain consistent formatting and style with existing codebase -2. **Acceptance criteria** — Ensure all acceptance criteria from Step 1 are fully met -3. **Update TODO** — Mark tasks as `in_progress` when starting, `completed` when finished -4. **Test as you go** — Validate changes incrementally to catch issues early -5. **Optimize complexity** — Analyze and improve Big O time/space complexity where possible -6. **Lint check** — Run linters after implementation (see below) -7. **Deployment requiremnt** - Update deployment pipeline if there are dependancies - -### Frontend lint - -```bash -cd frontend && npx tsc --noEmit -``` - -### Backend lint - -Compile-check all modified Python files: -```bash -cd backend && python -m py_compile app.py -# Also check any modified processor or core files, e.g.: -# python -m py_compile processors/mediapipe_processor.py -# python -m py_compile core/websocket_handler.py -# python -m py_compile config.py -``` - ---- - -## Step 4: Validate - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST read and execute EVERY sub-step defined in [.claude/skills/test/SKILL.md](.claude/skills/test/SKILL.md) before proceeding to Step 5. -> **This is NOT optional.** Do NOT skip, abbreviate, summarize, or reorder any sub-step. -> The test skill defines four compulsory steps plus user-approval gates — ALL must be completed. -> **If you skip this step or any sub-step, the entire workflow is invalid.** - -Execute the test skill steps in order: - -| Test Step | What to do | Gate | -|-----------|-----------|------| -| **Step 0** Pre-flight | Kill all servers; verify ports 49101 and 8585 are free | — | -| **Step 1** Automated E2E | Run `cd tests && npm test`; verify screenshots in `tests/results/` | — | -| **Step 2** Manual testing | Start dev servers; test video file and camera with all three pose models | **STOP — Ask user for approval before continuing to Step 3** | -| **Step 3** Staging | Commit → PR to `staging` → wait for CI deploy → health check → run staging tests | **STOP — Ask user for approval before continuing to Step 4** | -| **Step 4** Remote GPU | SSH health check + tail logs on staging and production backends | — | - -**After ALL test steps pass and the user explicitly approves at each gate**, return here and continue to Step 5. Do NOT proceed to Step 5 without completing every test step. - ---- - -## Step 5: Update Documentation - -> **MANDATORY — BLOCKING STEP — DO NOT SKIP** -> -> You MUST update all applicable documentation files listed below before proceeding to Step 6. -> **This is NOT optional.** Do NOT skip this step. Do NOT proceed to commit without completing documentation updates. -> For each file below, either update it OR explicitly state to the user why no update is needed. - -### 1. [CHANGELOG.md](CHANGELOG.md) - -Add an entry at the top of the appropriate `CHANGELOG.md`: - -```markdown -## - - -- : -``` - -If the version section doesn't exist, create it. - -### 2. [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) - -If changes affect project architecture (new files, moved directories, new processors), update this file accordingly. - -### 3. [README.md](README.md) - -If changes affect setup, usage, or dependencies, update the relevant `README.md`. - -### 4. TODO status - -Mark all completed tasks in your TODO list as `completed` using `TodoWrite`. - -### 5. Skill update - -Update any `.claude/skills/*/SKILL.md` files that are affected by workflow changes discovered during this session. - ---- - -## Step 6: Developer Review (Optional) - -Use `AskUserQuestion` to ask whether the user wants to run a Developer Review before committing: - -**Question:** "Would you like to run a Developer Review before committing?" - -**Options:** -- Yes (Recommended) — Review changes against base branch to catch issues early -- No — Skip review and proceed to commit - -If the user selects "Yes", invoke the `code-review` skill using the `Skill` tool: -``` -skill: "code-review" -``` -This executes [.claude/skills/code-review/SKILL.md](.claude/skills/code-review/SKILL.md). - ---- - -## Step 7: Commit Changes - -### Commit message format: `type: description` - -| Type | Purpose | Example | -|------|---------|---------| -| `feat:` | New feature | `feat: add avatar joint rotation support` | -| `fix:` | Bug fix | `fix: resolve pose detection accuracy issue` | -| `docs:` | Documentation | `docs: update PROJECT_STRUCTURE.md` | -| `style:` | Formatting only | `style: fix indentation in pose module` | -| `refactor:` | Code restructuring | `refactor: extract avatar axis alignment logic` | -| `perf:` | Performance improvement | `perf: optimize skeleton rendering` | -| `test:` | Tests | `test: add unit tests for pose service` | -| `chore:` | Build/tooling | `chore: update Python dependencies` | - -### Commit rules - -1. **Stage specific files** — Use `git add <file>` for each file (avoid `git add -A` or `git add .`) -2. **Imperative mood** — "add" not "added", "fix" not "fixed" -3. **Under 72 characters** -4. **No trailing period** - ---- - -## Step 8: Push Branch and Create Pull Request - -### Rebase before push - -```bash -git fetch origin -git rebase origin/staging -``` - -Resolve any conflicts before pushing. - -### Push - -```bash -git push -u origin <branch-name> -``` - -### Create PR - -Use `gh pr create` CLI command. - -**PR title:** Clear, descriptive, under 70 characters. - -**PR description** must include: -1. **Summary of changes** — Brief overview of what was implemented -2. **Link to issue/ticket** — Reference the original requirement or issue number -3. **Testing notes** — How to test the changes, steps to reproduce -4. **Screenshots** — Include screenshots for UI changes -5. **Acceptance criteria** — Confirm all criteria from Step 1 are met - ---- - -## Pre-PR Checklist - -Before creating the pull request, verify **every item**: - -- [ ] All acceptance criteria from Step 1 are fully met -- [ ] All TODO tasks are marked as `completed` -- [ ] **Test Step 0** — Stale servers killed, ports verified free -- [ ] **Test Step 1** — Playwright E2E tests passed; screenshots reviewed -- [ ] **Test Step 2** — Manual testing completed; user approved to proceed -- [ ] **Test Step 3** — Staging deployed, health check OK, staging tests passed; user approved to proceed -- [ ] **Test Step 4** — Remote GPU health check and logs reviewed -- [ ] Lint checks pass (`tsc --noEmit`, `py_compile`) -- [ ] Commit messages follow `type: description` format -- [ ] Branch name follows `type/brief-description` convention -- [ ] PR title is clear and descriptive -- [ ] Documentation updated ([CHANGELOG.md](CHANGELOG.md), [README.md](README.md), [.claude/PROJECT_STRUCTURE.md](.claude/PROJECT_STRUCTURE.md) as needed) -- [ ] No secrets, credentials, or sensitive data in committed code -- [ ] Code follows project conventions and existing patterns -- [ ] No unnecessary files added (temp files, IDE configs, build artifacts) diff --git a/.claude/skills/ssh-servers/SKILL.md b/.claude/skills/ssh-servers/SKILL.md deleted file mode 100644 index 669d8f7..0000000 --- a/.claude/skills/ssh-servers/SKILL.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -name: ssh-servers ---- - -SSH into the backend and/or frontend remote servers to inspect, debug, or manage deployed code. - -## Prerequisites - -Before connecting, ensure the SSH agent has the deploy key loaded: - -```bash -ssh-add -l # Check if keys are loaded -ssh-add ~/.ssh/github_deploy # RSA key (frontend) -ssh-add ~/.ssh/github_deploy_ed25519 # Ed25519 key (backend) -``` - -If `github_deploy_ed25519` has wrong permissions: -```bash -chmod 600 ~/.ssh/github_deploy_ed25519 -``` - -## SSH Hosts - -Both servers use Cloudflare Tunnels (via `cloudflared access ssh`) as configured in `~/.ssh/config`. - -### Backend (GPU Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-backend` | -| Hostname | `pose-backend-ssh.yingliu.site` | -| User | `root` | -| Key | `~/.ssh/github_deploy_ed25519` | -| Production container | `pose-spatial-studio-backend` (port 49101) | -| Staging container | `pose-spatial-studio-backend-staging` (port 49102) | -| Code path | `/root/backend/` (inside container) | -| Logs | `/root/backend/logs/app.log` (inside container) | - -**Connect:** -```bash -ssh pose-backend -``` - -**Interactive shell inside the container:** -```bash -docker exec -it pose-spatial-studio-backend bash -``` -This drops you into the container at `/root/backend/` where you can run Python, inspect files, install packages, and debug directly. - -**Common commands (run on host, prefix container commands with `docker exec`):** - -```bash -# Check backend health -docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health - -# View recent logs -docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log - -# Inspect deployed code -docker exec pose-spatial-studio-backend cat /root/backend/app.py -docker exec pose-spatial-studio-backend cat /root/backend/config.py -docker exec pose-spatial-studio-backend ls -la /root/backend/processors/ - -# Check installed packages -docker exec pose-spatial-studio-backend pip list | grep -iE 'mediapipe|rtm|onnx|torch|opencv' - -# Check GPU status -docker exec pose-spatial-studio-backend python3 -c "import onnxruntime as ort; print(ort.get_available_providers())" - -# Restart the backend app -docker exec pose-spatial-studio-backend bash -c "pkill -f 'python.*app.py'" || true -sleep 2 -docker start pose-spatial-studio-backend 2>/dev/null || true -docker exec -d pose-spatial-studio-backend bash -c "cd /root/backend && python app.py >> logs/app.log 2>&1" -``` - -### Frontend (Nginx Server) - -| Field | Value | -|-------|-------| -| SSH alias | `pose-frontend` | -| Hostname | `pose-frontend-ssh.yingliu.site` | -| User | `sophia` | -| Key | `~/.ssh/github_deploy` (RSA) | -| Code path | `/var/www/frontend/` | -| Serves | `https://robot.yingliu.site` | - -**Connect:** -```bash -ssh pose-frontend -``` - -**Common commands:** - -```bash -# List deployed files -ls -la /var/www/frontend/ -ls -la /var/www/frontend/assets/ - -# View deployed index.html -cat /var/www/frontend/index.html - -# Check nginx status -sudo systemctl status nginx - -# Reload nginx after manual changes -sudo systemctl reload nginx -``` - -## Debugging Workflow - -When a problem exists on remote but not locally: - -1. **Check logs** on the backend for errors: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend tail -80 /root/backend/logs/app.log" - ``` - -2. **Compare deployed code** with local code: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend cat /root/backend/<file>" | diff - backend/<file> - ``` - -3. **Compare package versions** (common source of discrepancies): - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend pip list" > /tmp/remote-packages.txt - diff <(cat /tmp/remote-packages.txt) <(cd backend && .venv.nosync/bin/pip list) - ``` - -4. **Check frontend build output** — the remote serves a production Vite build, not dev mode: - ```bash - ssh pose-frontend "cat /var/www/frontend/index.html" - ``` - -## Quick Health Check (No SSH Required) - -```bash -# Production -curl -s https://pose-backend.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://robot.yingliu.site - -# Staging -curl -s https://pose-backend-staging.yingliu.site/health | python3 -m json.tool -curl -s -o /dev/null -w "%{http_code}" https://staging.robot.yingliu.site -``` diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md deleted file mode 100644 index 3c3a11f..0000000 --- a/.claude/skills/test/SKILL.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -name: test ---- - -Run the validation suite for **$ARGUMENTS** (or full validation if no arguments given). - -**CRITICAL: Steps 0, 1, 2, 3, and 4 are ALL MANDATORY and BLOCKING — do NOT skip ANY of them.** -**Steps 2 and 3 have USER-APPROVAL GATES — you MUST stop and get explicit user approval before continuing past them.** -**If you skip any step or gate, the entire validation is invalid and must be restarted from Step 0.** - -**IMPORTANT — run from the correct project:** All tests MUST run from the **working project directory** (the project with your code changes), NOT from wherever this SKILL.md lives. If you have multiple project copies (e.g., `pose-spatial-studio` and `pose-spatial-studio-1`), always `cd` into the one with your active feature branch before running any commands. - -**IMPORTANT — Do not make any changes in Staging and produc server from local project. It can only be done through GitHub CI/CD pipeline** ---- - -## Step 0: Pre-flight checks (MANDATORY) - -Before running any tests, **kill all existing servers** to prevent stale/wrong backends from being reused: - -```bash -pkill -f "app.py" 2>/dev/null; pkill -f "uvicorn" 2>/dev/null; pkill -f "run_server.sh" 2>/dev/null -pkill -f "vite" 2>/dev/null; pkill -f "run_ui.sh" 2>/dev/null -sleep 2 -# Verify ports are free: -lsof -i :49101 2>&1; lsof -i :8585 2>&1 -``` - -This is critical because Playwright's `reuseExistingServer: true` (the local default) will silently reuse any backend already running on port 49101 — even if it's from a different project copy. This causes false-positive test results. - ---- - -## Step 1: Automated E2E tests (MANDATORY) - -Run Playwright tests from the **working project's** `tests/` directory. Playwright auto-starts both backend (port 49101) and frontend (port 8585) via `webServer` config if they aren't already running. - -**Run all tests:** -```bash -cd tests && npm test -``` - -**Run a single spec on Chromium (minimum bar):** -```bash -cd tests && npx playwright test specs/pose-validation.spec.ts --project=chromium -``` - -**Other commands:** -```bash -cd tests && npm run test:headed # Visible browser -cd tests && npm run test:debug # Debug with breakpoints -cd tests && npm run test:ui # Interactive Playwright UI -cd tests && npm run test:report # View HTML report after a run -``` - -**Test specs** live in `tests/specs/`: -| Spec | Purpose | Config | -|------|---------|--------| -| `pose-validation.spec.ts` | Main E2E suite — UI controls, video upload, pose detection | default | -| `staging-video-test.spec.ts` | Staging backend smoke test | `playwright.staging.config.ts` | - -If tests fail, diagnose and fix before continuing. - -**Visual verification** — after tests pass, examine screenshots in `tests/results/` using the Read tool and verify: -- `pose-validation-avatar.png` — **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips -- `pose-validation-skeleton.png` — **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally -- Both screenshots should show the camera feed with a 2D skeleton overlay drawn on the image - ---- - -## Step 2: Manual testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Manual testing is complete. May I proceed to Step 3 (Staging environment testing)?" Do NOT continue to Step 3 without explicit user approval.** - -1. **Start dev environment** (if not already running): - - python venv environment is in `backend/.venv.nosync` - - Backend: `cd backend && ./run_server.sh` (port 49101) - - Frontend: `cd frontend && ./run_ui.sh` (port 8585) - - Confirm frontend connects to the **local** backend - -2. **Test with video file** (no camera needed): - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector (left sidebar) - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching: use the model selector dropdown in the 3D view to switch between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** functions - -3. **Visual checks** (take screenshots and examine with Read tool): - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips. avatar is not jittering. the movement is like a human. - - Toggle between Avatar and Skeleton using the inline button in the 3D view toolbar - - Switch model via the dropdown → re-check all of the above - -4. **Test with live camera** (when relevant): - - Select source type **Camera** → choose a camera device → click **Start** - - Verify pose landmarks overlay and 3D skeleton/avatar respond in real time - ---- - -## Step 3: Staging environment testing (MANDATORY) - -> **STOP GATE — After completing this step, you MUST use `AskUserQuestion` to ask the user: "Staging tests are complete. May I proceed to Step 4 (Remote GPU validation)?" Do NOT continue to Step 4 without explicit user approval.** - -Validate changes against the staging backend before production deployment. - -**Staging infrastructure:** -- Backend container: `pose-spatial-studio-backend-staging` on VM2 port 49102 -- Staging URL: `https://pose-backend-staging.yingliu.site` -- CI/CD: `deploy_backend_staging.yml` triggers on push to `staging` branch - -**Workflow:** - -1. Commit changes and create/merge PR to `staging` branch and approve merge -2. Wait for the **Deploy backend (staging)** GitHub Actions workflow to pass -3. Health check: - ```bash - curl -s https://pose-backend-staging.yingliu.site/health - ``` -4. Run frontend locally against staging backend and wait for user approval: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev - ``` -5. **Test with video file:** - - Open `http://localhost:8585` - - Select **3D Pose Estimation** from the function selector - - Choose source type **Video File** → upload `tests/test.mp4` - - Click **Start** → verify stream initializes and video plays - - Test model switching between **MediaPipe** and **YOLO+RTMPose** - - Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -6. run the automated staging test: - ```bash - cd frontend && VITE_BACKEND_URL=https://pose-backend-staging.yingliu.site npm run dev & - cd tests && npx playwright test specs/staging-video-test.spec.ts --config=playwright.staging.config.ts --project=chromium - ``` -7. **Visual verification** — take a screenshot and examine with the Read tool: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - - Toggle Avatar/Skeleton via inline button → switch model via dropdown → re-check -8. Check staging backend logs: - ```bash - ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -30 /root/backend/logs/app.log" - ``` -9. **STOP** — Use `AskUserQuestion` to ask the user whether staging tests passed and whether to proceed to Step 4 - ---- - -## Step 4: Remote GPU validation (MANDATORY) - -Verify the staging backend is healthy and GPU-accessible. This applies to **all changes** because the backend always runs on GPU infrastructure. - -```bash -# Staging backend -ssh pose-backend "docker exec pose-spatial-studio-backend-staging curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend-staging tail -50 /root/backend/logs/app.log" - -# Production backend -ssh pose-backend "docker exec pose-spatial-studio-backend curl -s http://localhost:49101/health" -ssh pose-backend "docker exec pose-spatial-studio-backend tail -50 /root/backend/logs/app.log" -``` - -**Visual verification** — run frontend against the production backend, take a screenshot, and examine with the Read tool: -1. `cd frontend && VITE_BACKEND_URL=https://pose-backend.yingliu.site npm run dev` -2. Open `http://localhost:8585` → Select **3D Pose Estimation** → Source type **Video File** → upload `tests/test.mp4` -3. Click **Start** → verify stream initializes → switch models between **MediaPipe** and **YOLO+RTMPose** -4. Also test **2D Pose Estimation**, **Object Detection**, and **Hand Gesture Recognition** -5. Verify via screenshot: - - 2D views: annotated frame shows skeleton overlay drawn on the video - - 3D view **Skeleton mode**: 3D skeleton is a reasonable human body shape and moves naturally - - 3D view **Avatar mode**: head is above hips, arms are in front of the body, legs are below hips - -See `/ssh-servers` skill for full remote debugging commands. - ---- - -## Validation checklist - -Before marking validation as complete, confirm ALL of the following: - -- [ ] **Step 0** — Pre-flight checks completed (servers killed, ports free) -- [ ] **Step 1** — Automated Playwright tests passed; screenshots verified -- [ ] **Step 2** — Manual testing completed; user approved to proceed -- [ ] **Step 3** — Staging environment tested (deployed, health check OK, pose detection works); user approved to proceed -- [ ] **Step 4** — Remote GPU validation passed (health check + logs reviewed) -- [ ] Implementation meets all acceptance criteria - -If any step fails → fix the issue and re-run from that step. - -**After ALL steps pass, return to develop/SKILL.md and continue with Step 5 (Update Documentation). Do NOT skip documentation updates.** \ No newline at end of file diff --git a/.github/workflows/deploy_backend.yml b/.github/workflows/deploy_backend.yml index 5f3b217..9d9b7d5 100644 --- a/.github/workflows/deploy_backend.yml +++ b/.github/workflows/deploy_backend.yml @@ -89,6 +89,9 @@ jobs: echo "=== Installing dependencies ===" docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" + echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime onnxruntime-gpu -y --break-system-packages 2>/dev/null; pip install --break-system-packages --force-reinstall onnxruntime-gpu -q" + echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR if ! docker exec $CONTAINER test -f "$TCPFORMER_DIR/$TCPFORMER_FILE"; then diff --git a/.github/workflows/deploy_backend_staging.yml b/.github/workflows/deploy_backend_staging.yml index fde7976..710125c 100644 --- a/.github/workflows/deploy_backend_staging.yml +++ b/.github/workflows/deploy_backend_staging.yml @@ -80,6 +80,9 @@ jobs: echo "=== Installing dependencies ===" docker exec $CONTAINER bash -c "cd /root/backend && pip install --break-system-packages -r requirements.txt -q" + echo "=== Fixing onnxruntime GPU (pip install may pull CPU variant) ===" + docker exec $CONTAINER bash -c "pip uninstall onnxruntime onnxruntime-gpu -y --break-system-packages 2>/dev/null; pip install --break-system-packages --force-reinstall onnxruntime-gpu -q" + echo "=== Pre-downloading TCPFormer checkpoint (if not cached) ===" docker exec $CONTAINER mkdir -p $TCPFORMER_DIR if ! docker exec $CONTAINER test -f "$TCPFORMER_DIR/$TCPFORMER_FILE"; then diff --git a/.github/workflows/deploy_frontend_staging.yml b/.github/workflows/deploy_frontend_staging.yml deleted file mode 100644 index 47468ac..0000000 --- a/.github/workflows/deploy_frontend_staging.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Deploy frontend (staging) - -on: - workflow_dispatch: - push: - branches: - - staging - paths: - - 'frontend/**' - - '.github/workflows/deploy_frontend_staging.yml' - -jobs: - deploy: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: frontend - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install deps - run: npm install - - - name: Build - run: npm run build - env: - VITE_BACKEND_URL: ${{ secrets.VITE_STAGING_BACKEND_URL }} - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Install cloudflared - run: | - curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - sudo mv cloudflared /usr/local/bin/cloudflared - sudo chmod +x /usr/local/bin/cloudflared - - - name: Configure SSH for Cloudflare - run: | - mkdir -p ~/.ssh - cat << 'EOF' >> ~/.ssh/config - Host vm-ssh - HostName ${{ secrets.SSH_HOST }} - User ${{ secrets.SSH_USER }} - ProxyCommand cloudflared access ssh --hostname %h --id ${{ secrets.CF_ACCESS_CLIENT_ID }} --secret ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - EOF - chmod 600 ~/.ssh/config - - - name: Test SSH via Cloudflare - run: ssh -o StrictHostKeyChecking=accept-new vm-ssh "echo 'SSH OK via Cloudflare'" - - - name: Deploy via rsync - run: | - ssh vm-ssh "mkdir -p /var/www/staging" - rsync -az --delete dist/ vm-ssh:/var/www/staging - - - name: Reload nginx - run: | - ssh vm-ssh "sudo systemctl reload nginx" diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bc20a..a1108f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to the Pose Spatial Studio project will be documented in thi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.4.2] - 4 April 2026 + +### Changed +- Avatar Voice Control hidden behind "Coming Soon" placeholder with 15-click easter egg to unlock +- Added `hidden` flag to FunctionDefinition type for hiding unreleased features + +### Infrastructure +- Added supervisord process supervisor to backend Docker container for automatic app restart on crash +- Backend container image committed with supervisord entrypoint (`/root/entrypoint.sh`) +- Container restart policy set to `unless-stopped` + ## [1.4.1] - 3 April 2026 ### Added diff --git a/README.md b/README.md index e37e8e3..b5907bf 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Real-time pose estimation, object detection, and 3D avatar rendering with WebSoc - **Object Detection** — EfficientDet-Lite2 bounding boxes and labels - **Hand Gesture Recognition** — Per-hand landmark tracking with gesture classification - **Live 3D Avatar** — Mixamo-rigged avatar driven by FK quaternions from pose estimation -- **Avatar Voice Control** — Voice/text commands to control 3D avatar via SecondBrain AI +- **Avatar Voice Control** — Voice/text commands to control 3D avatar via SecondBrain AI (coming soon) - **Camera & Video Input** — Live camera streams or video file upload - **Real-time Log Streaming** — Backend logs streamed live to the frontend - **Auto-Deployment** — GitHub Actions CI/CD to production and staging @@ -61,7 +61,7 @@ Open `http://localhost:8585` in your browser. - 3D view supports orbit controls (rotate, pan, zoom) and toggle between Avatar and Skeleton rendering - 3D Pose supports model switching between MediaPipe and YOLO+RTMPose 5. **View logs** in the right sidebar panel for real-time backend diagnostics -6. **Avatar Voice Control** — select from function menu, type or speak commands like "wave your right hand" +6. **Avatar Voice Control** — coming soon (hidden easter egg: click 15 times to unlock) ## Architecture diff --git a/backend/processors/mediapipe_hand_gesture_processor.py b/backend/processors/mediapipe_hand_gesture_processor.py index 55b02f9..23e4753 100644 --- a/backend/processors/mediapipe_hand_gesture_processor.py +++ b/backend/processors/mediapipe_hand_gesture_processor.py @@ -3,7 +3,7 @@ Uses MediaPipe GestureRecognizer for combined hand landmark detection and gesture classification. Draws hand bounding boxes, 21-point hand skeleton with connections, and gesture/handedness labels on the frame. -Supports LIVE_STREAM (camera) and VIDEO (file) running modes. +Uses synchronous VIDEO running mode to avoid LIVE_STREAM segfaults. """ import cv2 @@ -11,7 +11,6 @@ import numpy as np from typing import Dict, Any, Optional import logging -import threading import subprocess import os @@ -44,10 +43,10 @@ ]) # Colors (BGR) -_LEFT_HAND_COLOR = (255, 255, 0) # cyan -_RIGHT_HAND_COLOR = (255, 0, 255) # magenta -_JOINT_COLOR = (0, 255, 0) # green -_BONE_COLOR = (255, 128, 0) # light blue +_LEFT_HAND_COLOR = (180, 120, 0) # dark teal +_RIGHT_HAND_COLOR = (140, 0, 140) # dark magenta +_JOINT_COLOR = (0, 180, 0) # dark green +_BONE_COLOR = (160, 80, 0) # dark blue class MediaPipeHandGestureProcessor(BaseProcessor): @@ -67,18 +66,9 @@ def __init__(self, processor_id: str, 'min_tracking_confidence', 0.5) self.num_hands = int(pose_cfg.get('num_hands', 2)) self.device = pose_cfg.get('device', 'cpu') - self.source_type = pose_cfg.get('source_type', 'camera') self.gesture_recognizer = None self.last_timestamp = 0 - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_gesture_result = None - - def _gesture_result_callback(self, result, output_image, timestamp_ms): - with self.result_lock: - self.latest_gesture_result = result - def _get_delegate(self): if self.device == 'cuda': logger.info("HandGesture: using GPU delegate") @@ -96,9 +86,7 @@ def _try_initialize_with_delegate(self, delegate) -> bool: ["wget", "-O", self.model_path, _GESTURE_MODEL_URL], check=True) - use_live = self.source_type == 'camera' - running_mode = (mp.tasks.vision.RunningMode.LIVE_STREAM if use_live - else mp.tasks.vision.RunningMode.VIDEO) + running_mode = mp.tasks.vision.RunningMode.VIDEO logger.info(f"HandGesture running mode: {running_mode.name}") gr_kwargs = dict( @@ -111,8 +99,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: min_hand_presence_confidence=self.min_detection_confidence, min_tracking_confidence=self.min_tracking_confidence, ) - if use_live: - gr_kwargs['result_callback'] = self._gesture_result_callback self.gesture_recognizer = ( mp.tasks.vision.GestureRecognizer.create_from_options( mp.tasks.vision.GestureRecognizerOptions(**gr_kwargs))) @@ -145,9 +131,6 @@ def cleanup(self): if self.gesture_recognizer: self.gesture_recognizer.close() self.gesture_recognizer = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_gesture_result = None self._is_initialized = False logger.info( f"HandGesture processor {self.processor_id} cleaned up") @@ -180,15 +163,9 @@ def process_frame(self, frame: np.ndarray, image_format=mp.ImageFormat.SRGB, data=annotated.copy()) - # Run gesture recognition - if self.source_type == 'camera': - self.gesture_recognizer.recognize_async(mp_image, timestamp_ms) - with self.result_lock: - gesture_result = self.latest_gesture_result - else: - gesture_result = ( - self.gesture_recognizer.recognize_for_video( - mp_image, timestamp_ms)) + # Run gesture recognition (synchronous VIDEO mode) + gesture_result = self.gesture_recognizer.recognize_for_video( + mp_image, timestamp_ms) hands = [] if (gesture_result and gesture_result.hand_landmarks diff --git a/backend/processors/mediapipe_object_detector_processor.py b/backend/processors/mediapipe_object_detector_processor.py index 5a87cbe..44bccc5 100644 --- a/backend/processors/mediapipe_object_detector_processor.py +++ b/backend/processors/mediapipe_object_detector_processor.py @@ -2,7 +2,7 @@ Detects all object categories using MediaPipe ObjectDetector (EfficientDet). Draws colored bounding boxes and category labels on the video frame. -Supports both LIVE_STREAM (camera) and VIDEO (file) running modes. +Uses synchronous VIDEO running mode to avoid LIVE_STREAM segfaults. """ import cv2 @@ -10,7 +10,6 @@ import numpy as np from typing import Dict, Any, Optional import logging -import threading import subprocess import os @@ -20,18 +19,18 @@ logger = logging.getLogger(__name__) -# Color palette for different object categories (BGR) +# Color palette for different object categories (BGR) — dark tones for readability _CATEGORY_COLORS = [ - (0, 255, 0), # green - (255, 0, 0), # blue - (0, 0, 255), # red - (255, 255, 0), # cyan - (0, 255, 255), # yellow - (255, 0, 255), # magenta - (128, 255, 0), # spring green - (255, 128, 0), # sky blue - (0, 128, 255), # orange - (128, 0, 255), # violet + (0, 160, 0), # dark green + (200, 0, 0), # dark blue + (0, 0, 200), # dark red + (160, 120, 0), # dark teal + (0, 120, 180), # dark orange + (160, 0, 160), # dark magenta + (60, 140, 0), # dark spring green + (180, 80, 0), # dark sky blue + (0, 80, 180), # dark amber + (120, 0, 180), # dark violet ] @@ -54,18 +53,9 @@ def __init__(self, processor_id: str, self.min_detection_confidence = pose_cfg.get('min_detection_confidence', 0.5) self.max_results = int(pose_cfg.get('max_results', 10)) self.device = pose_cfg.get('device', 'cpu') - self.source_type = pose_cfg.get('source_type', 'camera') self.object_detector = None self.last_timestamp = 0 - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_object_result = None - - def _detection_result_callback(self, result, output_image, timestamp_ms): - with self.result_lock: - self.latest_object_result = result - def _get_delegate(self): if self.device == 'cuda': logger.info("ObjectDetector: using GPU delegate") @@ -82,9 +72,7 @@ def _try_initialize_with_delegate(self, delegate) -> bool: subprocess.run( ["wget", "-O", self.model_path, model_url], check=True) - use_live = self.source_type == 'camera' - running_mode = (mp.tasks.vision.RunningMode.LIVE_STREAM if use_live - else mp.tasks.vision.RunningMode.VIDEO) + running_mode = mp.tasks.vision.RunningMode.VIDEO logger.info(f"ObjectDetector running mode: {running_mode.name}") od_kwargs = dict( @@ -94,8 +82,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: running_mode=running_mode, max_results=self.max_results, score_threshold=self.min_detection_confidence) - if use_live: - od_kwargs['result_callback'] = self._detection_result_callback self.object_detector = ( mp.tasks.vision.ObjectDetector.create_from_options( mp.tasks.vision.ObjectDetectorOptions(**od_kwargs))) @@ -128,9 +114,6 @@ def cleanup(self): if self.object_detector: self.object_detector.close() self.object_detector = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_object_result = None self._is_initialized = False logger.info( f"ObjectDetector processor {self.processor_id} cleaned up") @@ -163,14 +146,9 @@ def process_frame(self, frame: np.ndarray, data=cv2.resize(frame_rgb, (self.frame_width, self.frame_height))) - # Run detection - if self.source_type == 'camera': - self.object_detector.detect_async(mp_image, timestamp_ms) - with self.result_lock: - object_result = self.latest_object_result - else: - object_result = self.object_detector.detect_for_video( - mp_image, timestamp_ms) + # Run detection (synchronous VIDEO mode) + object_result = self.object_detector.detect_for_video( + mp_image, timestamp_ms) objects = [] if object_result and object_result.detections: diff --git a/backend/processors/mediapipe_processor.py b/backend/processors/mediapipe_processor.py index a8966fb..b3860f3 100644 --- a/backend/processors/mediapipe_processor.py +++ b/backend/processors/mediapipe_processor.py @@ -3,11 +3,8 @@ import numpy as np from typing import Dict, Any, Optional, List import logging -import threading from processors.base_processor import BaseProcessor import config -import subprocess -import os from utils.kinetic import Converter @@ -76,33 +73,15 @@ def __init__(self, processor_id: str, config_dict: Optional[Dict[str, Any]] = No self.config = config.merge_configs(config_dict) pose_processor_config = self.config['pose_processor'] self.pose_landmarker_model_path = pose_processor_config['pose_landmarker_model_name'] - self.object_detector_model_path = pose_processor_config['object_detector_model_name'] self.min_detection_confidence = pose_processor_config['min_detection_confidence'] self.min_tracking_confidence = pose_processor_config['min_tracking_confidence'] self.min_presence_confidence = pose_processor_config['min_presence_confidence'] self.pose_landmarker_frame_width = pose_processor_config['pose_landmarker_frame_width'] self.pose_landmarker_frame_height = pose_processor_config['pose_landmarker_frame_height'] self.num_poses = pose_processor_config['num_poses'] - self.object_detector_frame_width = pose_processor_config['object_detector_frame_width'] - self.object_detector_frame_height = pose_processor_config['object_detector_frame_height'] self.device = pose_processor_config.get('device', 'cpu') - self.source_type = pose_processor_config.get('source_type', 'camera') self.landmarker = None - self.object_detector = None self.last_timestamp = 0 - # LIVE_STREAM mode (camera) needs callbacks and a lock for async results - if self.source_type == 'camera': - self.result_lock = threading.Lock() - self.latest_pose_result = None - self.latest_object_result = None - - def _pose_result_callback(self, result: mp.tasks.vision.PoseLandmarkerResult, output_image: mp.Image, timestamp_ms: int): - with self.result_lock: - self.latest_pose_result = result - - def _detection_result_callback(self, result: mp.tasks.vision.ObjectDetectorResult, output_image: mp.Image, timestamp_ms: int): - with self.result_lock: - self.latest_object_result = result def _get_delegate(self) -> 'mp.tasks.BaseOptions.Delegate': """Return GPU delegate if device is cuda, otherwise CPU.""" @@ -113,27 +92,9 @@ def _get_delegate(self) -> 'mp.tasks.BaseOptions.Delegate': return mp.tasks.BaseOptions.Delegate.CPU def _try_initialize_with_delegate(self, delegate) -> bool: - """Attempt to create MediaPipe tasks with the given delegate.""" - if not os.path.exists(self.object_detector_model_path): - model_url = MODEL_LINK.get(os.path.basename(self.object_detector_model_path)) - if model_url is None: - raise ValueError(f"No download URL found for model {self.object_detector_model_path}") - subprocess.run(["wget", "-O", self.object_detector_model_path, model_url], check=True) - - use_live = self.source_type == 'camera' - running_mode = mp.tasks.vision.RunningMode.LIVE_STREAM if use_live else mp.tasks.vision.RunningMode.VIDEO - logger.info(f"MediaPipe running mode: {running_mode.name} (source_type={self.source_type})") - - od_kwargs = dict( - base_options=mp.tasks.BaseOptions( - model_asset_path=self.object_detector_model_path, - delegate=delegate), - running_mode=running_mode, - max_results=5) - if use_live: - od_kwargs['result_callback'] = self._detection_result_callback - self.object_detector = mp.tasks.vision.ObjectDetector.create_from_options( - mp.tasks.vision.ObjectDetectorOptions(**od_kwargs)) + """Attempt to create MediaPipe PoseLandmarker with the given delegate.""" + running_mode = mp.tasks.vision.RunningMode.VIDEO + logger.info(f"MediaPipe running mode: {running_mode.name}") lm_kwargs = dict( base_options=mp.tasks.BaseOptions( @@ -145,8 +106,6 @@ def _try_initialize_with_delegate(self, delegate) -> bool: min_pose_presence_confidence=self.min_presence_confidence, min_tracking_confidence=self.min_tracking_confidence, output_segmentation_masks=False) - if use_live: - lm_kwargs['result_callback'] = self._pose_result_callback self.landmarker = mp.tasks.vision.PoseLandmarker.create_from_options( mp.tasks.vision.PoseLandmarkerOptions(**lm_kwargs)) return True @@ -196,37 +155,10 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: annotated_frame = cv2.resize(frame, (self.pose_landmarker_frame_width, self.pose_landmarker_frame_height)).copy() mp_pose_image = self._mp_image_from_frame(frame, self.pose_landmarker_frame_width, self.pose_landmarker_frame_height) - mp_object_image = self._mp_image_from_frame(frame, self.object_detector_frame_width, self.object_detector_frame_height) - if self.source_type == 'camera': - # LIVE_STREAM: async detection, read previous result - self.landmarker.detect_async(mp_pose_image, timestamp_ms) - self.object_detector.detect_async(mp_object_image, timestamp_ms) - with self.result_lock: - pose_result = self.latest_pose_result - object_result = self.latest_object_result - else: - # VIDEO: synchronous detection, returns current frame result - pose_result = self.landmarker.detect_for_video(mp_pose_image, timestamp_ms) - object_result = self.object_detector.detect_for_video(mp_object_image, timestamp_ms) + pose_result = self.landmarker.detect_for_video(mp_pose_image, timestamp_ms) landmarks, world_landmarks = [], [] - - if object_result and object_result.detections: - for obj in object_result.detections: - if not obj.categories or obj.categories[0].category_name != "person": - continue - bounding_box = obj.bounding_box - - scale_x = self.pose_landmarker_frame_width / self.object_detector_frame_width - scale_y = self.pose_landmarker_frame_height / self.object_detector_frame_height - - x1 = int(bounding_box.origin_x * scale_x) - y1 = int(bounding_box.origin_y * scale_y) - x2 = int((bounding_box.origin_x + bounding_box.width) * scale_x) - y2 = int((bounding_box.origin_y + bounding_box.height) * scale_y) - - cv2.rectangle(annotated_frame, (x1, y1), (x2, y2), (0, 255, 0), 2) if pose_result and pose_result.pose_landmarks: h, w, _ = annotated_frame.shape @@ -318,13 +250,6 @@ def cleanup(self): if self.landmarker: self.landmarker.close() self.landmarker = None - if self.object_detector: - self.object_detector.close() - self.object_detector = None - if self.source_type == 'camera': - with self.result_lock: - self.latest_pose_result = None - self.latest_object_result = None self._is_initialized = False logger.info(f"MediaPipe processor {self.processor_id} cleaned up") diff --git a/backend/processors/rtmpose_processor.py b/backend/processors/rtmpose_processor.py index 3e04ddb..d351756 100644 --- a/backend/processors/rtmpose_processor.py +++ b/backend/processors/rtmpose_processor.py @@ -1,5 +1,5 @@ import cv2 -from rtmlib import Wholebody3d, draw_skeleton +from rtmlib import PoseTracker, Wholebody3d, draw_skeleton from typing import Optional, List, Dict, Any from processors.base_processor import BaseProcessor from utils.kinetic import Converter @@ -58,13 +58,17 @@ def __init__(self, processor_id: str, config_dict: Optional[Dict[str, Any]] = No def initialize(self) -> bool: self._is_initialized = True - self.wholebody = Wholebody3d( + self.pose_tracker = PoseTracker( + Wholebody3d, + det_frequency=7, + tracking=False, mode='balanced', to_openpose=False, backend=self.backend, device=self.device ) self._z_root_filter = MedianFilter(window_size=5) + self._root_positions = [] logger.info(f"RTMpose 3D processor {self.processor_id} initialized") return True @@ -76,14 +80,13 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: return None frame = np.ascontiguousarray(frame) - keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.wholebody(frame) + keypoints_3d, scores, keypoints_simcc, keypoints_2d = self.pose_tracker(frame) - # Fix rtmlib z-depth bug: rtmlib decodes z using image height (384/2=192) - # but the model codec uses z_input_size (288/2=144). Re-decode from raw - # simcc pixel values to get correct root-relative depth in meters. + # Fix z-depth: rtmlib decodes z using image height (384/2=192) instead of + # the codec z input size (288/2=144). Re-decode z from raw simcc values. + # x,y are kept from keypoints_2d (stable image-space pixels) in _build_world_landmarks. if keypoints_3d is not None and len(keypoints_3d) > 0: - z_pixel = keypoints_simcc[..., 2] - keypoints_3d[..., 2] = (z_pixel / _Z_INPUT_HALF - 1.0) * _Z_RANGE + keypoints_3d[..., 2] = (keypoints_simcc[..., 2] / _Z_INPUT_HALF - 1.0) * _Z_RANGE annotated_frame = draw_skeleton(frame, keypoints_2d, scores, kpt_thr=0.5) annotated_frame = cv2.resize(annotated_frame, (640, 480)) @@ -110,14 +113,13 @@ def process_frame(self, frame: np.ndarray, timestamp_ms: int) -> Dict[str, Any]: fk_data = self._fk_processing(world_landmarks) root_position = None - if world_landmarks: - hip_data = world_landmarks[0].get("hipCentre", {}) - if hip_data: - root_position = { - "x": float(hip_data.get("x", 0)), - "y": float(-hip_data.get("y", 0)), - "z": float(-hip_data.get("z", 0)) - } + if self._root_positions: + rp = self._root_positions[0] + root_position = { + "x": float(rp["x"]), + "y": float(-rp["y"]), + "z": float(-rp["z"]) + } return { "processed_frame": annotated_frame, @@ -160,21 +162,23 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, keypoints_2d: np.ndarray, scores: np.ndarray, w: int, h: int) -> List[Dict]: - """Build 3D world landmarks combining 2D image-space x,y with 3D z. - - Uses keypoints_2d (inverse-affine-transformed to image space by rtmlib) - for x,y with per-joint depth-corrected perspective unprojection. - Each joint's absolute depth (z_root + z_relative) is used instead of - a constant z_root, producing consistent 3D positions for accurate - FK bone direction computation. z comes from keypoints_3d (already - corrected for rtmlib's codec mismatch). + """Build 3D world landmarks using 2D keypoints for x,y and simcc z for depth. + + x,y: from keypoints_2d (stable image-space pixels) with perspective + unprojection using a shared z_root for all joints — skeleton + proportions come from stable 2D pixel ratios. + z: from keypoints_3d (corrected simcc z) — root-relative depth. + Root position computed separately for scene placement. """ - f_est = float(max(w, h)) # estimated focal length (~53° VFOV) + f_est = float(max(w, h)) + self._root_positions = [] result = [] for person_3d, person_2d, person_scores in zip( keypoints_3d, keypoints_2d, scores): - # Estimate root depth from visible body extent (indices 5-16) + hip_indices = [11, 12] + + # Estimate z_root from visible body extent (smoothed) visible_ys = [person_2d[i][1] for i in range(5, 17) if i < len(person_2d) and person_scores[i] > 0.3] if len(visible_ys) >= 2: @@ -182,16 +186,24 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, z_root = _TORSO_LEG_HEIGHT * f_est / max(body_height_px, 50.0) else: z_root = 3.0 - - # Smooth z_root across frames to reduce scale jitter z_root = float(self._z_root_filter.filter(np.array([z_root]))[0]) - # Person center in image space (average of hip keypoints) - hip_indices = [11, 12] + # Hip center in image space valid_hips = [i for i in hip_indices if i < len(person_2d)] cx = float(np.mean([person_2d[i][0] for i in valid_hips])) cy = float(np.mean([person_2d[i][1] for i in valid_hips])) + # Root position for scene placement + self._root_positions.append({ + "x": (cx - w / 2) * z_root / f_est, + "y": (cy - h / 2) * z_root / f_est, + "z": z_root + }) + + # Hip center z from corrected simcc (for root-relative depth) + hip_3d_z = float(np.mean([person_3d[i][2] for i in hip_indices])) + + # Build landmarks: x,y from 2D pixels (shared z_root), z from simcc landmark_dict = {} for joint_name, indices in COCO133_TO_OUTPUT_JOINTS.items(): valid = [i for i in indices if i < len(person_2d)] @@ -202,23 +214,16 @@ def _build_world_landmarks(self, keypoints_3d: np.ndarray, } continue - # Per-joint depth-corrected perspective unprojection: - # use each joint's absolute depth instead of constant z_root - xs, ys, zs = [], [], [] - for i in valid: - z_rel = person_3d[i][2] - z_abs = max(z_root + z_rel, 0.5) - xs.append((person_2d[i][0] - cx) * z_abs / f_est) - ys.append((person_2d[i][1] - cy) * z_abs / f_est) - zs.append(z_rel) - + # x,y: perspective unprojection with shared z_root (stable proportions) + # z: root-relative depth from corrected simcc landmark_dict[joint_name] = { - "x": float(np.mean(xs)), - "y": float(np.mean(ys)), - "z": float(np.mean(zs)), + "x": float(np.mean([(person_2d[i][0] - cx) * z_root / f_est for i in valid])), + "y": float(np.mean([(person_2d[i][1] - cy) * z_root / f_est for i in valid])), + "z": float(np.mean([person_3d[i][2] - hip_3d_z for i in valid])), "visibility": float(np.mean([person_scores[i] for i in valid])), "presence": float(np.mean([person_scores[i] for i in valid])), } + result.append(landmark_dict) return result diff --git a/frontend/src/components/CameraCapture.tsx b/frontend/src/components/CameraCapture.tsx index b1ded3a..8c0717d 100644 --- a/frontend/src/components/CameraCapture.tsx +++ b/frontend/src/components/CameraCapture.tsx @@ -182,7 +182,12 @@ export function CameraCapture({ canvasRef.current.height = videoRef.current.videoHeight; } + if (sourceType === 'camera') { + ctx.translate(canvasRef.current.width, 0); + ctx.scale(-1, 1); + } ctx.drawImage(videoRef.current, 0, 0); + ctx.setTransform(1, 0, 0, 1, 0, 0); isProcessingFrameRef.current = true; canvasRef.current.toBlob( (blob) => { diff --git a/frontend/src/components/Skeleton3DViewer.tsx b/frontend/src/components/Skeleton3DViewer.tsx index c057a74..8b5fac4 100644 --- a/frontend/src/components/Skeleton3DViewer.tsx +++ b/frontend/src/components/Skeleton3DViewer.tsx @@ -68,8 +68,8 @@ export function Skeleton3DViewer({ poseResult, videoElement, processedCanvas, re <div style={{ width: '100%', height: '100%', backgroundColor: '#000' }}> <ViewerErrorBoundary> <Canvas> - <PerspectiveCamera makeDefault position={[0, 0, 3]} /> - <OrbitControls enableDamping dampingFactor={0.05} /> + <PerspectiveCamera makeDefault position={[0, 1.2, 5]} /> + <OrbitControls enableDamping dampingFactor={0.05} target={[0, 0.6, 0]} /> <ambientLight intensity={0.5} /> <directionalLight position={[10, 10, 5]} intensity={1} /> diff --git a/frontend/src/components/View2D.tsx b/frontend/src/components/View2D.tsx index 839ad4c..e01abb4 100644 --- a/frontend/src/components/View2D.tsx +++ b/frontend/src/components/View2D.tsx @@ -38,7 +38,7 @@ export function View2D({ socket }: View2DProps) { }, [isStreamActive]); return ( - <div className="view-container" style={{ position: 'relative', width: '100%', height: '100%', minHeight: 400 }}> + <div className="view-container" style={{ position: 'relative', width: '100%', aspectRatio: '4 / 3', margin: '0 auto' }}> {isStreamActive && <CameraCapture socket={socket} />} {!isStreamActive && !backendResult ? ( @@ -61,7 +61,6 @@ export function View2D({ socket }: View2DProps) { style={{ width: '100%', height: '100%', - objectFit: 'contain', display: 'block', backgroundColor: '#000', borderRadius: 16,