From 06fc1df6c08590259f1fb02bcfed3ebddfd3c01b Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:58:47 +0200 Subject: [PATCH 1/4] modify camera UI --- src/components/control/VisualizerPanel.tsx | 138 +++++- src/components/landing/RecordingModal.tsx | 87 +++- src/components/landing/TeleoperationModal.tsx | 181 ++++++-- src/components/ui/CameraDetectionButton.tsx | 29 ++ src/components/ui/CameraDetectionModal.tsx | 439 ++++++++++++++++++ 5 files changed, 823 insertions(+), 51 deletions(-) create mode 100644 src/components/ui/CameraDetectionButton.tsx create mode 100644 src/components/ui/CameraDetectionModal.tsx diff --git a/src/components/control/VisualizerPanel.tsx b/src/components/control/VisualizerPanel.tsx index 75c411b..7edaa64 100644 --- a/src/components/control/VisualizerPanel.tsx +++ b/src/components/control/VisualizerPanel.tsx @@ -1,11 +1,12 @@ -import React from "react"; +import React, { useState, useEffect } from "react"; import { Button } from "@/components/ui/button"; -import { ArrowLeft, VideoOff } from "lucide-react"; +import { ArrowLeft, VideoOff, Camera, Wifi, WifiOff } from "lucide-react"; import { cn } from "@/lib/utils"; import UrdfViewer from "../UrdfViewer"; import UrdfProcessorInitializer from "../UrdfProcessorInitializer"; import Logo from "@/components/Logo"; +import { useApi } from "@/contexts/ApiContext"; interface VisualizerPanelProps { onGoBack: () => void; @@ -16,6 +17,64 @@ const VisualizerPanel: React.FC = ({ onGoBack, className, }) => { + const { baseUrl, fetchWithHeaders } = useApi(); + const [cameraConfig, setCameraConfig] = useState<{[key: string]: any}>({}); + const [isLoadingCameras, setIsLoadingCameras] = useState(true); + const [cameraStreams, setCameraStreams] = useState<{[key: string]: string}>({}); + const [streamErrors, setStreamErrors] = useState<{[key: string]: boolean}>({}); + + // Load camera configuration and streaming URLs + useEffect(() => { + const loadCameraConfig = async () => { + try { + // Load camera configuration + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + if (data.status === "success" && data.camera_config) { + const cameras = data.camera_config.cameras || {}; + setCameraConfig(cameras); + + // Generate streaming URLs for each configured camera + const streams: {[key: string]: string} = {}; + Object.keys(cameras).forEach(cameraName => { + streams[cameraName] = `${baseUrl}/cameras/stream/${encodeURIComponent(cameraName)}`; + }); + setCameraStreams(streams); + } + } catch (error) { + console.error("Error loading camera config:", error); + } finally { + setIsLoadingCameras(false); + } + }; + + loadCameraConfig(); + }, [baseUrl, fetchWithHeaders]); + + // Handle camera stream errors + const handleStreamError = (cameraName: string) => { + setStreamErrors(prev => ({ ...prev, [cameraName]: true })); + }; + + // Handle camera stream load success + const handleStreamLoad = (cameraName: string) => { + setStreamErrors(prev => ({ ...prev, [cameraName]: false })); + }; + + // Get camera entries - only show configured cameras + const getCameraSlots = () => { + const configuredCameras = Object.entries(cameraConfig); + const cameraSlots = []; + + // Add only configured cameras (no empty slots) + configuredCameras.forEach(([name, config]) => { + cameraSlots.push({ name, config, isConfigured: true }); + }); + + return cameraSlots; + }; + return (
= ({
-
- {[1, 2, 3, 4].map((cam) => ( -
+
+ {getCameraSlots().length > 0 ? ( + getCameraSlots().map((cameraSlot, index) => ( +
+ {isLoadingCameras ? ( + <> + + + Loading cameras... + + + ) : ( +
+
+ {cameraStreams[cameraSlot.name!] && !streamErrors[cameraSlot.name!] ? ( + {`${cameraSlot.name} handleStreamError(cameraSlot.name!)} + onLoad={() => handleStreamLoad(cameraSlot.name!)} + /> + ) : streamErrors[cameraSlot.name!] ? ( +
+ + Stream Error +
+ ) : ( +
+ + Connecting... +
+ )} + + {/* Stream status indicator */} +
+ {cameraStreams[cameraSlot.name!] && !streamErrors[cameraSlot.name!] ? ( +
+ ) : streamErrors[cameraSlot.name!] ? ( +
+ ) : ( +
+ )} +
+
+
+ + {cameraSlot.name} + + + {cameraSlot.config.width}x{cameraSlot.config.height} @ {cameraSlot.config.fps}fps + +
+
+ )} +
+ )) + ) : ( +
- No Camera Available + No Cameras Configured + + + Configure cameras in teleoperation settings
- ))} + )}
); diff --git a/src/components/landing/RecordingModal.tsx b/src/components/landing/RecordingModal.tsx index 87da0a3..703225f 100644 --- a/src/components/landing/RecordingModal.tsx +++ b/src/components/landing/RecordingModal.tsx @@ -19,6 +19,8 @@ import { import { QrCode } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; +import CameraDetectionModal from "@/components/ui/CameraDetectionModal"; +import CameraDetectionButton from "@/components/ui/CameraDetectionButton"; import QrCodeModal from "@/components/recording/QrCodeModal"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; @@ -74,6 +76,8 @@ const RecordingModal: React.FC = ({ >("leader"); const [showQrCodeModal, setShowQrCodeModal] = useState(false); const [sessionId, setSessionId] = useState(""); + const [showCameraDetection, setShowCameraDetection] = useState(false); + const [cameraConfig, setCameraConfig] = useState({}); const handlePortDetection = (robotType: "leader" | "follower") => { setDetectionRobotType(robotType); @@ -151,6 +155,9 @@ const RecordingModal: React.FC = ({ if (followerConfigData.status === "success" && followerConfigData.default_config) { setFollowerConfig(followerConfigData.default_config); } + + // Load camera configuration + await loadCameraConfig(); } catch (error) { console.error("Error loading saved data:", error); } @@ -169,6 +176,26 @@ const RecordingModal: React.FC = ({ setSessionId(newSessionId); setShowQrCodeModal(true); }; + + const handleCameraSelected = (cameraData: any) => { + setCameraConfig(prev => ({ + ...prev, + [cameraData.name]: cameraData.config + })); + }; + + const loadCameraConfig = async () => { + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + if (data.status === "success" && data.camera_config) { + setCameraConfig(data.camera_config.cameras || {}); + } + } catch (error) { + console.error("Error loading camera config:", error); + } + }; return ( <> @@ -323,6 +350,58 @@ const RecordingModal: React.FC = ({
+ {/* Camera Configuration Section */} +
+
+

+ Camera Configuration +

+ setShowCameraDetection(true)} + /> +
+ + {Object.keys(cameraConfig).length > 0 ? ( +
+
+ Configured Cameras ({Object.keys(cameraConfig).length}) +
+ {Object.entries(cameraConfig).map(([name, config]: [string, any]) => ( +
+
+
{name}
+
+ {config.type} - {config.width}x{config.height} @ {config.fps}fps +
+
+ +
+ ))} +
+ ) : ( +
+ No cameras configured. Click the camera icon to detect and configure cameras. +
+ )} +
+

Dataset Configuration @@ -411,7 +490,13 @@ const RecordingModal: React.FC = ({ onOpenChange={setShowQrCodeModal} sessionId={sessionId} /> + + ); }; -export default RecordingModal; +export default RecordingModal; \ No newline at end of file diff --git a/src/components/landing/TeleoperationModal.tsx b/src/components/landing/TeleoperationModal.tsx index 75bc1f0..a60b935 100644 --- a/src/components/landing/TeleoperationModal.tsx +++ b/src/components/landing/TeleoperationModal.tsx @@ -19,6 +19,8 @@ import { import { Settings } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; +import CameraDetectionModal from "@/components/ui/CameraDetectionModal"; +import CameraDetectionButton from "@/components/ui/CameraDetectionButton"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; @@ -61,6 +63,90 @@ const TeleoperationModal: React.FC = ({ const [detectionRobotType, setDetectionRobotType] = useState< "leader" | "follower" >("leader"); + const [showCameraDetection, setShowCameraDetection] = useState(false); + const [cameraConfig, setCameraConfig] = useState({}); + + const handlePortDetection = (robotType: "leader" | "follower") => { + setDetectionRobotType(robotType); + setShowPortDetection(true); + }; + + const handlePortDetected = (port: string) => { + if (detectionRobotType === "leader") { + setLeaderPort(port); + } else { + setFollowerPort(port); + } + }; + + // Enhanced port change handlers that save automatically + const handleLeaderPortChange = (value: string) => { + setLeaderPort(value); + // Auto-save with debouncing to avoid excessive API calls + debouncedSavePort("leader", value); + }; + + const handleFollowerPortChange = (value: string) => { + setFollowerPort(value); + // Auto-save with debouncing to avoid excessive API calls + debouncedSavePort("follower", value); + }; + + // Enhanced config change handlers that save automatically + const handleLeaderConfigChange = (value: string) => { + setLeaderConfig(value); + // Auto-save with debouncing to avoid excessive API calls + debouncedSaveConfig("leader", value); + }; + + const handleFollowerConfigChange = (value: string) => { + setFollowerConfig(value); + // Auto-save with debouncing to avoid excessive API calls + debouncedSaveConfig("follower", value); + }; + + const handleCameraSelected = (cameraData: any) => { + setCameraConfig(prev => ({ + ...prev, + [cameraData.name]: cameraData.config + })); + }; + + const loadCameraConfig = async () => { + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + if (data.status === "success" && data.camera_config) { + setCameraConfig(data.camera_config.cameras || {}); + } + } catch (error) { + console.error("Error loading camera config:", error); + } + }; + + const handleRemoveCamera = async (cameraName: string) => { + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/config/${encodeURIComponent(cameraName)}`, { + method: "DELETE", + }); + + const result = await response.json(); + + if (result.status === "success") { + setCameraConfig(prev => { + const newConfig = { ...prev }; + delete newConfig[cameraName]; + return newConfig; + }); + console.log(`Camera "${cameraName}" removed successfully`); + } else { + console.error("Error removing camera:", result.message); + } + } catch (error) { + console.error("Error removing camera:", error); + } + }; // Load saved ports and configurations on component mount useEffect(() => { @@ -101,6 +187,9 @@ const TeleoperationModal: React.FC = ({ if (followerConfigData.status === "success" && followerConfigData.default_config) { setFollowerConfig(followerConfigData.default_config); } + + // Load camera configuration + await loadCameraConfig(); } catch (error) { console.error("Error loading saved data:", error); } @@ -111,47 +200,9 @@ const TeleoperationModal: React.FC = ({ } }, [open, setLeaderPort, setFollowerPort, setLeaderConfig, setFollowerConfig, leaderConfigs, followerConfigs, baseUrl, fetchWithHeaders]); - const handlePortDetection = (robotType: "leader" | "follower") => { - setDetectionRobotType(robotType); - setShowPortDetection(true); - }; - - const handlePortDetected = (port: string) => { - if (detectionRobotType === "leader") { - setLeaderPort(port); - } else { - setFollowerPort(port); - } - }; - - // Enhanced port change handlers that save automatically - const handleLeaderPortChange = (value: string) => { - setLeaderPort(value); - // Auto-save with debouncing to avoid excessive API calls - debouncedSavePort("leader", value); - }; - - const handleFollowerPortChange = (value: string) => { - setFollowerPort(value); - // Auto-save with debouncing to avoid excessive API calls - debouncedSavePort("follower", value); - }; - - // Enhanced config change handlers that save automatically - const handleLeaderConfigChange = (value: string) => { - setLeaderConfig(value); - // Auto-save with debouncing to avoid excessive API calls - debouncedSaveConfig("leader", value); - }; - - const handleFollowerConfigChange = (value: string) => { - setFollowerConfig(value); - // Auto-save with debouncing to avoid excessive API calls - debouncedSaveConfig("follower", value); - }; return ( - +
@@ -274,6 +325,50 @@ const TeleoperationModal: React.FC = ({

+
+
+

+ Camera Configuration +

+ setShowCameraDetection(true)} + /> +
+ + {Object.keys(cameraConfig).length > 0 ? ( +
+
+ Configured Cameras ({Object.keys(cameraConfig).length}) +
+ {Object.entries(cameraConfig).map(([name, config]: [string, any]) => ( +
+
+
{name}
+
+ {config.type} - {config.width}x{config.height} @ {config.fps}fps +
+
+ +
+ ))} +
+ ) : ( +
+ No cameras configured. Click the camera icon to detect and configure cameras. +
+ )} +
+
+ ); +}; + +export default CameraDetectionButton; \ No newline at end of file diff --git a/src/components/ui/CameraDetectionModal.tsx b/src/components/ui/CameraDetectionModal.tsx new file mode 100644 index 0000000..dc47e57 --- /dev/null +++ b/src/components/ui/CameraDetectionModal.tsx @@ -0,0 +1,439 @@ +import React, { useState, useEffect } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { Loader2, Camera, CheckCircle, XCircle, AlertCircle } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; + +interface Camera { + id: string | number; + name: string; + type: string; + backend_api?: string; + preview_image?: string; // Auto-captured preview from backend + default_stream_profile?: { + width: number; + height: number; + fps: number; + format?: string; + }; + available_resolutions?: Array<{ + width: number; + height: number; + }>; +} + +interface CameraDetectionModalProps { + open: boolean; + onOpenChange: (open: boolean) => void; + onCameraSelected?: (cameraConfig: any) => void; +} + +const CameraDetectionModal: React.FC = ({ + open, + onOpenChange, + onCameraSelected, +}) => { + const { baseUrl, fetchWithHeaders } = useApi(); + const [isDetecting, setIsDetecting] = useState(false); + const [cameras, setCameras] = useState([]); + const [selectedCamera, setSelectedCamera] = useState(null); + const [capturingCamera, setCapturingCamera] = useState(null); + const [cameraImages, setCameraImages] = useState<{[key: string]: string}>({}); + + // Camera configuration settings + const [cameraName, setCameraName] = useState(""); + const [selectedWidth, setSelectedWidth] = useState(640); + const [selectedHeight, setSelectedHeight] = useState(480); + const [selectedFps, setSelectedFps] = useState(30); + + const detectCameras = async (cameraType?: string) => { + setIsDetecting(true); + try { + const url = cameraType + ? `${baseUrl}/cameras/detect?camera_type=${cameraType}` + : `${baseUrl}/cameras/detect`; + + const response = await fetchWithHeaders(url); + const data = await response.json(); + + if (data.status === "success") { + setCameras(data.cameras || []); + } else { + console.error("Error detecting cameras:", data.message); + setCameras([]); + } + } catch (error) { + console.error("Error detecting cameras:", error); + setCameras([]); + } finally { + setIsDetecting(false); + } + }; + + const captureImage = async (camera: Camera) => { + const cameraKey = `${camera.type}_${camera.id}`; + setCapturingCamera(cameraKey); + + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/capture`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ camera_info: camera }), + }); + + const result = await response.json(); + if (result.status === "success" && result.image_data) { + setCameraImages(prev => ({ + ...prev, + [cameraKey]: result.image_data + })); + } + } catch (error) { + console.error("Error capturing image:", error); + } finally { + setCapturingCamera(null); + } + }; + + const handleCameraSelect = (camera: Camera) => { + setSelectedCamera(camera); + setCameraName(camera.name || `Camera_${camera.id}`); + + // Set default resolution from camera info + if (camera.default_stream_profile) { + setSelectedWidth(camera.default_stream_profile.width); + setSelectedHeight(camera.default_stream_profile.height); + setSelectedFps(camera.default_stream_profile.fps); + } + }; + + const handleSaveCamera = async () => { + if (!selectedCamera || !cameraName.trim()) { + return; + } + + try { + // Create camera configuration + const response = await fetchWithHeaders(`${baseUrl}/cameras/create-config`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + camera_info: selectedCamera, + custom_settings: { + width: selectedWidth, + height: selectedHeight, + fps: selectedFps, + } + }), + }); + + const result = await response.json(); + + if (result.status === "success") { + // Save to camera config + const saveResponse = await fetchWithHeaders(`${baseUrl}/cameras/config/update`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + camera_name: cameraName, + camera_config: result.camera_config + }), + }); + + const saveResult = await saveResponse.json(); + + if (saveResult.status === "success") { + if (onCameraSelected) { + onCameraSelected({ + name: cameraName, + config: result.camera_config + }); + } + onOpenChange(false); + } else { + console.error("Error saving camera config:", saveResult.message); + } + } else { + console.error("Error creating camera config:", result.message); + } + } catch (error) { + console.error("Error handling camera save:", error); + } + }; + + const getAvailableResolutions = (camera: Camera) => { + if (camera.available_resolutions && camera.available_resolutions.length > 0) { + return camera.available_resolutions; + } + + // Fallback to common resolutions + return [ + { width: 320, height: 240 }, + { width: 640, height: 480 }, + { width: 800, height: 600 }, + { width: 1280, height: 720 }, + { width: 1920, height: 1080 }, + ]; + }; + + const getCameraPreview = (camera: Camera) => { + const cameraKey = `${camera.type}_${camera.id}`; + const capturedImage = cameraImages[cameraKey]; + const autoPreview = (camera as any).preview_image; // Preview from detection + + if (capturingCamera === cameraKey) { + return ( +
+ +
+ ); + } + + // Use captured image first, then auto preview, then placeholder + const imageData = capturedImage || autoPreview; + if (imageData) { + return ( + Camera preview + ); + } + + return ( +
+ +
+ ); + }; + + useEffect(() => { + if (open) { + detectCameras(); + } + }, [open]); + + return ( + + + + + + Camera Detection & Configuration + + + +
+ {/* Detection Controls */} +
+ + + +
+ + {/* Camera List */} +
+

+ Detected Cameras ({cameras.length}) +

+ + {cameras.length === 0 && !isDetecting && ( +
+ No cameras detected. Click "Detect Cameras" to scan for available cameras. +
+ )} + + {cameras.map((camera) => { + const isSelected = selectedCamera?.id === camera.id && selectedCamera?.type === camera.type; + + return ( +
handleCameraSelect(camera)} + > +
+
+ {getCameraPreview(camera)} +
+

{camera.name}

+
+ ID: {camera.id} | Type: {camera.type} + {camera.backend_api && ` | Backend: ${camera.backend_api}`} +
+ {camera.default_stream_profile && ( +
+ {camera.default_stream_profile.width}x{camera.default_stream_profile.height} @ {camera.default_stream_profile.fps}fps +
+ )} +
+
+ + +
+
+ ); + })} +
+ + {/* Camera Configuration */} + {selectedCamera && ( +
+

Configure Selected Camera

+ +
+
+ + setCameraName(e.target.value)} + placeholder="Enter camera name" + className="bg-gray-800 border-gray-700 text-white" + /> +
+ +
+ + +
+ +
+ + +
+
+ +
+ + +
+
+ )} +
+ + + ); +}; + +export default CameraDetectionModal; \ No newline at end of file From 423029190a8e6792b727ac019d4b0663dbc66113 Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Fri, 20 Jun 2025 12:27:34 +0200 Subject: [PATCH 2/4] Revert "Merge branch 'main' into teleoperation-cameras-config" This reverts commit 426c07b4472e40aefc387b6544fd56ee87f225b3, reversing changes made to 06fc1df6c08590259f1fb02bcfed3ebddfd3c01b. --- .gitignore | 76 -- README.md | 9 - src/App.tsx | 4 +- src/components/landing/ActionList.tsx | 25 +- src/components/landing/LandingHeader.tsx | 35 +- src/components/landing/NgrokConfigModal.tsx | 212 ++++++ src/components/landing/RecordingModal.tsx | 73 +- .../recording/CameraConfiguration.tsx | 693 ------------------ src/components/recording/PhoneCameraFeed.tsx | 117 +++ src/components/recording/QrCodeModal.tsx | 145 ++++ src/contexts/ApiContext.tsx | 85 ++- src/pages/Landing.tsx | 108 +-- src/pages/PhoneCamera.tsx | 223 ++++++ src/pages/Recording.tsx | 180 ++--- 14 files changed, 944 insertions(+), 1041 deletions(-) delete mode 100644 .gitignore create mode 100644 src/components/landing/NgrokConfigModal.tsx delete mode 100644 src/components/recording/CameraConfiguration.tsx create mode 100644 src/components/recording/PhoneCameraFeed.tsx create mode 100644 src/components/recording/QrCodeModal.tsx create mode 100644 src/pages/PhoneCamera.tsx diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 5fd30d7..0000000 --- a/.gitignore +++ /dev/null @@ -1,76 +0,0 @@ -# Dependencies -/node_modules -/.pnp -.pnp.js - -# Testing -/coverage - -# Production -/build -/dist - -# Misc -.DS_Store -.env -.env.local -.env.development.local -.env.test.local -.env.production.local -.env*.local - -# Debug logs -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# IDE -.idea/ -.vscode/ -*.swp -*.swo - -# TypeScript -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# Virtual Environment -venv/ -ENV/ diff --git a/README.md b/README.md index 1f0b1a7..b314709 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,3 @@ app_port: 7860 pinned: false short_description: Simple Interface to use LeRobot --- - -Screenshot 2025-06-19 at 00 23 55 - -# LeLab official Space repository -If you've used [LeLab](https://huggingface.co/spaces/jurmy24/leLab) and want to contribute or found a bug, this is the place to be. This repo is directly hooked up to the Hugging Face space. - -Here's the equivalent [backend](https://github.com/nicolas-rabault/leLab) that keeps the FastAPI server you run to actually wrap the LeRobot library. - -We'll be updating this README shortly. diff --git a/src/App.tsx b/src/App.tsx index 53b6e47..60aa064 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -18,7 +18,7 @@ import Training from "@/pages/Training"; import ReplayDataset from "@/pages/ReplayDataset"; import EditDataset from "@/pages/EditDataset"; import Upload from "@/pages/Upload"; - +import PhoneCamera from "@/pages/PhoneCamera"; import NotFound from "@/pages/NotFound"; import "./App.css"; import { TooltipProvider } from "@radix-ui/react-tooltip"; @@ -44,7 +44,7 @@ function App() { } /> } /> } /> - + } /> } /> diff --git a/src/components/landing/ActionList.tsx b/src/components/landing/ActionList.tsx index 7b70c7c..880a223 100644 --- a/src/components/landing/ActionList.tsx +++ b/src/components/landing/ActionList.tsx @@ -1,13 +1,9 @@ -import React from "react"; + +import React from 'react'; import { Button } from "@/components/ui/button"; import { ArrowRight, AlertTriangle } from "lucide-react"; -import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, -} from "@/components/ui/tooltip"; -import { Action } from "./types"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { Action } from './types'; interface ActionListProps { actions: Action[]; @@ -28,8 +24,7 @@ const ActionList: React.FC = ({ actions, robotModel }) => { )} {isLeKiwi && (

- LeKiwi model is not yet supported. Please select another model to - continue. + LeKiwi model is not yet supported. Please select another model to continue.

)}
@@ -43,9 +38,7 @@ const ActionList: React.FC = ({ actions, robotModel }) => {
-

- {action.title} -

+

{action.title}

{action.isWorkInProgress && (
@@ -56,13 +49,11 @@ const ActionList: React.FC = ({ actions, robotModel }) => {

Work in progress

 - - Work in Progress - + Work in Progress
)}
-

+

{action.description}

diff --git a/src/components/landing/LandingHeader.tsx b/src/components/landing/LandingHeader.tsx index 0725799..71ba13c 100644 --- a/src/components/landing/LandingHeader.tsx +++ b/src/components/landing/LandingHeader.tsx @@ -1,16 +1,49 @@ import React from "react"; import { Button } from "@/components/ui/button"; -import { Info } from "lucide-react"; +import { Info, Globe, Wifi, WifiOff } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; interface LandingHeaderProps { onShowInstructions: () => void; + onShowNgrokConfig: () => void; } const LandingHeader: React.FC = ({ onShowInstructions, + onShowNgrokConfig, }) => { + const { isNgrokEnabled } = useApi(); + return (
+ {/* Ngrok button in top right */} +
+ +
+ {/* Main header content */}
void; +} + +const NgrokConfigModal: React.FC = ({ + open, + onOpenChange, +}) => { + const { + ngrokUrl, + isNgrokEnabled, + setNgrokUrl, + resetToLocalhost, + fetchWithHeaders, + } = useApi(); + const [inputUrl, setInputUrl] = useState(ngrokUrl); + const [isTestingConnection, setIsTestingConnection] = useState(false); + const { toast } = useToast(); + + const handleSave = async () => { + if (!inputUrl.trim()) { + resetToLocalhost(); + toast({ + title: "Ngrok Disabled", + description: "Switched back to localhost mode.", + }); + onOpenChange(false); + return; + } + + setIsTestingConnection(true); + + try { + // Clean the URL + let cleanUrl = inputUrl.trim(); + if (!cleanUrl.startsWith("http")) { + cleanUrl = `https://${cleanUrl}`; + } + cleanUrl = cleanUrl.replace(/\/$/, ""); + + // Test the connection + const testResponse = await fetchWithHeaders(`${cleanUrl}/health`, { + method: "GET", + headers: { + Accept: "application/json", + }, + }); + + if (testResponse.ok) { + setNgrokUrl(cleanUrl); + toast({ + title: "Ngrok Configured Successfully", + description: `Connected to ${cleanUrl}. All API calls will now use this URL.`, + }); + onOpenChange(false); + } else { + throw new Error(`Server responded with status ${testResponse.status}`); + } + } catch (error) { + console.error("Failed to connect to ngrok URL:", error); + toast({ + title: "Connection Failed", + description: `Could not connect to ${inputUrl}. Please check the URL and ensure your ngrok tunnel is running.`, + variant: "destructive", + }); + } finally { + setIsTestingConnection(false); + } + }; + + const handleReset = () => { + resetToLocalhost(); + setInputUrl(""); + toast({ + title: "Reset to Localhost", + description: "All API calls will now use localhost:8000.", + }); + onOpenChange(false); + }; + + return ( + + + +
+ +
+ + Ngrok Configuration + + + Configure ngrok tunnel for external access and phone camera + features. + + + +
+ {/* Current Status */} +
+
+ {isNgrokEnabled ? ( + + ) : ( + + )} + + Current Mode: {isNgrokEnabled ? "Ngrok" : "Localhost"} + +
+

+ {isNgrokEnabled + ? `Using: ${ngrokUrl}` + : "Using: http://localhost:8000"} +

+
+ + {/* URL Input */} +
+ + setInputUrl(e.target.value)} + placeholder="https://abc123.ngrok.io" + className="bg-gray-800 border-gray-700 text-white text-sm sm:text-base" + /> +

+ Enter your ngrok tunnel URL. Leave empty to use localhost. +

+
+ + {/* Benefits */} +
+

+ Why use Ngrok? +

+
    +
  • • Access your robot from anywhere on the internet
    • +
  • • Use phone cameras as secondary recording angles
    • +
  • • Share your robot session with remote collaborators
    • +
  • • Test your setup from different devices
    • +
+
+ + {/* Action Buttons */} +
+ + +
+ {isNgrokEnabled && ( + + )} + + +
+
+
+ + + ); +}; + +export default NgrokConfigModal; diff --git a/src/components/landing/RecordingModal.tsx b/src/components/landing/RecordingModal.tsx index 70e4e16..703225f 100644 --- a/src/components/landing/RecordingModal.tsx +++ b/src/components/landing/RecordingModal.tsx @@ -16,16 +16,12 @@ import { DialogTitle, DialogDescription, } from "@/components/ui/dialog"; - +import { QrCode } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; import CameraDetectionModal from "@/components/ui/CameraDetectionModal"; import CameraDetectionButton from "@/components/ui/CameraDetectionButton"; import QrCodeModal from "@/components/recording/QrCodeModal"; - -import CameraConfiguration, { - CameraConfig, -} from "@/components/recording/CameraConfiguration"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; interface RecordingModalProps { @@ -47,11 +43,8 @@ interface RecordingModalProps { setSingleTask: (value: string) => void; numEpisodes: number; setNumEpisodes: (value: number) => void; - cameras: CameraConfig[]; - setCameras: (cameras: CameraConfig[]) => void; isLoadingConfigs: boolean; onStart: () => void; - releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; } const RecordingModal: React.FC = ({ open, @@ -72,11 +65,8 @@ const RecordingModal: React.FC = ({ setSingleTask, numEpisodes, setNumEpisodes, - cameras, - setCameras, isLoadingConfigs, onStart, - releaseStreamsRef, }) => { const { baseUrl, fetchWithHeaders } = useApi(); const { debouncedSavePort, debouncedSaveConfig } = useAutoSave(); @@ -84,7 +74,6 @@ const RecordingModal: React.FC = ({ const [detectionRobotType, setDetectionRobotType] = useState< "leader" | "follower" >("leader"); - const [showQrCodeModal, setShowQrCodeModal] = useState(false); const [sessionId, setSessionId] = useState(""); const [showCameraDetection, setShowCameraDetection] = useState(false); @@ -151,29 +140,19 @@ const RecordingModal: React.FC = ({ // Load leader configuration const leaderConfigResponse = await fetchWithHeaders( - `${baseUrl}/robot-config/leader?available_configs=${leaderConfigs.join( - "," - )}` + `${baseUrl}/robot-config/leader?available_configs=${leaderConfigs.join(',')}` ); const leaderConfigData = await leaderConfigResponse.json(); - if ( - leaderConfigData.status === "success" && - leaderConfigData.default_config - ) { + if (leaderConfigData.status === "success" && leaderConfigData.default_config) { setLeaderConfig(leaderConfigData.default_config); } // Load follower configuration const followerConfigResponse = await fetchWithHeaders( - `${baseUrl}/robot-config/follower?available_configs=${followerConfigs.join( - "," - )}` + `${baseUrl}/robot-config/follower?available_configs=${followerConfigs.join(',')}` ); const followerConfigData = await followerConfigResponse.json(); - if ( - followerConfigData.status === "success" && - followerConfigData.default_config - ) { + if (followerConfigData.status === "success" && followerConfigData.default_config) { setFollowerConfig(followerConfigData.default_config); } @@ -187,17 +166,7 @@ const RecordingModal: React.FC = ({ if (open && leaderConfigs.length > 0 && followerConfigs.length > 0) { loadSavedData(); } - }, [ - open, - setLeaderPort, - setFollowerPort, - setLeaderConfig, - setFollowerConfig, - leaderConfigs, - followerConfigs, - baseUrl, - fetchWithHeaders, - ]); + }, [open, setLeaderPort, setFollowerPort, setLeaderConfig, setFollowerConfig, leaderConfigs, followerConfigs, baseUrl, fetchWithHeaders]); const handleQrCodeClick = () => { // Generate a session ID for this recording session @@ -227,7 +196,6 @@ const RecordingModal: React.FC = ({ console.error("Error loading camera config:", error); } }; - return ( <> @@ -248,6 +216,23 @@ const RecordingModal: React.FC = ({ recording. +
+

+ Need an extra angle? +

+

+ Add your phone as a secondary camera. +

+ +
+

@@ -319,9 +304,7 @@ const RecordingModal: React.FC = ({ - handleFollowerPortChange(e.target.value) - } + onChange={(e) => handleFollowerPortChange(e.target.value)} placeholder="/dev/tty.usbmodem5A460816621" className="bg-gray-800 border-gray-700 text-white flex-1" /> @@ -473,14 +456,6 @@ const RecordingModal: React.FC = ({

- -
- -
diff --git a/src/components/recording/CameraConfiguration.tsx b/src/components/recording/CameraConfiguration.tsx deleted file mode 100644 index 2fb7e6d..0000000 --- a/src/components/recording/CameraConfiguration.tsx +++ /dev/null @@ -1,693 +0,0 @@ -import React, { useState, useEffect, useRef, useCallback } from "react"; -import { Button } from "@/components/ui/button"; -import { Label } from "@/components/ui/label"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Input } from "@/components/ui/input"; -import { Camera, Plus, X, Video, VideoOff } from "lucide-react"; -import { useApi } from "@/contexts/ApiContext"; -import { useToast } from "@/hooks/use-toast"; - -export interface CameraConfig { - id: string; - name: string; - type: string; - camera_index?: number; // Keep for backend compatibility - device_id: string; // Use this for actual camera selection - width: number; - height: number; - fps?: number; -} - -interface CameraConfigurationProps { - cameras: CameraConfig[]; - onCamerasChange: (cameras: CameraConfig[]) => void; - releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; // Ref to expose stream release function -} - -interface AvailableCamera { - index: number; - deviceId: string; - name: string; - available: boolean; -} - -const CameraConfiguration: React.FC = ({ - cameras, - onCamerasChange, - releaseStreamsRef, -}) => { - const { baseUrl, fetchWithHeaders } = useApi(); - const { toast } = useToast(); - - const [availableCameras, setAvailableCameras] = useState( - [] - ); - const [selectedCameraIndex, setSelectedCameraIndex] = useState(""); - const [cameraName, setCameraName] = useState(""); - const [isLoadingCameras, setIsLoadingCameras] = useState(false); - const [cameraStreams, setCameraStreams] = useState>( - new Map() - ); - - // Fetch available cameras on component mount - useEffect(() => { - fetchAvailableCameras(); - }, []); - - const fetchAvailableCameras = async () => { - console.log("🚀 fetchAvailableCameras() called"); - setIsLoadingCameras(true); - try { - console.log( - "📡 Trying backend endpoint:", - `${baseUrl}/available-cameras` - ); - const response = await fetchWithHeaders(`${baseUrl}/available-cameras`); - console.log("📡 Backend response status:", response.status, response.ok); - - if (response.ok) { - const data = await response.json(); - console.log("📡 Backend camera data received:", data); - setAvailableCameras(data.cameras || []); - - // Always also try browser detection to get device IDs - console.log("🔄 Also running browser detection for device IDs..."); - await detectBrowserCameras(); - } else { - console.log("📡 Backend failed, falling back to browser detection"); - // Fallback to browser camera detection - await detectBrowserCameras(); - } - } catch (error) { - console.error("📡 Error fetching cameras from backend:", error); - console.log("🔄 Falling back to browser detection due to error"); - // Fallback to browser camera detection - await detectBrowserCameras(); - } finally { - setIsLoadingCameras(false); - console.log("✅ fetchAvailableCameras() completed"); - } - }; - - const detectBrowserCameras = async () => { - try { - // First, request camera permissions to get proper device IDs and labels - console.log("🔐 Requesting camera permissions for device detection..."); - try { - const tempStream = await navigator.mediaDevices.getUserMedia({ - video: true, - }); - console.log("✅ Camera permission granted, stopping temp stream"); - tempStream.getTracks().forEach((track) => track.stop()); - } catch (permError) { - console.warn( - "⚠️ Camera permission denied, device IDs may be empty:", - permError - ); - } - - const devices = await navigator.mediaDevices.enumerateDevices(); - const videoDevices = devices.filter( - (device) => device.kind === "videoinput" - ); - - console.log( - "🔍 Raw video devices from enumerateDevices:", - videoDevices.map((d) => ({ - deviceId: d.deviceId, - label: d.label, - kind: d.kind, - })) - ); - - const detectedCameras = videoDevices.map((device, index) => ({ - index, - deviceId: device.deviceId || `fallback_${index}`, // Fallback if deviceId is empty - name: device.label || `Camera ${index + 1}`, - available: true, - })); - - console.log("🎬 Browser cameras with indices mapped:", detectedCameras); - setAvailableCameras(detectedCameras); - } catch (error) { - console.error("Error detecting browser cameras:", error); - toast({ - title: "Camera Detection Failed", - description: - "Could not detect available cameras. Please check permissions.", - variant: "destructive", - }); - } - }; - - const startCameraPreview = async (cameraConfig: CameraConfig) => { - try { - console.log( - "🎥 Starting camera preview for:", - cameraConfig.name, - "with device_id:", - cameraConfig.device_id, - "camera_index:", - cameraConfig.camera_index - ); - - // Create constraints with fallbacks to avoid OverconstrainedError - const constraints: MediaStreamConstraints = { - video: { - width: { ideal: cameraConfig.width, min: 320, max: 1920 }, - height: { ideal: cameraConfig.height, min: 240, max: 1080 }, - frameRate: { ideal: cameraConfig.fps || 30, min: 10, max: 60 }, - }, - }; - - // Only add deviceId if it's not a fallback - if ( - cameraConfig.device_id && - !cameraConfig.device_id.startsWith("fallback_") - ) { - (constraints.video as MediaTrackConstraints).deviceId = { - exact: cameraConfig.device_id, // Changed from 'ideal' to 'exact' - }; - console.log( - "🔧 Using EXACT deviceId constraint:", - cameraConfig.device_id - ); - } else { - console.log("⚠️ No valid deviceId, will use default camera"); - } - - console.log( - "📋 Final constraints:", - JSON.stringify(constraints, null, 2) - ); - - const stream = await navigator.mediaDevices.getUserMedia(constraints); - - // Get the actual device being used - const videoTrack = stream.getVideoTracks()[0]; - if (videoTrack) { - const settings = videoTrack.getSettings(); - console.log("✅ Actual camera settings:", { - deviceId: settings.deviceId, - label: videoTrack.label, - width: settings.width, - height: settings.height, - }); - - // Check if we got the camera we requested - if ( - cameraConfig.device_id && - settings.deviceId !== cameraConfig.device_id - ) { - console.warn( - "⚠️ CAMERA MISMATCH! Requested:", - cameraConfig.device_id, - "Got:", - settings.deviceId - ); - } else { - console.log("✅ Camera match confirmed!"); - } - } - - console.log( - "Camera stream created successfully for:", - cameraConfig.name, - { - streamId: stream.id, - tracks: stream.getTracks().length, - videoTracks: stream.getVideoTracks().length, - active: stream.active, - } - ); - - setCameraStreams((prev) => { - const newMap = new Map(prev.set(cameraConfig.id, stream)); - console.log("Updated camera streams map:", Array.from(newMap.keys())); - return newMap; - }); - - // Force a small delay to ensure state update - await new Promise((resolve) => setTimeout(resolve, 100)); - - return stream; - } catch (error: unknown) { - console.error("Error starting camera preview:", error); - - const isMediaError = error instanceof Error; - const errorName = isMediaError ? error.name : ""; - const errorMessage = isMediaError ? error.message : "Unknown error"; - - // If constraints failed, try with basic constraints - if ( - errorName === "OverconstrainedError" || - errorName === "NotReadableError" - ) { - try { - console.log("Retrying with basic constraints..."); - const basicStream = await navigator.mediaDevices.getUserMedia({ - video: { width: 640, height: 480 }, - }); - - setCameraStreams( - (prev) => new Map(prev.set(cameraConfig.id, basicStream)) - ); - toast({ - title: "Camera Preview Started", - description: `${cameraConfig.name} started with basic settings due to constraint issues.`, - }); - return basicStream; - } catch (basicError) { - console.error("Error with basic constraints:", basicError); - } - } - - toast({ - title: "Camera Preview Failed", - description: `Could not start preview for ${cameraConfig.name}: ${errorMessage}`, - variant: "destructive", - }); - return null; - } - }; - - const stopCameraPreview = (cameraId: string) => { - const stream = cameraStreams.get(cameraId); - if (stream) { - stream.getTracks().forEach((track) => track.stop()); - setCameraStreams((prev) => { - const newMap = new Map(prev); - newMap.delete(cameraId); - return newMap; - }); - } - }; - - const addCamera = async () => { - if (!selectedCameraIndex || !cameraName.trim()) { - toast({ - title: "Missing Information", - description: "Please select a camera and provide a name.", - variant: "destructive", - }); - return; - } - - const cameraIndex = parseInt(selectedCameraIndex); - const selectedCamera = availableCameras.find( - (cam) => cam.index === cameraIndex - ); - - if (!selectedCamera) { - toast({ - title: "Invalid Camera", - description: "Selected camera is not available.", - variant: "destructive", - }); - return; - } - - // Check if camera is already added - if (cameras.some((cam) => cam.camera_index === cameraIndex)) { - toast({ - title: "Camera Already Added", - description: "This camera is already in the configuration.", - variant: "destructive", - }); - return; - } - - const newCamera: CameraConfig = { - id: `camera_${Date.now()}`, - name: cameraName.trim(), - type: "opencv", - camera_index: selectedCamera.index, - device_id: selectedCamera.deviceId, - width: 640, - height: 480, - fps: 30, - }; - - console.log("🆕 Creating new camera config:", { - name: newCamera.name, - camera_index: newCamera.camera_index, - device_id: newCamera.device_id, - selectedCamera: selectedCamera, - }); - - const updatedCameras = [...cameras, newCamera]; - onCamerasChange(updatedCameras); - - // Start preview for the new camera - await startCameraPreview(newCamera); - - // Reset form - setSelectedCameraIndex(""); - setCameraName(""); - - toast({ - title: "Camera Added", - description: `${newCamera.name} has been added to the configuration.`, - }); - }; - - const removeCamera = (cameraId: string) => { - stopCameraPreview(cameraId); - const updatedCameras = cameras.filter((cam) => cam.id !== cameraId); - onCamerasChange(updatedCameras); - - toast({ - title: "Camera Removed", - description: "Camera has been removed from the configuration.", - }); - }; - - const updateCamera = (cameraId: string, updates: Partial) => { - const updatedCameras = cameras.map((cam) => - cam.id === cameraId ? { ...cam, ...updates } : cam - ); - onCamerasChange(updatedCameras); - }; - - // Function to release all camera streams (for recording start) - const releaseAllCameraStreams = useCallback(() => { - console.log("🔓 Releasing all camera streams for recording..."); - cameraStreams.forEach((stream, cameraId) => { - console.log(`🔓 Stopping stream for camera: ${cameraId}`); - stream.getTracks().forEach((track) => track.stop()); - }); - setCameraStreams(new Map()); - console.log("✅ All camera streams released"); - }, [cameraStreams]); - - // Expose the release function to parent component via ref - useEffect(() => { - if (releaseStreamsRef) { - releaseStreamsRef.current = releaseAllCameraStreams; - } - }, [releaseStreamsRef, releaseAllCameraStreams]); - - // Clean up streams on component unmount - useEffect(() => { - return () => { - cameraStreams.forEach((stream) => { - stream.getTracks().forEach((track) => track.stop()); - }); - }; - }, []); - - return ( -
-

- Camera Configuration -

- - {/* Add Camera Section */} -
-

Add Camera

- -
-
- - -
- -
- - setCameraName(e.target.value)} - placeholder="e.g., workspace_cam" - className="bg-gray-800 border-gray-700 text-white" - /> -
- -
- -
-
-
- - {/* Configured Cameras */} - {cameras.length > 0 && ( -
-

- Configured Cameras ({cameras.length}) -

- -
- {cameras.map((camera) => ( - removeCamera(camera.id)} - onUpdate={(updates) => updateCamera(camera.id, updates)} - onStartPreview={() => startCameraPreview(camera)} - /> - ))} -
-
- )} - - {cameras.length === 0 && ( -
- -

No cameras configured. Add a camera to get started.

-
- )} -
- ); -}; - -interface CameraPreviewProps { - camera: CameraConfig; - stream?: MediaStream; - onRemove: () => void; - onUpdate: (updates: Partial) => void; - onStartPreview: () => void; -} - -const CameraPreview: React.FC = ({ - camera, - stream, - onRemove, - onUpdate, - onStartPreview, -}) => { - const videoRef = useRef(null); - const [isPreviewActive, setIsPreviewActive] = useState(false); - - // Debug logging for props - console.log("CameraPreview render for:", camera.name, { - hasStream: !!stream, - streamActive: stream?.active, - isPreviewActive, - streamId: stream?.id, - }); - - useEffect(() => { - const video = videoRef.current; - if (video && stream) { - console.log("Setting stream to video element for camera:", camera.name); - video.srcObject = stream; - - // Explicitly play the video to ensure it starts - const playVideo = async () => { - try { - await video.play(); - console.log("Video playing successfully for camera:", camera.name); - setIsPreviewActive(true); - } catch (error) { - console.error("Error playing video for camera:", camera.name, error); - // Try to play without audio in case autoplay is blocked - video.muted = true; - try { - await video.play(); - console.log("Video playing muted for camera:", camera.name); - setIsPreviewActive(true); - } catch (mutedError) { - console.error("Error playing muted video:", mutedError); - setIsPreviewActive(false); - } - } - }; - - // Wait for metadata to load before playing - if (video.readyState >= 1) { - playVideo(); - } else { - video.addEventListener("loadedmetadata", playVideo, { once: true }); - } - } else { - console.log("No stream or video element for camera:", camera.name); - setIsPreviewActive(false); - } - }, [stream, camera.name]); - - useEffect(() => { - // Auto-start preview when camera is added - if (!stream && !isPreviewActive) { - console.log("Auto-starting preview for camera:", camera.name); - onStartPreview(); - } - }, [stream, isPreviewActive, onStartPreview, camera.name]); - - return ( -
- {/* Camera Preview */} -
- {/* Always show the video element if we have a stream, regardless of isPreviewActive */} - {stream ? ( - <> -
- - {/* Camera Info */} -
-
-
{camera.name}
- -
- -
-
- Resolution: -
- - onUpdate({ width: parseInt(e.target.value) || 640 }) - } - className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" - min="320" - max="1920" - /> - × - - onUpdate({ height: parseInt(e.target.value) || 480 }) - } - className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" - min="240" - max="1080" - /> -
-
-
- FPS: - - onUpdate({ fps: parseInt(e.target.value) || 30 }) - } - className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" - min="10" - max="60" - /> -
-
- -
- Type: {camera.type} | Device: {camera.device_id?.substring(0, 10)}... -
-
-
- ); -}; - -export default CameraConfiguration; diff --git a/src/components/recording/PhoneCameraFeed.tsx b/src/components/recording/PhoneCameraFeed.tsx new file mode 100644 index 0000000..8037e9a --- /dev/null +++ b/src/components/recording/PhoneCameraFeed.tsx @@ -0,0 +1,117 @@ +import React, { useEffect, useRef, useState } from "react"; +import { Smartphone, WifiOff } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; + +interface PhoneCameraFeedProps { + sessionId: string; +} + +const PhoneCameraFeed: React.FC = ({ sessionId }) => { + const { wsBaseUrl } = useApi(); + const videoRef = useRef(null); + const [isConnected, setIsConnected] = useState(false); + const [error, setError] = useState(null); + const wsRef = useRef(null); + const mediaSourceRef = useRef(null); + + useEffect(() => { + if (!sessionId) return; + + const connectWebSocket = () => { + try { + const ws = new WebSocket(`${wsBaseUrl}/ws/camera/${sessionId}`); + wsRef.current = ws; + + ws.onopen = () => { + console.log("Camera feed WebSocket connected"); + setIsConnected(true); + setError(null); + }; + + ws.onmessage = (event) => { + // Handle incoming video chunks + if (event.data instanceof Blob) { + handleVideoChunk(event.data); + } else if (event.data === "camera_connected") { + setIsConnected(true); + } + }; + + ws.onclose = () => { + console.log("Camera feed WebSocket disconnected"); + setIsConnected(false); + }; + + ws.onerror = (error) => { + console.error("Camera feed WebSocket error:", error); + setError("Failed to connect to camera feed"); + setIsConnected(false); + }; + } catch (error) { + console.error("Failed to create WebSocket:", error); + setError("Failed to establish connection"); + } + }; + + const handleVideoChunk = (chunk: Blob) => { + // For now, we'll just log that we received a chunk + // In a full implementation, this would use MediaSource API + console.log("Received video chunk:", chunk.size, "bytes"); + }; + + connectWebSocket(); + + return () => { + if (wsRef.current) { + wsRef.current.close(); + } + if (mediaSourceRef.current) { + mediaSourceRef.current.endOfStream(); + } + }; + }, [sessionId]); + + if (error) { + return ( +
+ +

{error}

+
+ ); + } + + if (!isConnected) { + return ( +
+ +

+ Waiting for phone camera... +

+
+
+ ); + } + + return ( +
+
+ ); +}; + +export default PhoneCameraFeed; diff --git a/src/components/recording/QrCodeModal.tsx b/src/components/recording/QrCodeModal.tsx new file mode 100644 index 0000000..c7919b8 --- /dev/null +++ b/src/components/recording/QrCodeModal.tsx @@ -0,0 +1,145 @@ + +import React, { useEffect, useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Copy, Smartphone, QrCode } from "lucide-react"; +import { useToast } from "@/hooks/use-toast"; + +interface QrCodeModalProps { + open: boolean; + onOpenChange: (open: boolean) => void; + sessionId: string; +} + +const QrCodeModal: React.FC = ({ + open, + onOpenChange, + sessionId, +}) => { + const { toast } = useToast(); + const [qrCodeUrl, setQrCodeUrl] = useState(""); + const [phoneUrl, setPhoneUrl] = useState(""); + + useEffect(() => { + if (sessionId) { + // Get current host URL - in production this would be the deployed URL + const currentHost = window.location.origin; + const phonePageUrl = `${currentHost}/phone-camera?sessionId=${sessionId}`; + setPhoneUrl(phonePageUrl); + + // Generate QR code using a public QR code API + const qrApiUrl = `https://api.qrserver.com/v1/create-qr-code/?size=200x200&data=${encodeURIComponent(phonePageUrl)}`; + setQrCodeUrl(qrApiUrl); + } + }, [sessionId]); + + const copyToClipboard = async () => { + try { + await navigator.clipboard.writeText(phoneUrl); + toast({ + title: "URL Copied!", + description: "Phone camera URL has been copied to clipboard.", + }); + } catch (error) { + toast({ + title: "Copy Failed", + description: "Could not copy URL to clipboard.", + variant: "destructive", + }); + } + }; + + return ( + + + +
+
+ +
+
+ + Add Phone Camera + + + Scan the QR code with your phone to add a camera feed to your recording session. + + + +
+ {/* QR Code Display */} +
+ {qrCodeUrl ? ( +
+ QR Code for phone camera +
+ ) : ( +
+ +
+ )} +
+ + {/* Instructions */} +
+

+ How to connect your phone camera: +

+
    +
  1. Open your phone's camera app
    2. +
  2. Scan the QR code above
    3. +
  3. Allow camera permissions when prompted
    4. +
  4. Your phone camera feed will appear in the recording interface
    5. +
+
+ + {/* Manual URL Option */} +
+

+ Or open this URL manually on your phone: +

+
+ + +
+
+ + {/* Close Button */} +
+ +
+
+ + + ); +}; + +export default QrCodeModal; diff --git a/src/contexts/ApiContext.tsx b/src/contexts/ApiContext.tsx index 581ad74..8b1ee60 100644 --- a/src/contexts/ApiContext.tsx +++ b/src/contexts/ApiContext.tsx @@ -1,8 +1,19 @@ -import React, { createContext, useContext, ReactNode } from "react"; +import React, { + createContext, + useContext, + useState, + useEffect, + ReactNode, +} from "react"; interface ApiContextType { baseUrl: string; wsBaseUrl: string; + isNgrokEnabled: boolean; + setNgrokUrl: (url: string) => void; + resetToLocalhost: () => void; + ngrokUrl: string; + getHeaders: () => Record; fetchWithHeaders: (url: string, options?: RequestInit) => Promise; } @@ -16,8 +27,69 @@ interface ApiProviderProps { } export const ApiProvider: React.FC = ({ children }) => { - const baseUrl = DEFAULT_LOCALHOST; - const wsBaseUrl = DEFAULT_WS_LOCALHOST; + const [ngrokUrl, setNgrokUrlState] = useState(""); + const [isNgrokEnabled, setIsNgrokEnabled] = useState(false); + + // Load saved ngrok configuration on mount + useEffect(() => { + const savedNgrokUrl = localStorage.getItem("ngrok-url"); + const savedNgrokEnabled = localStorage.getItem("ngrok-enabled") === "true"; + + if (savedNgrokUrl && savedNgrokEnabled) { + setNgrokUrlState(savedNgrokUrl); + setIsNgrokEnabled(true); + } + }, []); + + const setNgrokUrl = (url: string) => { + // Clean and validate the URL + let cleanUrl = url.trim(); + if (cleanUrl && !cleanUrl.startsWith("http")) { + cleanUrl = `https://${cleanUrl}`; + } + + // Remove trailing slash + cleanUrl = cleanUrl.replace(/\/$/, ""); + + setNgrokUrlState(cleanUrl); + setIsNgrokEnabled(!!cleanUrl); + + // Persist to localStorage + if (cleanUrl) { + localStorage.setItem("ngrok-url", cleanUrl); + localStorage.setItem("ngrok-enabled", "true"); + } else { + localStorage.removeItem("ngrok-url"); + localStorage.removeItem("ngrok-enabled"); + } + }; + + const resetToLocalhost = () => { + setNgrokUrlState(""); + setIsNgrokEnabled(false); + localStorage.removeItem("ngrok-url"); + localStorage.removeItem("ngrok-enabled"); + }; + + const baseUrl = isNgrokEnabled && ngrokUrl ? ngrokUrl : DEFAULT_LOCALHOST; + const wsBaseUrl = + isNgrokEnabled && ngrokUrl + ? ngrokUrl.replace("https://", "wss://").replace("http://", "ws://") + : DEFAULT_WS_LOCALHOST; + + // Helper function to get headers with ngrok skip warning if needed + const getHeaders = (): Record => { + const headers: Record = { + "Content-Type": "application/json", + }; + + // Add ngrok skip warning header when using ngrok + if (isNgrokEnabled && ngrokUrl) { + headers["ngrok-skip-browser-warning"] = "true"; + } + + return headers; + }; // Enhanced fetch function that automatically includes necessary headers const fetchWithHeaders = async ( @@ -27,7 +99,7 @@ export const ApiProvider: React.FC = ({ children }) => { const enhancedOptions: RequestInit = { ...options, headers: { - "Content-Type": "application/json", + ...getHeaders(), ...options.headers, }, }; @@ -40,6 +112,11 @@ export const ApiProvider: React.FC = ({ children }) => { value={{ baseUrl, wsBaseUrl, + isNgrokEnabled, + setNgrokUrl, + resetToLocalhost, + ngrokUrl, + getHeaders, fetchWithHeaders, }} > diff --git a/src/pages/Landing.tsx b/src/pages/Landing.tsx index 3bca968..ee19bd5 100644 --- a/src/pages/Landing.tsx +++ b/src/pages/Landing.tsx @@ -1,4 +1,4 @@ -import React, { useState, useRef, useEffect } from "react"; +import React, { useState } from "react"; import { useNavigate } from "react-router-dom"; import { useToast } from "@/hooks/use-toast"; import { ArrowRight } from "lucide-react"; @@ -8,19 +8,18 @@ import ActionList from "@/components/landing/ActionList"; import PermissionModal from "@/components/landing/PermissionModal"; import TeleoperationModal from "@/components/landing/TeleoperationModal"; import RecordingModal from "@/components/landing/RecordingModal"; - +import NgrokConfigModal from "@/components/landing/NgrokConfigModal"; import { Action } from "@/components/landing/types"; import UsageInstructionsModal from "@/components/landing/UsageInstructionsModal"; import DirectFollowerModal from "@/components/landing/DirectFollowerModal"; import { useApi } from "@/contexts/ApiContext"; -import { CameraConfig } from "@/components/recording/CameraConfiguration"; const Landing = () => { const [robotModel, setRobotModel] = useState("SO101"); const [showPermissionModal, setShowPermissionModal] = useState(false); const [showTeleoperationModal, setShowTeleoperationModal] = useState(false); const [showUsageModal, setShowUsageModal] = useState(false); - + const [showNgrokModal, setShowNgrokModal] = useState(false); const [leaderPort, setLeaderPort] = useState("/dev/tty.usbmodem5A460816421"); const [followerPort, setFollowerPort] = useState( "/dev/tty.usbmodem5A460816621" @@ -45,10 +44,6 @@ const Landing = () => { const [datasetRepoId, setDatasetRepoId] = useState(""); const [singleTask, setSingleTask] = useState(""); const [numEpisodes, setNumEpisodes] = useState(5); - const [cameras, setCameras] = useState([]); - - // Camera stream release ref - const releaseStreamsRef = useRef<(() => void) | null>(null); // Direct follower control state const [showDirectFollowerModal, setShowDirectFollowerModal] = useState(false); @@ -60,30 +55,6 @@ const Landing = () => { const navigate = useNavigate(); const { toast } = useToast(); - // Clear camera state and release streams when returning to landing page - useEffect(() => { - // If we have cameras and returning from a recording session, clear them - if (cameras.length > 0) { - console.log( - "🧹 Landing page: Cleaning up camera state from previous session" - ); - if (releaseStreamsRef.current) { - releaseStreamsRef.current(); - } - setCameras([]); // Clear camera configuration - } - }, []); // Only run on mount - - // Cleanup when leaving landing page - useEffect(() => { - return () => { - if (releaseStreamsRef.current) { - console.log("🧹 Landing page: Cleaning up camera streams on unmount"); - releaseStreamsRef.current(); - } - }; - }, []); - const loadConfigs = async () => { setIsLoadingConfigs(true); try { @@ -128,15 +99,6 @@ const Landing = () => { } }; - const handleRecordingModalClose = (open: boolean) => { - setShowRecordingModal(open); - // Release camera streams when modal is closed - if (!open && releaseStreamsRef.current) { - console.log("🧹 Modal closed: Releasing camera streams"); - releaseStreamsRef.current(); - } - }; - const handleTrainingClick = () => { if (robotModel) { navigate("/training"); @@ -220,40 +182,6 @@ const Landing = () => { return; } - // 🔓 CRITICAL: Release all camera streams before backend accesses them - if (cameras.length > 0 && releaseStreamsRef.current) { - console.log("🔓 Releasing camera streams before starting recording..."); - - toast({ - title: "Preparing Camera Resources", - description: `Releasing ${cameras.length} camera stream(s) for recording...`, - }); - - releaseStreamsRef.current(); - - // Wait a moment for camera resources to be fully released - await new Promise((resolve) => setTimeout(resolve, 500)); - console.log("✅ Camera streams released, proceeding with recording..."); - - toast({ - title: "Camera Resources Ready", - description: - "Camera streams released successfully. Starting recording...", - }); - } - - // Convert cameras to the LeRobot format - const cameraDict = cameras.reduce((acc, cam) => { - acc[cam.name] = { - type: cam.type, - camera_index: cam.camera_index, - width: cam.width, - height: cam.height, - fps: cam.fps, - }; - return acc; - }, {} as Record); - const recordingConfig = { leader_port: recordLeaderPort, follower_port: recordFollowerPort, @@ -268,7 +196,6 @@ const Landing = () => { video: true, push_to_hub: false, resume: false, - cameras: cameraDict, }; setShowRecordingModal(false); @@ -364,18 +291,11 @@ const Landing = () => { handler: handleTeleoperationClick, color: "bg-yellow-500 hover:bg-yellow-600", }, - { - title: "Record Dataset", - description: "Record episodes for training data.", - handler: handleRecordingClick, - color: "bg-red-500 hover:bg-red-600", - }, { title: "Direct Follower Control", - description: "Control robot arm with mouse movements.", + description: "Train a model on your datasets.", handler: handleDirectFollowerClick, color: "bg-blue-500 hover:bg-blue-600", - isWorkInProgress: true, }, { title: "Calibration", @@ -384,6 +304,12 @@ const Landing = () => { color: "bg-indigo-500 hover:bg-indigo-600", isWorkInProgress: true, }, + { + title: "Record Dataset", + description: "Record episodes for training data.", + handler: handleRecordingClick, + color: "bg-red-500 hover:bg-red-600", + }, { title: "Training", description: "Train a model on your datasets.", @@ -403,7 +329,10 @@ const Landing = () => { return (
- setShowUsageModal(true)} /> + setShowUsageModal(true)} + onShowNgrokConfig={() => setShowNgrokModal(true)} + />
@@ -444,7 +373,7 @@ const Landing = () => { { setSingleTask={setSingleTask} numEpisodes={numEpisodes} setNumEpisodes={setNumEpisodes} - cameras={cameras} - setCameras={setCameras} isLoadingConfigs={isLoadingConfigs} onStart={handleStartRecording} - releaseStreamsRef={releaseStreamsRef} /> { isLoadingConfigs={isLoadingConfigs} onStart={handleStartDirectFollower} /> +
); }; diff --git a/src/pages/PhoneCamera.tsx b/src/pages/PhoneCamera.tsx new file mode 100644 index 0000000..632b3e2 --- /dev/null +++ b/src/pages/PhoneCamera.tsx @@ -0,0 +1,223 @@ +import React, { useEffect, useRef, useState } from "react"; +import { useSearchParams } from "react-router-dom"; +import { Camera, WifiOff, Smartphone } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { useApi } from "@/contexts/ApiContext"; + +const PhoneCamera = () => { + const [searchParams] = useSearchParams(); + const sessionId = searchParams.get("sessionId"); + const { wsBaseUrl } = useApi(); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const wsRef = useRef(null); + const streamRef = useRef(null); + const [isConnected, setIsConnected] = useState(false); + const [isStreaming, setIsStreaming] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!sessionId) { + setError("No session ID provided"); + return; + } + + connectWebSocket(); + return () => { + cleanup(); + }; + }, [sessionId]); + + const connectWebSocket = () => { + try { + const ws = new WebSocket(`${wsBaseUrl}/ws/camera/${sessionId}`); + wsRef.current = ws; + + ws.onopen = () => { + console.log("Phone camera WebSocket connected"); + setIsConnected(true); + setError(null); + // Notify the recording page that a camera is connected + ws.send("camera_connected"); + }; + + ws.onclose = () => { + console.log("Phone camera WebSocket disconnected"); + setIsConnected(false); + }; + + ws.onerror = (error) => { + console.error("Phone camera WebSocket error:", error); + setError("Failed to connect to recording session"); + setIsConnected(false); + }; + } catch (error) { + console.error("Failed to create WebSocket:", error); + setError("Failed to establish connection"); + } + }; + + const startCamera = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + video: { + facingMode: "environment", // Use back camera + width: { ideal: 640 }, + height: { ideal: 480 }, + }, + audio: false, + }); + + streamRef.current = stream; + if (videoRef.current) { + videoRef.current.srcObject = stream; + } + + setIsStreaming(true); + startVideoStreaming(); + } catch (error) { + console.error("Error accessing camera:", error); + setError("Could not access camera. Please check permissions."); + } + }; + + const startVideoStreaming = () => { + if (!videoRef.current || !canvasRef.current || !wsRef.current) return; + + const video = videoRef.current; + const canvas = canvasRef.current; + const ctx = canvas.getContext("2d"); + const ws = wsRef.current; + + const sendFrame = () => { + if (!ctx || !isConnected || !isStreaming) return; + + // Draw video frame to canvas + canvas.width = video.videoWidth || 640; + canvas.height = video.videoHeight || 480; + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + + // Convert to blob and send via WebSocket + canvas.toBlob( + (blob) => { + if (blob && ws.readyState === WebSocket.OPEN) { + ws.send(blob); + } + }, + "image/jpeg", + 0.7 + ); + }; + + // Send frames at ~10 FPS + const interval = setInterval(sendFrame, 100); + + return () => clearInterval(interval); + }; + + const cleanup = () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + if (wsRef.current) { + wsRef.current.close(); + } + }; + + if (!sessionId) { + return ( +
+
+ +

Invalid Session

+

No session ID provided in the URL.

+
+
+ ); + } + + return ( +
+ {/* Header */} +
+
+

LeLab Camera

+
+
+ + {/* Camera Section */} +
+ {error ? ( +
+
+ +

Connection Error

+

{error}

+ +
+
+ ) : !isConnected ? ( +
+
+
+

Connecting...

+

Connecting to recording session

+
+
+ ) : !isStreaming ? ( +
+ +

Ready to Start

+

+ Tap the button below to start your camera and begin streaming to + the recording session. +

+ +
+ ) : ( +
+
+
+
+ STREAMING +
+ +
+ +
+
+ + + +
+

+ Your camera is now streaming to the recording session. Keep this + page open during recording. +

+
+
+ )} +
+
+ ); +}; + +export default PhoneCamera; diff --git a/src/pages/Recording.tsx b/src/pages/Recording.tsx index 25ce296..7a5ccf2 100644 --- a/src/pages/Recording.tsx +++ b/src/pages/Recording.tsx @@ -5,7 +5,7 @@ import { useToast } from "@/hooks/use-toast"; import { ArrowLeft, Square, SkipForward, RotateCcw, Play } from "lucide-react"; import UrdfViewer from "@/components/UrdfViewer"; import UrdfProcessorInitializer from "@/components/UrdfProcessorInitializer"; - +import PhoneCameraFeed from "@/components/recording/PhoneCameraFeed"; import { useApi } from "@/contexts/ApiContext"; interface RecordingConfig { @@ -56,9 +56,10 @@ const Recording = () => { ); const [recordingSessionStarted, setRecordingSessionStarted] = useState(false); - // Local UI state for immediate user feedback - const [transitioningToReset, setTransitioningToReset] = useState(false); - const [transitioningToNext, setTransitioningToNext] = useState(false); + // QR Code and camera states + const [showQrModal, setShowQrModal] = useState(false); + const [sessionId, setSessionId] = useState(""); + const [phoneCameraConnected, setPhoneCameraConnected] = useState(false); // Redirect if no config provided useEffect(() => { @@ -91,25 +92,8 @@ const Recording = () => { ); if (response.ok) { const status = await response.json(); - console.log( - `📊 Backend Status: ${status.current_phase} | Transition States: reset=${transitioningToReset}, next=${transitioningToNext}` - ); setBackendStatus(status); - // 🎯 CLEAR TRANSITION STATES: Only clear when backend actually reaches the expected phase - if (status.current_phase === "resetting" && transitioningToReset) { - console.log( - "✅ Clearing transitioningToReset - backend reached resetting phase" - ); - setTransitioningToReset(false); - } - if (status.current_phase === "recording" && transitioningToNext) { - console.log( - "✅ Clearing transitioningToNext - backend reached recording phase" - ); - setTransitioningToNext(false); - } - // If backend recording stopped and session ended, navigate to upload if ( !status.recording_active && @@ -142,14 +126,52 @@ const Recording = () => { return () => { if (statusInterval) clearInterval(statusInterval); }; - }, [ - recordingSessionStarted, - recordingConfig, - navigate, - toast, - transitioningToReset, - transitioningToNext, - ]); + }, [recordingSessionStarted, recordingConfig, navigate, toast]); + + // Generate session ID when component loads + useEffect(() => { + const newSessionId = `session_${Date.now()}_${Math.random() + .toString(36) + .substr(2, 9)}`; + setSessionId(newSessionId); + }, []); + + // Listen for phone camera connections + useEffect(() => { + if (!sessionId) return; + + const connectToPhoneCameraWS = () => { + const ws = new WebSocket(`${wsBaseUrl}/ws/camera/${sessionId}`); + + ws.onopen = () => { + console.log("Phone camera WebSocket connected"); + }; + + ws.onmessage = (event) => { + if (event.data === "camera_connected" && !phoneCameraConnected) { + setPhoneCameraConnected(true); + toast({ + title: "Phone Camera Connected!", + description: "New camera feed detected and connected successfully.", + }); + } + }; + + ws.onclose = () => { + console.log("Phone camera WebSocket disconnected"); + setPhoneCameraConnected(false); + }; + + ws.onerror = (error) => { + console.error("Phone camera WebSocket error:", error); + }; + + return ws; + }; + + const ws = connectToPhoneCameraWS(); + return () => ws.close(); + }, [sessionId, phoneCameraConnected, toast]); const formatTime = (seconds: number): string => { const mins = Math.floor(seconds / 60); @@ -196,24 +218,6 @@ const Recording = () => { const handleExitEarly = async () => { if (!backendStatus?.available_controls.exit_early) return; - // 🎯 IMMEDIATE UI FEEDBACK: Show transition state before backend response - const currentPhase = backendStatus.current_phase; - if (currentPhase === "recording") { - console.log("🎯 Setting transitioningToReset = true"); - setTransitioningToReset(true); - toast({ - title: "Ending Episode Recording", - description: `Moving to reset phase for episode ${backendStatus.current_episode}...`, - }); - } else if (currentPhase === "resetting") { - console.log("🎯 Setting transitioningToNext = true"); - setTransitioningToNext(true); - toast({ - title: "Reset Complete", - description: `Moving to next episode...`, - }); - } - try { const response = await fetchWithHeaders( `${baseUrl}/recording-exit-early`, @@ -224,12 +228,19 @@ const Recording = () => { const data = await response.json(); if (response.ok) { - // ✅ SUCCESS: Don't clear transition states here - let them persist until backend phase changes - // The transition states will be cleared when the backend status actually updates to the new phase + const currentPhase = backendStatus.current_phase; + if (currentPhase === "recording") { + toast({ + title: "Episode Recording Ended", + description: `Episode ${backendStatus.current_episode} recording completed. Moving to reset phase.`, + }); + } else if (currentPhase === "resetting") { + toast({ + title: "Reset Complete", + description: `Moving to next episode...`, + }); + } } else { - // Clear transition states on error - setTransitioningToReset(false); - setTransitioningToNext(false); toast({ title: "Error", description: data.message, @@ -237,9 +248,6 @@ const Recording = () => { }); } } catch (error) { - // Clear transition states on error - setTransitioningToReset(false); - setTransitioningToNext(false); toast({ title: "Connection Error", description: "Could not connect to the backend server.", @@ -351,20 +359,12 @@ const Recording = () => { const sessionElapsedTime = backendStatus.session_elapsed_seconds || 0; const getPhaseTitle = () => { - // 🎯 IMMEDIATE FEEDBACK: Show transition titles - if (transitioningToReset) return "Transitioning to Reset"; - if (transitioningToNext) return "Moving to Next Episode"; - if (currentPhase === "recording") return "Episode Recording Time"; if (currentPhase === "resetting") return "Environment Reset Time"; return "Phase Time"; }; const getStatusText = () => { - // 🎯 IMMEDIATE FEEDBACK: Show transition states - if (transitioningToReset) return "MOVING TO RESET PHASE"; - if (transitioningToNext) return "MOVING TO NEXT EPISODE"; - if (currentPhase === "recording") return `RECORDING EPISODE ${currentEpisode}`; if (currentPhase === "resetting") return "RESET THE ENVIRONMENT"; @@ -373,10 +373,6 @@ const Recording = () => { }; const getStatusColor = () => { - // 🎯 IMMEDIATE FEEDBACK: Show transition state colors - if (transitioningToReset) return "text-blue-400"; // Blue for transition - if (transitioningToNext) return "text-blue-400"; // Blue for transition - if (currentPhase === "recording") return "text-red-400"; if (currentPhase === "resetting") return "text-orange-400"; if (currentPhase === "preparing") return "text-yellow-400"; @@ -384,10 +380,6 @@ const Recording = () => { }; const getDotColor = () => { - // 🎯 IMMEDIATE FEEDBACK: Show transition state dots with animation - if (transitioningToReset) return "bg-blue-500 animate-pulse"; // Blue pulsing for transition - if (transitioningToNext) return "bg-blue-500 animate-pulse"; // Blue pulsing for transition - if (currentPhase === "recording") return "bg-red-500 animate-pulse"; if (currentPhase === "resetting") return "bg-orange-500 animate-pulse"; if (currentPhase === "preparing") return "bg-yellow-500"; @@ -505,23 +497,11 @@ const Recording = () => {
+ + {/* Phone Camera Feed - takes up 1 column */} + {phoneCameraConnected && ( +
+

+ Phone Camera +

+ +
+ )}
{/* URDF Viewer Section */} From f53852ad6af36737753d18c1b667c028e85731db Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Mon, 23 Jun 2025 18:12:44 +0200 Subject: [PATCH 3/4] improve camera management with WebRTC --- src/components/control/VisualizerPanel.tsx | 82 +- src/components/landing/RecordingModal.tsx | 93 +- src/components/landing/TeleoperationModal.tsx | 108 +- .../recording/CameraConfiguration.tsx | 951 ++++++++++++++++++ .../recording/RobustCameraConfiguration.tsx | 755 ++++++++++++++ .../webrtc/WebRTCCameraConfiguration.tsx | 856 ++++++++++++++++ .../webrtc/WebRTCVisualizerPanel.tsx | 406 ++++++++ src/pages/Landing.tsx | 24 +- src/pages/Teleoperation.tsx | 8 +- src/types/webrtc.ts | 117 +++ src/utils/cameraUtils.ts | 149 +++ src/utils/webrtc/BrowserEventEmitter.ts | 61 ++ src/utils/webrtc/WebRTCManager.ts | 395 ++++++++ 13 files changed, 3807 insertions(+), 198 deletions(-) create mode 100644 src/components/recording/CameraConfiguration.tsx create mode 100644 src/components/recording/RobustCameraConfiguration.tsx create mode 100644 src/components/webrtc/WebRTCCameraConfiguration.tsx create mode 100644 src/components/webrtc/WebRTCVisualizerPanel.tsx create mode 100644 src/types/webrtc.ts create mode 100644 src/utils/cameraUtils.ts create mode 100644 src/utils/webrtc/BrowserEventEmitter.ts create mode 100644 src/utils/webrtc/WebRTCManager.ts diff --git a/src/components/control/VisualizerPanel.tsx b/src/components/control/VisualizerPanel.tsx index 7edaa64..9d66312 100644 --- a/src/components/control/VisualizerPanel.tsx +++ b/src/components/control/VisualizerPanel.tsx @@ -23,7 +23,7 @@ const VisualizerPanel: React.FC = ({ const [cameraStreams, setCameraStreams] = useState<{[key: string]: string}>({}); const [streamErrors, setStreamErrors] = useState<{[key: string]: boolean}>({}); - // Load camera configuration and streaming URLs + // Load camera configuration and streaming URLs using robust system useEffect(() => { const loadCameraConfig = async () => { try { @@ -35,12 +35,23 @@ const VisualizerPanel: React.FC = ({ const cameras = data.camera_config.cameras || {}; setCameraConfig(cameras); - // Generate streaming URLs for each configured camera + console.log("🎬 Loaded camera config for teleoperation:", cameras); + + // Generate streaming URLs using robust identifiers const streams: {[key: string]: string} = {}; - Object.keys(cameras).forEach(cameraName => { - streams[cameraName] = `${baseUrl}/cameras/stream/${encodeURIComponent(cameraName)}`; + Object.entries(cameras).forEach(([cameraName, config]: [string, any]) => { + // CRITICAL: Use hash if available for consistency, otherwise fall back to name + const streamingIdentifier = config.hash || cameraName; + const streamUrl = `${baseUrl}/cameras/stream/${encodeURIComponent(streamingIdentifier)}`; + + console.log(`🔗 Camera ${cameraName}: streaming via ${streamingIdentifier} -> ${streamUrl}`); + + // Use camera name as key for UI, but stream via hash/identifier + streams[cameraName] = streamUrl; }); setCameraStreams(streams); + + console.log("🔗 Generated streaming URLs:", streams); } } catch (error) { console.error("Error loading camera config:", error); @@ -62,19 +73,57 @@ const VisualizerPanel: React.FC = ({ setStreamErrors(prev => ({ ...prev, [cameraName]: false })); }; - // Get camera entries - only show configured cameras + // Get camera entries - only show configured cameras (sorted by name for consistency) const getCameraSlots = () => { const configuredCameras = Object.entries(cameraConfig); const cameraSlots = []; + // Simple alphabetical sort by name - same as configuration modal + configuredCameras.sort(([nameA], [nameB]) => nameA.localeCompare(nameB)); + // Add only configured cameras (no empty slots) configuredCameras.forEach(([name, config]) => { cameraSlots.push({ name, config, isConfigured: true }); }); + console.log("🎬 Camera slots in teleoperation (sorted by name):", cameraSlots.map(slot => ({ + name: slot.name, + system_index: slot.config.system_index, + index_or_path: slot.config.index_or_path, + device_id: slot.config.device_id + }))); + return cameraSlots; }; + // Get responsive layout classes based on number of cameras + const getCameraLayoutClasses = () => { + const cameraCount = getCameraSlots().length; + + if (cameraCount === 0) return "lg:w-80"; + if (cameraCount === 1) return "lg:w-80"; // 1 camera: single column, full width + if (cameraCount === 2) return "lg:w-80"; // 2 cameras: single column, stacked + if (cameraCount === 3) return "lg:w-80 lg:max-h-[70vh] lg:overflow-y-auto"; // 3 cameras: single column with scroll + if (cameraCount === 4) return "lg:w-96"; // 4 cameras: 2x2 grid + if (cameraCount <= 6) return "lg:w-[32rem]"; // 5-6 cameras: 3x2 grid + + return "lg:w-[36rem]"; // 7+ cameras: wider grid + }; + + // Get grid classes for camera layout + const getCameraGridClasses = () => { + const cameraCount = getCameraSlots().length; + + if (cameraCount === 0) return ""; + if (cameraCount === 1) return "flex flex-col gap-3"; // 1 camera: full width + if (cameraCount === 2) return "flex flex-col gap-3"; // 2 cameras: stacked vertically + if (cameraCount === 3) return "flex flex-col gap-3"; // 3 cameras: stacked vertically with scroll + if (cameraCount === 4) return "grid grid-cols-2 gap-3"; // 4 cameras: 2x2 + if (cameraCount <= 6) return "grid grid-cols-3 gap-2"; // 5-6 cameras: 3x2 + + return "grid grid-cols-3 gap-2"; // 7+ cameras: 3 columns + }; + return (
= ({
-
+
{getCameraSlots().length > 0 ? ( - getCameraSlots().map((cameraSlot, index) => ( -
+
+ {getCameraSlots().map((cameraSlot, index) => ( +
{isLoadingCameras ? ( <> @@ -152,7 +202,10 @@ const VisualizerPanel: React.FC = ({
- {cameraSlot.name} + {cameraSlot.name} {/* Show consistent user-given name */} + + + {cameraSlot.config.hash ? `Hash: ${cameraSlot.config.hash}` : `Index: ${cameraSlot.config.index_or_path}`} {cameraSlot.config.width}x{cameraSlot.config.height} @ {cameraSlot.config.fps}fps @@ -160,8 +213,9 @@ const VisualizerPanel: React.FC = ({
)} -
- )) +
+ ))} +
) : (
diff --git a/src/components/landing/RecordingModal.tsx b/src/components/landing/RecordingModal.tsx index 703225f..e37cac2 100644 --- a/src/components/landing/RecordingModal.tsx +++ b/src/components/landing/RecordingModal.tsx @@ -19,8 +19,9 @@ import { import { QrCode } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; -import CameraDetectionModal from "@/components/ui/CameraDetectionModal"; -import CameraDetectionButton from "@/components/ui/CameraDetectionButton"; +import WebRTCCameraConfiguration, { + CameraConfig, +} from "@/components/webrtc/WebRTCCameraConfiguration"; import QrCodeModal from "@/components/recording/QrCodeModal"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; @@ -43,8 +44,11 @@ interface RecordingModalProps { setSingleTask: (value: string) => void; numEpisodes: number; setNumEpisodes: (value: number) => void; + cameras: CameraConfig[]; + setCameras: (cameras: CameraConfig[]) => void; isLoadingConfigs: boolean; onStart: () => void; + releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; } const RecordingModal: React.FC = ({ open, @@ -65,8 +69,11 @@ const RecordingModal: React.FC = ({ setSingleTask, numEpisodes, setNumEpisodes, + cameras, + setCameras, isLoadingConfigs, onStart, + releaseStreamsRef, }) => { const { baseUrl, fetchWithHeaders } = useApi(); const { debouncedSavePort, debouncedSaveConfig } = useAutoSave(); @@ -76,8 +83,6 @@ const RecordingModal: React.FC = ({ >("leader"); const [showQrCodeModal, setShowQrCodeModal] = useState(false); const [sessionId, setSessionId] = useState(""); - const [showCameraDetection, setShowCameraDetection] = useState(false); - const [cameraConfig, setCameraConfig] = useState({}); const handlePortDetection = (robotType: "leader" | "follower") => { setDetectionRobotType(robotType); @@ -156,8 +161,6 @@ const RecordingModal: React.FC = ({ setFollowerConfig(followerConfigData.default_config); } - // Load camera configuration - await loadCameraConfig(); } catch (error) { console.error("Error loading saved data:", error); } @@ -177,25 +180,6 @@ const RecordingModal: React.FC = ({ setShowQrCodeModal(true); }; - const handleCameraSelected = (cameraData: any) => { - setCameraConfig(prev => ({ - ...prev, - [cameraData.name]: cameraData.config - })); - }; - - const loadCameraConfig = async () => { - try { - const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); - const data = await response.json(); - - if (data.status === "success" && data.camera_config) { - setCameraConfig(data.camera_config.cameras || {}); - } - } catch (error) { - console.error("Error loading camera config:", error); - } - }; return ( <> @@ -352,54 +336,12 @@ const RecordingModal: React.FC = ({ {/* Camera Configuration Section */}
-
-

- Camera Configuration -

- setShowCameraDetection(true)} - /> -
- - {Object.keys(cameraConfig).length > 0 ? ( -
-
- Configured Cameras ({Object.keys(cameraConfig).length}) -
- {Object.entries(cameraConfig).map(([name, config]: [string, any]) => ( -
-
-
{name}
-
- {config.type} - {config.width}x{config.height} @ {config.fps}fps -
-
- -
- ))} -
- ) : ( -
- No cameras configured. Click the camera icon to detect and configure cameras. -
- )} +
@@ -491,11 +433,6 @@ const RecordingModal: React.FC = ({ sessionId={sessionId} /> - ); }; diff --git a/src/components/landing/TeleoperationModal.tsx b/src/components/landing/TeleoperationModal.tsx index a60b935..3fb3278 100644 --- a/src/components/landing/TeleoperationModal.tsx +++ b/src/components/landing/TeleoperationModal.tsx @@ -19,8 +19,9 @@ import { import { Settings } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; -import CameraDetectionModal from "@/components/ui/CameraDetectionModal"; -import CameraDetectionButton from "@/components/ui/CameraDetectionButton"; +import WebRTCCameraConfiguration, { + CameraConfig, +} from "@/components/webrtc/WebRTCCameraConfiguration"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; @@ -37,6 +38,8 @@ interface TeleoperationModalProps { setFollowerConfig: (value: string) => void; leaderConfigs: string[]; followerConfigs: string[]; + cameras: CameraConfig[]; + setCameras: (cameras: CameraConfig[]) => void; isLoadingConfigs: boolean; onStart: () => void; } @@ -54,6 +57,8 @@ const TeleoperationModal: React.FC = ({ setFollowerConfig, leaderConfigs, followerConfigs, + cameras, + setCameras, isLoadingConfigs, onStart, }) => { @@ -63,8 +68,6 @@ const TeleoperationModal: React.FC = ({ const [detectionRobotType, setDetectionRobotType] = useState< "leader" | "follower" >("leader"); - const [showCameraDetection, setShowCameraDetection] = useState(false); - const [cameraConfig, setCameraConfig] = useState({}); const handlePortDetection = (robotType: "leader" | "follower") => { setDetectionRobotType(robotType); @@ -105,48 +108,6 @@ const TeleoperationModal: React.FC = ({ debouncedSaveConfig("follower", value); }; - const handleCameraSelected = (cameraData: any) => { - setCameraConfig(prev => ({ - ...prev, - [cameraData.name]: cameraData.config - })); - }; - - const loadCameraConfig = async () => { - try { - const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); - const data = await response.json(); - - if (data.status === "success" && data.camera_config) { - setCameraConfig(data.camera_config.cameras || {}); - } - } catch (error) { - console.error("Error loading camera config:", error); - } - }; - - const handleRemoveCamera = async (cameraName: string) => { - try { - const response = await fetchWithHeaders(`${baseUrl}/cameras/config/${encodeURIComponent(cameraName)}`, { - method: "DELETE", - }); - - const result = await response.json(); - - if (result.status === "success") { - setCameraConfig(prev => { - const newConfig = { ...prev }; - delete newConfig[cameraName]; - return newConfig; - }); - console.log(`Camera "${cameraName}" removed successfully`); - } else { - console.error("Error removing camera:", result.message); - } - } catch (error) { - console.error("Error removing camera:", error); - } - }; // Load saved ports and configurations on component mount useEffect(() => { @@ -188,8 +149,6 @@ const TeleoperationModal: React.FC = ({ setFollowerConfig(followerConfigData.default_config); } - // Load camera configuration - await loadCameraConfig(); } catch (error) { console.error("Error loading saved data:", error); } @@ -325,48 +284,12 @@ const TeleoperationModal: React.FC = ({
-
-
-

- Camera Configuration -

- setShowCameraDetection(true)} - /> -
- - {Object.keys(cameraConfig).length > 0 ? ( -
-
- Configured Cameras ({Object.keys(cameraConfig).length}) -
- {Object.entries(cameraConfig).map(([name, config]: [string, any]) => ( -
-
-
{name}
-
- {config.type} - {config.width}x{config.height} @ {config.fps}fps -
-
- -
- ))} -
- ) : ( -
- No cameras configured. Click the camera icon to detect and configure cameras. -
- )} +
+
@@ -395,11 +318,6 @@ const TeleoperationModal: React.FC = ({ onPortDetected={handlePortDetected} /> - ); }; diff --git a/src/components/recording/CameraConfiguration.tsx b/src/components/recording/CameraConfiguration.tsx new file mode 100644 index 0000000..b97e1db --- /dev/null +++ b/src/components/recording/CameraConfiguration.tsx @@ -0,0 +1,951 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { Camera, Plus, X, Video, VideoOff, RefreshCw } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; +import { useToast } from "@/hooks/use-toast"; + +export interface CameraConfig { + id: string; + name: string; + type: string; + camera_index?: number; // Keep for backend compatibility + device_id: string; // Use this for actual camera selection + width: number; + height: number; + fps?: number; +} + +interface CameraConfigurationProps { + cameras: CameraConfig[]; + onCamerasChange: (cameras: CameraConfig[]) => void; + releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; // Ref to expose stream release function + loadSavedCameras?: boolean; // If true, load saved cameras on mount +} + +interface AvailableCamera { + index: number; + deviceId: string; + name: string; + available: boolean; + preview_image?: string; // Base64 preview from backend +} + +const CameraConfiguration: React.FC = ({ + cameras, + onCamerasChange, + releaseStreamsRef, + loadSavedCameras = true, +}) => { + const { baseUrl, fetchWithHeaders } = useApi(); + const { toast } = useToast(); + + const [availableCameras, setAvailableCameras] = useState([]); + const [selectedCameraIndex, setSelectedCameraIndex] = useState(""); + const [cameraName, setCameraName] = useState(""); + const [isLoadingCameras, setIsLoadingCameras] = useState(false); + const [hasDetectedOnMount, setHasDetectedOnMount] = useState(false); + const [hasLoadedSavedCameras, setHasLoadedSavedCameras] = useState(false); + const [cameraStreams, setCameraStreams] = useState>(new Map()); + const [savedCameraConfigs, setSavedCameraConfigs] = useState<{[key: string]: any}>({}); + + // Load saved camera configurations on mount + useEffect(() => { + if (loadSavedCameras) { + setHasLoadedSavedCameras(false); + setHasDetectedOnMount(false); + loadSavedCameraConfigs(); + } + }, [loadSavedCameras]); + + // Auto-detect cameras on component mount, but only if no saved cameras were loaded and haven't detected yet + useEffect(() => { + if (!hasDetectedOnMount && hasLoadedSavedCameras && cameras.length === 0 && loadSavedCameras) { + console.log("🔍 No saved cameras found, starting auto-detection..."); + fetchAvailableCameras(); + setHasDetectedOnMount(true); + } + }, [hasDetectedOnMount, hasLoadedSavedCameras, cameras.length, loadSavedCameras]); + + const loadSavedCameraConfigs = async () => { + try { + console.log("🔄 Loading saved camera configurations from backend..."); + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + console.log("📡 Backend response:", data); + + if (data.status === "success" && data.camera_config && data.camera_config.cameras) { + const camerasFromBackend = data.camera_config.cameras; + console.log("📦 Raw cameras from backend:", camerasFromBackend); + + setSavedCameraConfigs(camerasFromBackend); + + // Simple conversion - just use device_id + const savedCameras: CameraConfig[] = Object.entries(camerasFromBackend).map(([name, config]: [string, any]) => ({ + id: `saved_${name}`, + name: name, + type: config.type || "browser", // Default to browser type + camera_index: config.index_or_path || 0, // Keep for reference + device_id: config.device_id, // Primary identifier + width: config.width || 640, + height: config.height || 480, + fps: config.fps || 30, + })); + + // Sort by name for consistent UI order (simple and predictable) + savedCameras.sort((a, b) => a.name.localeCompare(b.name)); + + console.log("🎬 Converted cameras for frontend (sorted by camera_index):", savedCameras); + onCamerasChange(savedCameras); + + // Start previews for saved cameras in the correct order + console.log("🔄 Starting previews in this order:", savedCameras.map(cam => ({ + name: cam.name, + camera_index: cam.camera_index, + id: cam.id + }))); + + savedCameras.forEach((camera, index) => { + setTimeout(() => { + console.log(`🎥 Starting preview ${index} for ${camera.name} (camera_index: ${camera.camera_index})`); + startCameraPreview(camera); + }, index * 100); // Stagger the preview starts + }); + + console.log("✅ Loaded saved camera configurations:", savedCameras); + console.log("🚫 Skipping auto-detection because saved cameras exist"); + } else { + console.log("ℹ️ No saved camera configurations found"); + onCamerasChange([]); // Ensure cameras array is empty + } + } catch (error) { + console.error("Error loading saved camera configs:", error); + onCamerasChange([]); // Ensure cameras array is empty on error + } finally { + setHasLoadedSavedCameras(true); + } + }; + + const fetchAvailableCameras = async () => { + console.log("🚀 fetchAvailableCameras() called"); + setIsLoadingCameras(true); + try { + // Use ONLY browser detection to avoid duplicates and device ID issues + console.log("🔍 Using pure browser detection for consistency..."); + await detectBrowserCameras(); + } catch (error) { + console.error("📡 Error fetching cameras:", error); + toast({ + title: "Camera Detection Failed", + description: "Could not detect available cameras. Please check permissions.", + variant: "destructive", + }); + } finally { + setIsLoadingCameras(false); + console.log("✅ fetchAvailableCameras() completed"); + } + }; + + const detectBrowserCameras = async () => { + try { + // First, request camera permissions to get proper device IDs and labels + console.log("🔐 Requesting camera permissions for device detection..."); + try { + const tempStream = await navigator.mediaDevices.getUserMedia({ video: true }); + console.log("✅ Camera permission granted, stopping temp stream"); + tempStream.getTracks().forEach((track) => track.stop()); + } catch (permError) { + console.warn("⚠️ Camera permission denied, device IDs may be empty:", permError); + } + + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter((device) => device.kind === "videoinput"); + + console.log("🔍 Raw video devices from enumerateDevices:", videoDevices.map((d) => ({ + deviceId: d.deviceId, + label: d.label, + kind: d.kind, + }))); + + const detectedCameras = videoDevices.map((device, index) => ({ + index, + deviceId: device.deviceId, + name: device.label || `Camera ${index + 1}`, + available: true, + })); + + console.log("🎬 Browser cameras detected:", detectedCameras); + setAvailableCameras(detectedCameras); + } catch (error) { + console.error("Error detecting browser cameras:", error); + toast({ + title: "Camera Detection Failed", + description: "Could not detect available cameras. Please check permissions.", + variant: "destructive", + }); + } + }; + + const startCameraPreview = async (cameraConfig: CameraConfig) => { + try { + console.log("🎥 Starting camera preview for:", cameraConfig.name, "with device_id:", cameraConfig.device_id, "camera_index:", cameraConfig.camera_index); + + // For saved cameras, try to use the actual device_id if it doesn't start with "saved_" + if (cameraConfig.device_id.startsWith("saved_")) { + const savedName = cameraConfig.device_id.replace("saved_", ""); + const savedConfig = savedCameraConfigs[savedName]; + + // If we have a real device_id in the saved config, use it + if (savedConfig && savedConfig.device_id && !savedConfig.device_id.startsWith("saved_")) { + const constraints: MediaStreamConstraints = { + video: { + deviceId: { exact: savedConfig.device_id }, + width: { ideal: cameraConfig.width, min: 320, max: 1920 }, + height: { ideal: cameraConfig.height, min: 240, max: 1080 }, + frameRate: { ideal: cameraConfig.fps || 30, min: 10, max: 60 }, + }, + }; + console.log("🔧 Using saved deviceId for camera:", savedConfig.device_id); + + const stream = await navigator.mediaDevices.getUserMedia(constraints); + setCameraStreams((prev) => new Map(prev.set(cameraConfig.id, stream))); + return stream; + } + + // Fallback: try to find device by camera index + if (savedConfig && typeof savedConfig.index_or_path === "number") { + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter(d => d.kind === "videoinput"); + + if (videoDevices[savedConfig.index_or_path]) { + const constraints: MediaStreamConstraints = { + video: { + deviceId: { exact: videoDevices[savedConfig.index_or_path].deviceId }, + width: { ideal: cameraConfig.width, min: 320, max: 1920 }, + height: { ideal: cameraConfig.height, min: 240, max: 1080 }, + frameRate: { ideal: cameraConfig.fps || 30, min: 10, max: 60 }, + }, + }; + console.log("🔧 Using deviceId by index for saved camera:", videoDevices[savedConfig.index_or_path].deviceId); + + const stream = await navigator.mediaDevices.getUserMedia(constraints); + setCameraStreams((prev) => new Map(prev.set(cameraConfig.id, stream))); + return stream; + } + } + } + + // For new cameras, use normal device ID + const constraints: MediaStreamConstraints = { + video: { + width: { ideal: cameraConfig.width, min: 320, max: 1920 }, + height: { ideal: cameraConfig.height, min: 240, max: 1080 }, + frameRate: { ideal: cameraConfig.fps || 30, min: 10, max: 60 }, + }, + }; + + // Only add deviceId if it's not a fallback or backend prefixed + if (cameraConfig.device_id && + !cameraConfig.device_id.startsWith("fallback_") && + !cameraConfig.device_id.startsWith("backend_") && + !cameraConfig.device_id.startsWith("saved_")) { + (constraints.video as MediaTrackConstraints).deviceId = { + exact: cameraConfig.device_id, + }; + console.log("🔧 Using EXACT deviceId constraint:", cameraConfig.device_id); + } else { + console.log("⚠️ No valid deviceId, will use default camera"); + } + + console.log("📋 Final constraints:", JSON.stringify(constraints, null, 2)); + + const stream = await navigator.mediaDevices.getUserMedia(constraints); + + // Get the actual device being used + const videoTrack = stream.getVideoTracks()[0]; + if (videoTrack) { + const settings = videoTrack.getSettings(); + console.log("✅ Actual camera settings:", { + deviceId: settings.deviceId, + label: videoTrack.label, + width: settings.width, + height: settings.height, + }); + } + + console.log("Camera stream created successfully for:", cameraConfig.name); + + setCameraStreams((prev) => { + const newMap = new Map(prev.set(cameraConfig.id, stream)); + console.log("Updated camera streams map:", Array.from(newMap.keys())); + return newMap; + }); + + // Force a small delay to ensure state update + await new Promise((resolve) => setTimeout(resolve, 100)); + + return stream; + } catch (error: unknown) { + console.error("Error starting camera preview:", error); + + const isMediaError = error instanceof Error; + const errorName = isMediaError ? error.name : ""; + const errorMessage = isMediaError ? error.message : "Unknown error"; + + // If constraints failed, try with basic constraints + if (errorName === "OverconstrainedError" || errorName === "NotReadableError") { + try { + console.log("Retrying with basic constraints..."); + const basicStream = await navigator.mediaDevices.getUserMedia({ + video: { width: 640, height: 480 }, + }); + + setCameraStreams((prev) => new Map(prev.set(cameraConfig.id, basicStream))); + toast({ + title: "Camera Preview Started", + description: `${cameraConfig.name} started with basic settings due to constraint issues.`, + }); + return basicStream; + } catch (basicError) { + console.error("Error with basic constraints:", basicError); + } + } + + toast({ + title: "Camera Preview Failed", + description: `Could not start preview for ${cameraConfig.name}: ${errorMessage}`, + variant: "destructive", + }); + return null; + } + }; + + const stopCameraPreview = (cameraId: string) => { + const stream = cameraStreams.get(cameraId); + if (stream) { + stream.getTracks().forEach((track) => track.stop()); + setCameraStreams((prev) => { + const newMap = new Map(prev); + newMap.delete(cameraId); + return newMap; + }); + } + }; + + // Function to find the actual system camera index for a given device ID + const findSystemIndexForDeviceId = async (deviceId: string): Promise => { + try { + console.log("🔍 Finding system index for device ID:", deviceId); + + // Get all video devices and find the index of our device + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + console.log("📋 All video devices found:", videoDevices.map((d, i) => ({ + index: i, + deviceId: d.deviceId, + label: d.label + }))); + + const deviceIndex = videoDevices.findIndex(device => device.deviceId === deviceId); + + if (deviceIndex !== -1) { + console.log(`✅ Device ID ${deviceId} mapped to system index ${deviceIndex}`); + + // Test that this mapping is correct by trying to open the camera + try { + const testStream = await navigator.mediaDevices.getUserMedia({ + video: { deviceId: { exact: deviceId } } + }); + testStream.getTracks().forEach(track => track.stop()); + console.log(`✅ Verified: Device ID ${deviceId} works at index ${deviceIndex}`); + } catch (testError) { + console.warn(`⚠️ Device ID ${deviceId} failed verification:`, testError); + } + + return deviceIndex; + } + + console.warn(`⚠️ Could not find system index for device ID: ${deviceId}`); + return -1; + } catch (error) { + console.error("Error finding system index for device ID:", error); + return -1; + } + }; + + const addCamera = async () => { + if (!selectedCameraIndex || !cameraName.trim()) { + toast({ + title: "Missing Information", + description: "Please select a camera and provide a name.", + variant: "destructive", + }); + return; + } + + const cameraIndex = parseInt(selectedCameraIndex); + const selectedCamera = availableCameras.find((cam) => cam.index === cameraIndex); + + if (!selectedCamera) { + toast({ + title: "Invalid Camera", + description: "Selected camera is not available.", + variant: "destructive", + }); + return; + } + + // Check if camera is already added + if (cameras.some((cam) => cam.camera_index === cameraIndex)) { + toast({ + title: "Camera Already Added", + description: "This camera is already in the configuration.", + variant: "destructive", + }); + return; + } + + // Map device_id to actual system index for backend compatibility + const systemIndex = await findSystemIndexForDeviceId(selectedCamera.deviceId); + + const newCamera: CameraConfig = { + id: `camera_${Date.now()}`, + name: cameraName.trim(), // Simple user name + type: "browser", // Use browser type + camera_index: systemIndex !== -1 ? systemIndex : selectedCamera.index, // Use mapped system index + device_id: selectedCamera.deviceId, // Keep device_id for frontend preview + width: 640, + height: 480, + fps: 30, + }; + + console.log("🆕 Creating new camera config:", { + name: newCamera.name, + camera_index: newCamera.camera_index, + device_id: newCamera.device_id, + systemIndex: systemIndex, + selectedCamera: selectedCamera, + }); + + // Save camera configuration to backend + try { + const configResponse = await fetchWithHeaders(`${baseUrl}/cameras/create-config`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + camera_info: { + id: newCamera.camera_index, + name: newCamera.name, + type: newCamera.type, + }, + custom_settings: { + width: newCamera.width, + height: newCamera.height, + fps: newCamera.fps, + device_id: newCamera.device_id, // Only device_id, no complex mapping + } + }), + }); + + const configResult = await configResponse.json(); + + if (configResult.status === "success") { + // Simple config with just device_id + const configWithDeviceId = { + ...configResult.camera_config, + device_id: newCamera.device_id, + type: "browser" // Mark as browser type for streaming + }; + + // Save to camera config + const saveResponse = await fetchWithHeaders(`${baseUrl}/cameras/config/update`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + camera_name: newCamera.name, + camera_config: configWithDeviceId + }), + }); + + const saveResult = await saveResponse.json(); + + if (saveResult.status === "success") { + console.log("Camera configuration saved successfully with device_id:", newCamera.device_id); + + // Update local saved configs + setSavedCameraConfigs(prev => ({ + ...prev, + [newCamera.name]: configWithDeviceId + })); + } else { + console.error("Error saving camera config:", saveResult.message); + } + } + } catch (error) { + console.error("Error saving camera configuration:", error); + // Continue even if save fails + } + + const updatedCameras = [...cameras, newCamera]; + onCamerasChange(updatedCameras); + + // Start preview for the new camera + await startCameraPreview(newCamera); + + // Reset form + setSelectedCameraIndex(""); + setCameraName(""); + + toast({ + title: "Camera Added", + description: `${newCamera.name} has been added to the configuration.`, + }); + }; + + const removeCamera = async (cameraId: string) => { + const camera = cameras.find(cam => cam.id === cameraId); + + if (camera) { + // Remove from backend - check if camera exists in saved configs + if (camera.device_id.startsWith("saved_") || savedCameraConfigs[camera.name]) { + try { + console.log(`Removing camera "${camera.name}" from backend...`); + const response = await fetchWithHeaders(`${baseUrl}/cameras/config/${encodeURIComponent(camera.name)}`, { + method: "DELETE", + }); + + const result = await response.json(); + + if (result.status === "success") { + console.log(`✅ Camera "${camera.name}" removed from backend successfully`); + + // Update local saved configs + setSavedCameraConfigs(prev => { + const newConfig = { ...prev }; + delete newConfig[camera.name]; + console.log("🗑️ Updated local saved configs after removal:", newConfig); + return newConfig; + }); + } else { + console.error("❌ Error removing camera from backend:", result.message); + } + } catch (error) { + console.error("Error removing camera from backend:", error); + } + } else { + console.log(`Camera "${camera.name}" is not saved in backend, removing only from local state`); + } + } + + stopCameraPreview(cameraId); + const updatedCameras = cameras.filter((cam) => cam.id !== cameraId); + onCamerasChange(updatedCameras); + + toast({ + title: "Camera Removed", + description: "Camera has been removed from the configuration.", + }); + }; + + const updateCamera = (cameraId: string, updates: Partial) => { + const updatedCameras = cameras.map((cam) => + cam.id === cameraId ? { ...cam, ...updates } : cam + ); + onCamerasChange(updatedCameras); + }; + + // Function to release all camera streams (for recording start) + const releaseAllCameraStreams = useCallback(() => { + console.log("🔓 Releasing all camera streams for recording..."); + cameraStreams.forEach((stream, cameraId) => { + console.log(`🔓 Stopping stream for camera: ${cameraId}`); + stream.getTracks().forEach((track) => track.stop()); + }); + setCameraStreams(new Map()); + console.log("✅ All camera streams released"); + }, [cameraStreams]); + + // Expose the release function to parent component via ref + useEffect(() => { + if (releaseStreamsRef) { + releaseStreamsRef.current = releaseAllCameraStreams; + } + }, [releaseStreamsRef, releaseAllCameraStreams]); + + // Clean up streams on component unmount + useEffect(() => { + return () => { + cameraStreams.forEach((stream) => { + stream.getTracks().forEach((track) => track.stop()); + }); + }; + }, []); + + return ( +
+
+

+ Camera Configuration +

+
+ + + +
+
+ + {/* Add Camera Section */} + {availableCameras.length > 0 && ( +
+

Add Camera

+ +
+
+ + +
+ +
+ + setCameraName(e.target.value)} + placeholder="e.g., workspace_cam" + className="bg-gray-800 border-gray-700 text-white" + /> +
+ +
+ +
+
+
+ )} + + {/* Configured Cameras */} + {cameras.length > 0 && ( +
+

+ Configured Cameras ({cameras.length}) +

+ +
+ {cameras.map((camera) => ( + removeCamera(camera.id)} + onUpdate={(updates) => updateCamera(camera.id, updates)} + onStartPreview={() => startCameraPreview(camera)} + /> + ))} +
+
+ )} + + {cameras.length === 0 && !isLoadingCameras && ( +
+ +

No cameras configured.

+

Click "Refresh Cameras" to detect available cameras and add them.

+
+ )} +
+ ); +}; + +interface CameraPreviewProps { + camera: CameraConfig; + stream?: MediaStream; + onRemove: () => void; + onUpdate: (updates: Partial) => void; + onStartPreview: () => void; +} + +const CameraPreview: React.FC = ({ + camera, + stream, + onRemove, + onUpdate, + onStartPreview, +}) => { + const videoRef = useRef(null); + const [isPreviewActive, setIsPreviewActive] = useState(false); + + useEffect(() => { + const video = videoRef.current; + if (video && stream) { + console.log("Setting stream to video element for camera:", camera.name); + video.srcObject = stream; + + const playVideo = async () => { + try { + await video.play(); + console.log("Video playing successfully for camera:", camera.name); + setIsPreviewActive(true); + } catch (error) { + console.error("Error playing video for camera:", camera.name, error); + video.muted = true; + try { + await video.play(); + console.log("Video playing muted for camera:", camera.name); + setIsPreviewActive(true); + } catch (mutedError) { + console.error("Error playing muted video:", mutedError); + setIsPreviewActive(false); + } + } + }; + + if (video.readyState >= 1) { + playVideo(); + } else { + video.addEventListener("loadedmetadata", playVideo, { once: true }); + } + } else { + console.log("No stream or video element for camera:", camera.name); + setIsPreviewActive(false); + } + }, [stream, camera.name]); + + useEffect(() => { + // Auto-start preview when camera is added + if (!stream && !isPreviewActive) { + console.log("Auto-starting preview for camera:", camera.name); + onStartPreview(); + } + }, [stream, isPreviewActive, onStartPreview, camera.name]); + + return ( +
+ {/* Camera Preview */} +
+ {stream ? ( + <> +
+ + {/* Camera Info */} +
+
+
+ {camera.name.split('_')[0]} {/* Show only the user-given name part */} +
+ +
+ +
+
+ Resolution: +
+ + onUpdate({ width: parseInt(e.target.value) || 640 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="320" + max="1920" + /> + × + + onUpdate({ height: parseInt(e.target.value) || 480 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="240" + max="1080" + /> +
+
+
+ FPS: + + onUpdate({ fps: parseInt(e.target.value) || 30 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="10" + max="60" + /> +
+
+ +
+ Type: {camera.type} | Index: {camera.camera_index} +
+
+
+ ); +}; + +export default CameraConfiguration; \ No newline at end of file diff --git a/src/components/recording/RobustCameraConfiguration.tsx b/src/components/recording/RobustCameraConfiguration.tsx new file mode 100644 index 0000000..e83466f --- /dev/null +++ b/src/components/recording/RobustCameraConfiguration.tsx @@ -0,0 +1,755 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { Camera, Plus, X, Video, VideoOff, RefreshCw, AlertTriangle } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; +import { useToast } from "@/hooks/use-toast"; +import { + RobustCameraConfig, + createCameraHash, + validateCameraConfig, + findCurrentDeviceIndex, + cleanupCameraConfigs, + getCameraDisplayName, + getCameraStreamingId, + sortCamerasConsistently, + convertFromRobustConfig, + convertToRobustConfig +} from "@/utils/cameraUtils"; + +// Keep legacy interface for compatibility +export interface CameraConfig { + id: string; + name: string; + type: string; + camera_index?: number; + device_id: string; + width: number; + height: number; + fps?: number; +} + +interface RobustCameraConfigurationProps { + cameras: CameraConfig[]; + onCamerasChange: (cameras: CameraConfig[]) => void; + releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; + loadSavedCameras?: boolean; +} + +interface DetectedCamera { + index: number; + deviceId: string; + name: string; + available: boolean; +} + +const RobustCameraConfiguration: React.FC = ({ + cameras, + onCamerasChange, + releaseStreamsRef, + loadSavedCameras = true, +}) => { + const { baseUrl, fetchWithHeaders } = useApi(); + const { toast } = useToast(); + + // Robust camera management state + const [robustConfigs, setRobustConfigs] = useState([]); + const [detectedCameras, setDetectedCameras] = useState([]); + const [selectedCameraIndex, setSelectedCameraIndex] = useState(""); + const [cameraName, setCameraName] = useState(""); + const [isLoadingCameras, setIsLoadingCameras] = useState(false); + const [hasDetectedOnMount, setHasDetectedOnMount] = useState(false); + const [hasLoadedSavedCameras, setHasLoadedSavedCameras] = useState(false); + const [cameraStreams, setCameraStreams] = useState>(new Map()); + + // Convert robust configs to legacy format for parent component + const syncToParent = useCallback((robustConfigs: RobustCameraConfig[]) => { + const legacyConfigs = sortCamerasConsistently(robustConfigs) + .filter(config => config.is_available) + .map(convertFromRobustConfig); + + console.log("🔄 Syncing to parent - robust configs:", robustConfigs); + console.log("🔄 Syncing to parent - legacy format:", legacyConfigs); + + onCamerasChange(legacyConfigs); + }, [onCamerasChange]); + + // Load saved camera configurations on mount + useEffect(() => { + if (loadSavedCameras) { + setHasLoadedSavedCameras(false); + setHasDetectedOnMount(false); + loadSavedCameraConfigs(); + } + }, [loadSavedCameras]); + + // Auto-detect cameras if no saved cameras were loaded + useEffect(() => { + if (!hasDetectedOnMount && hasLoadedSavedCameras && robustConfigs.length === 0 && loadSavedCameras) { + console.log("🔍 No saved cameras found, starting auto-detection..."); + detectAvailableCameras(); + setHasDetectedOnMount(true); + } + }, [hasDetectedOnMount, hasLoadedSavedCameras, robustConfigs.length, loadSavedCameras]); + + const loadSavedCameraConfigs = async () => { + try { + console.log("🔄 Loading saved camera configurations..."); + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + if (data.status === "success" && data.camera_config && data.camera_config.cameras) { + const camerasFromBackend = data.camera_config.cameras; + console.log("📦 Raw cameras from backend:", camerasFromBackend); + + // Convert legacy backend format to robust format + const robustConfigs: RobustCameraConfig[] = Object.entries(camerasFromBackend).map(([name, config]: [string, any]) => { + const deviceId = config.device_id || `fallback_${name}`; + const hash = createCameraHash(deviceId, name); + + return { + hash, + device_id: deviceId, + user_name: name, + width: config.width || 640, + height: config.height || 480, + fps: config.fps || 30, + last_detected_index: config.index_or_path || 0, + last_seen: config.last_seen || new Date().toISOString(), + is_available: true, // Will be validated next + }; + }); + + console.log("🔄 Converted to robust configs:", robustConfigs); + + // Validate and clean up configs + const cleanedConfigs = await cleanupCameraConfigs(robustConfigs); + console.log("🧹 Cleaned configs:", cleanedConfigs); + + setRobustConfigs(cleanedConfigs); + syncToParent(cleanedConfigs); + + // Start previews for available cameras + const availableConfigs = cleanedConfigs.filter(config => config.is_available); + console.log("🎥 Starting previews for available cameras:", availableConfigs.map(c => c.user_name)); + + availableConfigs.forEach((config, index) => { + setTimeout(() => { + startCameraPreview(convertFromRobustConfig(config)); + }, index * 100); + }); + } else { + console.log("ℹ️ No saved camera configurations found"); + setRobustConfigs([]); + syncToParent([]); + } + } catch (error) { + console.error("Error loading saved camera configs:", error); + setRobustConfigs([]); + syncToParent([]); + } finally { + setHasLoadedSavedCameras(true); + } + }; + + const detectAvailableCameras = async () => { + console.log("🚀 Detecting available cameras..."); + setIsLoadingCameras(true); + try { + // Request camera permissions + try { + const tempStream = await navigator.mediaDevices.getUserMedia({ video: true }); + tempStream.getTracks().forEach((track) => track.stop()); + console.log("✅ Camera permission granted"); + } catch (permError) { + console.warn("⚠️ Camera permission denied:", permError); + } + + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter((device) => device.kind === "videoinput"); + + console.log("🔍 Detected video devices:", videoDevices.map((d, i) => ({ + index: i, + deviceId: d.deviceId, + label: d.label, + }))); + + const detected = videoDevices.map((device, index) => ({ + index, + deviceId: device.deviceId, + name: device.label || `Camera ${index + 1}`, + available: true, + })); + + setDetectedCameras(detected); + console.log("✅ Camera detection completed:", detected); + } catch (error) { + console.error("📡 Error detecting cameras:", error); + toast({ + title: "Camera Detection Failed", + description: "Could not detect available cameras. Please check permissions.", + variant: "destructive", + }); + } finally { + setIsLoadingCameras(false); + } + }; + + const startCameraPreview = async (cameraConfig: CameraConfig) => { + try { + console.log("🎥 Starting preview for:", cameraConfig.name, "device_id:", cameraConfig.device_id); + + const constraints: MediaStreamConstraints = { + video: { + deviceId: { exact: cameraConfig.device_id }, + width: { ideal: cameraConfig.width, min: 320, max: 1920 }, + height: { ideal: cameraConfig.height, min: 240, max: 1080 }, + frameRate: { ideal: cameraConfig.fps || 30, min: 10, max: 60 }, + }, + }; + + const stream = await navigator.mediaDevices.getUserMedia(constraints); + + // Verify stream is actually from the correct device + const videoTrack = stream.getVideoTracks()[0]; + if (videoTrack) { + const settings = videoTrack.getSettings(); + console.log("✅ Preview started with device:", settings.deviceId, "label:", videoTrack.label); + + if (settings.deviceId !== cameraConfig.device_id) { + console.warn("⚠️ Device ID mismatch! Expected:", cameraConfig.device_id, "Got:", settings.deviceId); + } + } + + setCameraStreams((prev) => new Map(prev.set(cameraConfig.id, stream))); + return stream; + } catch (error) { + console.error("Error starting camera preview:", error); + toast({ + title: "Camera Preview Failed", + description: `Could not start preview for ${cameraConfig.name}`, + variant: "destructive", + }); + return null; + } + }; + + const stopCameraPreview = (cameraId: string) => { + const stream = cameraStreams.get(cameraId); + if (stream) { + stream.getTracks().forEach((track) => track.stop()); + setCameraStreams((prev) => { + const newMap = new Map(prev); + newMap.delete(cameraId); + return newMap; + }); + } + }; + + const addCamera = async () => { + if (!selectedCameraIndex || !cameraName.trim()) { + toast({ + title: "Missing Information", + description: "Please select a camera and provide a name.", + variant: "destructive", + }); + return; + } + + const cameraIndex = parseInt(selectedCameraIndex); + const selectedCamera = detectedCameras.find((cam) => cam.index === cameraIndex); + + if (!selectedCamera) { + toast({ + title: "Invalid Camera", + description: "Selected camera is not available.", + variant: "destructive", + }); + return; + } + + // Check if camera with this device_id already exists + if (robustConfigs.some((config) => config.device_id === selectedCamera.deviceId)) { + toast({ + title: "Camera Already Added", + description: "This camera is already in the configuration.", + variant: "destructive", + }); + return; + } + + // Create robust camera config + const hash = createCameraHash(selectedCamera.deviceId, cameraName.trim()); + const newRobustConfig: RobustCameraConfig = { + hash, + device_id: selectedCamera.deviceId, + user_name: cameraName.trim(), + width: 640, + height: 480, + fps: 30, + last_detected_index: selectedCamera.index, + last_seen: new Date().toISOString(), + is_available: true, + }; + + console.log("🆕 Creating new robust camera config:", newRobustConfig); + + // Save to backend using hash as identifier + try { + const backendConfig = { + type: "browser", + device_id: newRobustConfig.device_id, + index_or_path: newRobustConfig.last_detected_index, + width: newRobustConfig.width, + height: newRobustConfig.height, + fps: newRobustConfig.fps, + last_seen: newRobustConfig.last_seen, + hash: newRobustConfig.hash, // Include hash for future validation + }; + + const saveResponse = await fetchWithHeaders(`${baseUrl}/cameras/config/update`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + camera_name: newRobustConfig.user_name, + camera_config: backendConfig + }), + }); + + const saveResult = await saveResponse.json(); + + if (saveResult.status === "success") { + console.log("✅ Camera configuration saved to backend"); + } else { + console.error("❌ Error saving camera config:", saveResult.message); + } + } catch (error) { + console.error("Error saving camera configuration:", error); + } + + // Update local state + const updatedConfigs = [...robustConfigs, newRobustConfig]; + setRobustConfigs(updatedConfigs); + syncToParent(updatedConfigs); + + // Start preview + await startCameraPreview(convertFromRobustConfig(newRobustConfig)); + + // Reset form + setSelectedCameraIndex(""); + setCameraName(""); + + toast({ + title: "Camera Added", + description: `${newRobustConfig.user_name} has been added successfully.`, + }); + }; + + const removeCamera = async (cameraHash: string) => { + const config = robustConfigs.find(c => c.hash === cameraHash); + if (!config) return; + + // Remove from backend + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/config/${encodeURIComponent(config.user_name)}`, { + method: "DELETE", + }); + + const result = await response.json(); + if (result.status === "success") { + console.log(`✅ Camera "${config.user_name}" removed from backend`); + } + } catch (error) { + console.error("Error removing camera from backend:", error); + } + + // Stop preview + stopCameraPreview(config.hash); + + // Update local state + const updatedConfigs = robustConfigs.filter(c => c.hash !== cameraHash); + setRobustConfigs(updatedConfigs); + syncToParent(updatedConfigs); + + toast({ + title: "Camera Removed", + description: `${config.user_name} has been removed.`, + }); + }; + + const updateCamera = (cameraHash: string, updates: Partial) => { + const updatedConfigs = robustConfigs.map((config) => + config.hash === cameraHash ? { ...config, ...updates } : config + ); + setRobustConfigs(updatedConfigs); + syncToParent(updatedConfigs); + }; + + // Clean up streams on unmount + useEffect(() => { + return () => { + cameraStreams.forEach((stream) => { + stream.getTracks().forEach((track) => track.stop()); + }); + }; + }, []); + + // Function to release all camera streams + const releaseAllCameraStreams = useCallback(() => { + console.log("🔓 Releasing all camera streams..."); + cameraStreams.forEach((stream, cameraId) => { + stream.getTracks().forEach((track) => track.stop()); + }); + setCameraStreams(new Map()); + }, [cameraStreams]); + + // Expose release function to parent + useEffect(() => { + if (releaseStreamsRef) { + releaseStreamsRef.current = releaseAllCameraStreams; + } + }, [releaseStreamsRef, releaseAllCameraStreams]); + + // Get available cameras for display + const availableConfigs = sortCamerasConsistently(robustConfigs.filter(config => config.is_available)); + const unavailableConfigs = robustConfigs.filter(config => !config.is_available); + + return ( +
+
+

+ Robust Camera Configuration +

+
+ +
+
+ + {/* Add Camera Section */} + {detectedCameras.length > 0 && ( +
+

Add Camera

+ +
+
+ + +
+ +
+ + setCameraName(e.target.value)} + placeholder="e.g., workspace_cam" + className="bg-gray-800 border-gray-700 text-white" + /> +
+ +
+ +
+
+
+ )} + + {/* Available Cameras */} + {availableConfigs.length > 0 && ( +
+

+ Available Cameras ({availableConfigs.length}) +

+ +
+ {availableConfigs.map((config) => ( + removeCamera(config.hash)} + onUpdate={(updates) => updateCamera(config.hash, updates)} + onStartPreview={() => startCameraPreview(convertFromRobustConfig(config))} + /> + ))} +
+
+ )} + + {/* Unavailable Cameras Warning */} + {unavailableConfigs.length > 0 && ( +
+
+
+ +

+ Unavailable Cameras ({unavailableConfigs.length}) +

+
+

+ These cameras were previously configured but are no longer detected. They may be disconnected or in use by another application. +

+
+ {unavailableConfigs.map((config) => ( +
+
+ {config.user_name} + + Last seen: {new Date(config.last_seen).toLocaleString()} + +
+ +
+ ))} +
+
+
+ )} + + {availableConfigs.length === 0 && unavailableConfigs.length === 0 && !isLoadingCameras && ( +
+ +

No cameras configured.

+

Click "Refresh Cameras" to detect available cameras and add them.

+
+ )} +
+ ); +}; + +// Robust Camera Preview Component +interface RobustCameraPreviewProps { + config: RobustCameraConfig; + stream?: MediaStream; + onRemove: () => void; + onUpdate: (updates: Partial) => void; + onStartPreview: () => void; +} + +const RobustCameraPreview: React.FC = ({ + config, + stream, + onRemove, + onUpdate, + onStartPreview, +}) => { + const videoRef = useRef(null); + const [isPreviewActive, setIsPreviewActive] = useState(false); + + useEffect(() => { + const video = videoRef.current; + if (video && stream) { + console.log(`🎥 Setting stream for camera: ${config.user_name} (${config.hash})`); + video.srcObject = stream; + + const playVideo = async () => { + try { + await video.play(); + setIsPreviewActive(true); + } catch (error) { + console.error(`Error playing video for ${config.user_name}:`, error); + video.muted = true; + try { + await video.play(); + setIsPreviewActive(true); + } catch (mutedError) { + setIsPreviewActive(false); + } + } + }; + + if (video.readyState >= 1) { + playVideo(); + } else { + video.addEventListener("loadedmetadata", playVideo, { once: true }); + } + } else { + setIsPreviewActive(false); + } + }, [stream, config.user_name, config.hash]); + + useEffect(() => { + if (!stream && !isPreviewActive) { + onStartPreview(); + } + }, [stream, isPreviewActive, onStartPreview]); + + return ( +
+ {/* Camera Preview */} +
+ {stream ? ( + <> +
+ + {/* Camera Info */} +
+
+
+ {getCameraDisplayName(config)} +
+ +
+ +
+
+ Resolution: +
+ + onUpdate({ width: parseInt(e.target.value) || 640 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="320" + max="1920" + /> + × + + onUpdate({ height: parseInt(e.target.value) || 480 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="240" + max="1080" + /> +
+
+
+ FPS: + + onUpdate({ fps: parseInt(e.target.value) || 30 }) + } + className="bg-gray-800 border-gray-700 text-white text-xs h-6 px-2 w-16" + min="10" + max="60" + /> +
+
+ +
+
Device ID: {config.device_id.substring(0, 16)}...
+
Hash: {config.hash}
+
Index: {config.last_detected_index}
+
+
+
+ ); +}; + +export default RobustCameraConfiguration; \ No newline at end of file diff --git a/src/components/webrtc/WebRTCCameraConfiguration.tsx b/src/components/webrtc/WebRTCCameraConfiguration.tsx new file mode 100644 index 0000000..eced054 --- /dev/null +++ b/src/components/webrtc/WebRTCCameraConfiguration.tsx @@ -0,0 +1,856 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { + Camera, + Plus, + X, + Video, + VideoOff, + RefreshCw, + Wifi, + WifiOff, + Activity +} from "lucide-react"; +import { useToast } from "@/hooks/use-toast"; +import { webRTCManager } from "@/utils/webrtc/WebRTCManager"; +import { UnifiedCameraSource, CameraQuality, CAMERA_CONSTRAINTS } from "@/types/webrtc"; +import { useApi } from "@/contexts/ApiContext"; + +// Legacy interface for compatibility +export interface CameraConfig { + id: string; + name: string; + type: string; + camera_index?: number; + device_id: string; + width: number; + height: number; + fps?: number; +} + +interface WebRTCCameraConfigurationProps { + cameras: CameraConfig[]; + onCamerasChange: (cameras: CameraConfig[]) => void; + releaseStreamsRef?: React.MutableRefObject<(() => void) | null>; + loadSavedCameras?: boolean; +} + +interface DetectedCamera { + index: number; + deviceId: string; + name: string; + available: boolean; +} + +const WebRTCCameraConfiguration: React.FC = ({ + cameras, + onCamerasChange, + releaseStreamsRef, + loadSavedCameras = true, +}) => { + const { baseUrl, fetchWithHeaders } = useApi(); + const { toast } = useToast(); + + // WebRTC state + const [webrtcSources, setWebrtcSources] = useState([]); + const [isConnectedToSignaling, setIsConnectedToSignaling] = useState(false); + const [signalingStats, setSignalingStats] = useState(null); + + // Camera detection state + const [detectedCameras, setDetectedCameras] = useState([]); + const [selectedCameraIndex, setSelectedCameraIndex] = useState(""); + const [cameraName, setCameraName] = useState(""); + // Removed selectedQuality - will be configurable per camera after adding + const [isLoadingCameras, setIsLoadingCameras] = useState(false); + + // WebRTC video elements refs + const videoRefs = useRef>(new Map()); + + // WebRTC event handlers (with useCallback to maintain references) + const handleCameraAdded = useCallback((source: UnifiedCameraSource) => { + console.log("📹 Camera added to WebRTC:", source.name); + setWebrtcSources(prev => [...prev.filter(s => s.id !== source.id), source]); + }, []); + + const handleCameraRemoved = useCallback((sourceId: string) => { + console.log("🗑️ Camera removed from WebRTC:", sourceId); + setWebrtcSources(prev => prev.filter(s => s.id !== sourceId)); + + // Remove video element ref + const videoElement = videoRefs.current.get(sourceId); + if (videoElement) { + videoElement.srcObject = null; + videoRefs.current.delete(sourceId); + } + }, []); + + const handleCameraConnected = useCallback((sourceId: string, stream: MediaStream) => { + console.log("✅ Camera connected:", sourceId); + + // Update source status + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'connected', stream } + : source + )); + + // Attach stream to video element + const videoElement = videoRefs.current.get(sourceId); + if (videoElement && stream) { + videoElement.srcObject = stream; + videoElement.play().catch(console.error); + } + }, []); + + const handleCameraDisconnected = useCallback((sourceId: string) => { + console.log("🔌 Camera disconnected:", sourceId); + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'disconnected', stream: undefined } + : source + )); + }, []); + + const handleCameraError = useCallback((sourceId: string, error: Error) => { + console.error("❌ Camera error:", sourceId, error); + toast({ + title: "Camera Error", + description: `Error with camera ${sourceId}: ${error.message}`, + variant: "destructive", + }); + }, [toast]); + + const handleCameraUpdated = useCallback((sourceId: string, updatedSource: UnifiedCameraSource) => { + console.log("🔄 Camera updated in WebRTC manager:", sourceId); + setWebrtcSources(prev => prev.map(s => + s.id === sourceId ? updatedSource : s + )); + }, []); + + const fetchSignalingStats = useCallback(async () => { + try { + const response = await fetchWithHeaders(`${baseUrl}/webrtc/status`); + const data = await response.json(); + setSignalingStats(data.stats); + } catch (error) { + console.error("Error fetching signaling stats:", error); + } + }, [baseUrl, fetchWithHeaders]); + + const handleWebRTCConnected = useCallback(() => { + console.log("✅ Connected to WebRTC signaling server"); + setIsConnectedToSignaling(true); + fetchSignalingStats(); + }, [fetchSignalingStats]); + + const handleWebRTCDisconnected = useCallback(() => { + console.log("🔌 Disconnected from WebRTC signaling server"); + setIsConnectedToSignaling(false); + }, []); + + // Initialize WebRTC Manager + useEffect(() => { + const initializeWebRTC = async () => { + try { + console.log("🚀 Initializing WebRTC Manager..."); + + // Configure WebRTC manager + webRTCManager.config.signalingUrl = `${baseUrl.replace('http', 'ws')}/ws/webrtc`; + + // Setup event listeners + webRTCManager.on('camera-added', handleCameraAdded); + webRTCManager.on('camera-removed', handleCameraRemoved); + webRTCManager.on('camera-connected', handleCameraConnected); + webRTCManager.on('camera-disconnected', handleCameraDisconnected); + webRTCManager.on('camera-error', handleCameraError); + webRTCManager.on('camera-updated', handleCameraUpdated); + webRTCManager.on('connected', handleWebRTCConnected); + webRTCManager.on('disconnected', handleWebRTCDisconnected); + + // Connect to signaling server if not already connected + if (!webRTCManager.isConnectedToSignaling()) { + console.log("🔗 Connecting to signaling server..."); + await webRTCManager.connect(); + } else { + console.log("✅ Already connected to signaling server"); + // Set the connected state even if already connected + setIsConnectedToSignaling(true); + fetchSignalingStats(); + } + + // Load saved cameras if requested and no cameras exist yet + if (loadSavedCameras && webRTCManager.getAllCameras().length === 0) { + console.log("📂 No existing cameras, loading saved configurations..."); + await loadSavedCameraConfigs(); + } else if (loadSavedCameras) { + console.log("📂 Cameras already exist, skipping saved camera load"); + // Load existing cameras into local state + const existingCameras = webRTCManager.getAllCameras(); + console.log("📹 Loading existing cameras into state:", existingCameras.length); + setWebrtcSources([...existingCameras]); + } + + } catch (error) { + console.error("❌ Failed to initialize WebRTC:", error); + toast({ + title: "WebRTC Initialization Failed", + description: "Could not connect to WebRTC signaling server.", + variant: "destructive", + }); + } + }; + + initializeWebRTC(); + + // Cleanup on unmount + return () => { + // Only remove our specific listeners using the function references + webRTCManager.off('camera-added', handleCameraAdded); + webRTCManager.off('camera-removed', handleCameraRemoved); + webRTCManager.off('camera-connected', handleCameraConnected); + webRTCManager.off('camera-disconnected', handleCameraDisconnected); + webRTCManager.off('camera-error', handleCameraError); + webRTCManager.off('camera-updated', handleCameraUpdated); + + // Also remove our connection listeners specifically + webRTCManager.off('connected', handleWebRTCConnected); + webRTCManager.off('disconnected', handleWebRTCDisconnected); + + // Don't disconnect WebRTC manager as teleoperation might need it + }; + }, [baseUrl, loadSavedCameras, handleCameraAdded, handleCameraRemoved, handleCameraConnected, handleCameraDisconnected, handleCameraError, handleCameraUpdated, handleWebRTCConnected, handleWebRTCDisconnected]); + + // Sync WebRTC sources to parent component + const syncToParent = useCallback(() => { + const legacyConfigs: CameraConfig[] = webrtcSources + .filter(source => source.status === 'connected') + .map(source => ({ + id: source.id, + name: source.name, + type: "webrtc", + device_id: source.deviceId || source.id, + width: source.width, + height: source.height, + fps: source.fps, + })); + + console.log("🔄 Syncing WebRTC cameras to parent:", legacyConfigs); + onCamerasChange(legacyConfigs); + }, [webrtcSources, onCamerasChange]); + + // Update parent when sources change + useEffect(() => { + syncToParent(); + }, [syncToParent]); + + // Camera detection + const detectAvailableCameras = async () => { + console.log("🔍 Detecting available cameras..."); + setIsLoadingCameras(true); + + try { + // First enumeration (might show limited devices) + let devices = await navigator.mediaDevices.enumerateDevices(); + console.log("📹 Initial device enumeration:", devices.filter(d => d.kind === "videoinput")); + + // Request camera permissions to unlock full device list + const tempStream = await navigator.mediaDevices.getUserMedia({ + video: { + facingMode: { ideal: "user" } // This helps detect iPhone Continuity cameras + } + }); + tempStream.getTracks().forEach(track => track.stop()); + + // Wait a bit for system to register all cameras + await new Promise(resolve => setTimeout(resolve, 500)); + + // Re-enumerate devices after permissions granted + devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + console.log("📹 Final device enumeration:", videoDevices); + + const detected = videoDevices.map((device, index) => ({ + index, + deviceId: device.deviceId, + name: device.label || `Camera ${index + 1}`, + available: true, + })); + + setDetectedCameras(detected); + console.log("✅ Detected cameras:", detected); + + } catch (error) { + console.error("❌ Camera detection failed:", error); + toast({ + title: "Camera Detection Failed", + description: "Could not detect cameras. Please check permissions.", + variant: "destructive", + }); + } finally { + setIsLoadingCameras(false); + } + }; + + // Add camera to WebRTC system + const addCamera = async () => { + if (!selectedCameraIndex || !cameraName.trim()) { + toast({ + title: "Missing Information", + description: "Please select a camera and provide a name.", + variant: "destructive", + }); + return; + } + + if (!isConnectedToSignaling) { + toast({ + title: "Not Connected", + description: "Not connected to WebRTC signaling server.", + variant: "destructive", + }); + return; + } + + const cameraIndex = parseInt(selectedCameraIndex); + const selectedCamera = detectedCameras.find(cam => cam.index === cameraIndex); + + if (!selectedCamera) { + toast({ + title: "Invalid Camera", + description: "Selected camera is not available.", + variant: "destructive", + }); + return; + } + + // Check if camera already added (check both local state and WebRTC manager) + const isDuplicateInLocal = webrtcSources.some(source => source.deviceId === selectedCamera.deviceId); + const isDuplicateInManager = webRTCManager.getAllCameras().some(source => source.deviceId === selectedCamera.deviceId); + + if (isDuplicateInLocal || isDuplicateInManager) { + toast({ + title: "Camera Already Added", + description: "This camera is already in the configuration.", + variant: "destructive", + }); + return; + } + + try { + console.log(`🆕 Adding WebRTC camera: ${cameraName} (${selectedCamera.deviceId})`); + + const sourceId = await webRTCManager.addLocalCamera( + selectedCamera.deviceId, + cameraName.trim(), + "medium" // Default quality, user can change it later + ); + + // Save to backend + await saveCameraToBackend(sourceId, cameraName, selectedCamera.deviceId, "medium"); + + // Reset form + setSelectedCameraIndex(""); + setCameraName(""); + + toast({ + title: "Camera Added", + description: `${cameraName} has been added successfully via WebRTC.`, + }); + + } catch (error) { + console.error("❌ Failed to add camera:", error); + toast({ + title: "Camera Add Failed", + description: `Could not add camera: ${error.message}`, + variant: "destructive", + }); + } + }; + + // Update camera settings (resolution, FPS) + const updateCameraSettings = async (sourceId: string, updates: Partial) => { + const source = webrtcSources.find(s => s.id === sourceId); + if (!source) { + console.error("❌ Source not found for update:", sourceId); + return; + } + + try { + console.log(`🔧 Updating camera settings for ${source.name}:`, updates); + + // Get new constraints based on updates + const newWidth = updates.width || source.width; + const newHeight = updates.height || source.height; + const newFps = updates.fps || source.fps; + + // Get new media stream with updated constraints + const newStream = await navigator.mediaDevices.getUserMedia({ + video: { + deviceId: { exact: source.deviceId }, + width: { ideal: newWidth }, + height: { ideal: newHeight }, + frameRate: { ideal: newFps } + } + }); + + // Stop old stream + if (source.stream) { + source.stream.getTracks().forEach(track => track.stop()); + } + + // Update the source in WebRTC manager + const updatedSource = { + ...source, + width: newWidth, + height: newHeight, + fps: newFps, + stream: newStream + }; + + // Update video element first + const videoElement = videoRefs.current.get(sourceId); + if (videoElement) { + videoElement.srcObject = newStream; + videoElement.play().catch(console.error); + } + + // Update in WebRTC manager (this will handle peer connection updates) + await webRTCManager.updateCamera(sourceId, updatedSource); + + // Update local state + setWebrtcSources(prev => prev.map(s => + s.id === sourceId ? updatedSource : s + )); + + // Save updated config to backend + const quality = newWidth >= 1280 ? "high" : newWidth >= 640 ? "medium" : "low"; + await saveCameraToBackend(sourceId, source.name, source.deviceId!, quality); + + toast({ + title: "Camera Updated", + description: `${source.name} settings updated to ${newWidth}x${newHeight} @ ${newFps}fps`, + }); + + } catch (error) { + console.error("❌ Failed to update camera settings:", error); + toast({ + title: "Update Failed", + description: `Could not update camera settings: ${error.message}`, + variant: "destructive", + }); + } + }; + + // Remove camera + const removeCamera = async (sourceId: string) => { + const source = webrtcSources.find(s => s.id === sourceId); + if (!source) return; + + try { + console.log(`🗑️ Removing WebRTC camera: ${source.name}`); + + webRTCManager.removeCamera(sourceId); + + // Remove from backend + await removeCameraFromBackend(source.name); + + toast({ + title: "Camera Removed", + description: `${source.name} has been removed.`, + }); + + } catch (error) { + console.error("❌ Failed to remove camera:", error); + toast({ + title: "Camera Remove Failed", + description: `Could not remove camera: ${error.message}`, + variant: "destructive", + }); + } + }; + + // Backend integration + const saveCameraToBackend = async (sourceId: string, name: string, deviceId: string, quality: CameraQuality) => { + try { + const constraints = CAMERA_CONSTRAINTS[quality]; + const backendConfig = { + type: "webrtc", + device_id: deviceId, + source_id: sourceId, + width: constraints.width, + height: constraints.height, + fps: constraints.fps, + quality: quality, + created_at: new Date().toISOString(), + }; + + const response = await fetchWithHeaders(`${baseUrl}/cameras/config/update`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + camera_name: name, + camera_config: backendConfig + }), + }); + + const result = await response.json(); + if (result.status !== "success") { + console.error("❌ Backend save failed:", result.message); + } else { + console.log("✅ Camera saved to backend:", name); + } + } catch (error) { + console.error("❌ Error saving to backend:", error); + } + }; + + const removeCameraFromBackend = async (name: string) => { + try { + const response = await fetchWithHeaders(`${baseUrl}/cameras/config/${encodeURIComponent(name)}`, { + method: "DELETE", + }); + + const result = await response.json(); + if (result.status === "success") { + console.log("✅ Camera removed from backend:", name); + } + } catch (error) { + console.error("❌ Error removing from backend:", error); + } + }; + + const loadSavedCameraConfigs = async () => { + try { + console.log("🔄 Loading saved WebRTC camera configurations..."); + const response = await fetchWithHeaders(`${baseUrl}/cameras/config`); + const data = await response.json(); + + if (data.status === "success" && data.camera_config && data.camera_config.cameras) { + const savedCameras = data.camera_config.cameras; + console.log("📦 Found saved cameras:", Object.keys(savedCameras)); + + // Load WebRTC cameras + for (const [name, config] of Object.entries(savedCameras)) { + if ((config as any).type === "webrtc" && (config as any).device_id && (config as any).source_id) { + try { + const quality = (config as any).quality || "medium"; + console.log(`🔄 Restoring WebRTC camera: ${name}`); + + await webRTCManager.addLocalCamera((config as any).device_id, name, quality); + } catch (error) { + console.error(`❌ Failed to restore camera ${name}:`, error); + } + } + } + } + } catch (error) { + console.error("❌ Error loading saved cameras:", error); + } + }; + + // Release all streams function + const releaseAllStreams = useCallback(() => { + console.log("🔓 Releasing all WebRTC camera streams..."); + webrtcSources.forEach(source => { + if (source.stream) { + source.stream.getTracks().forEach(track => track.stop()); + } + }); + webRTCManager.disconnect(); + }, [webrtcSources]); + + // Expose release function to parent + useEffect(() => { + if (releaseStreamsRef) { + releaseStreamsRef.current = releaseAllStreams; + } + }, [releaseStreamsRef, releaseAllStreams]); + + // Auto-detect cameras on mount + useEffect(() => { + detectAvailableCameras(); + }, []); + + return ( +
+
+

+ Camera Configuration +

+
+ +
+
+ + {/* Add Camera Section */} + {detectedCameras.length > 0 && isConnectedToSignaling && ( +
+

Add Camera

+ +
+
+ + +
+ +
+ + setCameraName(e.target.value)} + placeholder="e.g., workspace_cam" + className="bg-gray-800 border-gray-700 text-white" + /> +
+ +
+ +
+
+
+ )} + + {/* WebRTC Cameras */} + {webrtcSources.length > 0 && ( +
+

+ Cameras ({webrtcSources.length}) +

+ +
+ {webrtcSources.map((source) => ( + removeCamera(source.id)} + onUpdateSource={(updates) => { + updateCameraSettings(source.id, updates); + }} + videoRef={(el) => { + if (el) { + videoRefs.current.set(source.id, el); + } else { + videoRefs.current.delete(source.id); + } + }} + /> + ))} +
+
+ )} + + {webrtcSources.length === 0 && !isLoadingCameras && ( +
+ +

No cameras configured.

+

+ {isConnectedToSignaling + ? "Click \"Refresh Cameras\" to detect available cameras and add them." + : "Connecting..." + } +

+
+ )} +
+ ); +}; + +// WebRTC Camera Preview Component +interface WebRTCCameraPreviewProps { + source: UnifiedCameraSource; + onRemove: () => void; + onUpdateSource: (updates: Partial) => void; + videoRef: (el: HTMLVideoElement | null) => void; +} + +const WebRTCCameraPreview: React.FC = ({ + source, + onRemove, + onUpdateSource, + videoRef, +}) => { + const getStatusColor = () => { + switch (source.status) { + case 'connected': return 'text-green-400'; + case 'connecting': return 'text-yellow-400'; + case 'disconnected': return 'text-red-400'; + case 'error': return 'text-red-500'; + default: return 'text-gray-400'; + } + }; + + const getStatusIcon = () => { + switch (source.status) { + case 'connected': return ; + case 'connecting': return ; + case 'disconnected': return ; + case 'error': return ; + default: return ; + } + }; + + return ( +
+ {/* Camera Preview */} +
+ {source.stream && source.status === 'connected' ? ( + <> +
+ + {/* Camera Info */} +
+
+
+ {source.name} +
+ +
+ + + {/* Camera Controls */} +
+
+ + +
+ +
+ + +
+
+
+
+ ); +}; + +export default WebRTCCameraConfiguration; \ No newline at end of file diff --git a/src/components/webrtc/WebRTCVisualizerPanel.tsx b/src/components/webrtc/WebRTCVisualizerPanel.tsx new file mode 100644 index 0000000..9a9c01d --- /dev/null +++ b/src/components/webrtc/WebRTCVisualizerPanel.tsx @@ -0,0 +1,406 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; +import { Button } from "@/components/ui/button"; +import { ArrowLeft, VideoOff, Camera, Wifi, WifiOff, Activity } from "lucide-react"; +import { cn } from "@/lib/utils"; +import UrdfViewer from "../UrdfViewer"; +import UrdfProcessorInitializer from "../UrdfProcessorInitializer"; +import Logo from "@/components/Logo"; +import { webRTCManager } from "@/utils/webrtc/WebRTCManager"; +import { UnifiedCameraSource } from "@/types/webrtc"; +import { useApi } from "@/contexts/ApiContext"; + +interface WebRTCVisualizerPanelProps { + onGoBack: () => void; + className?: string; +} + +const WebRTCVisualizerPanel: React.FC = ({ + onGoBack, + className, +}) => { + const { baseUrl } = useApi(); + const [webrtcSources, setWebrtcSources] = useState([]); + const [isConnectedToSignaling, setIsConnectedToSignaling] = useState(false); + const [signalingStats, setSignalingStats] = useState(null); + + // Video element refs for WebRTC streams + const videoRefs = useRef>(new Map()); + + // WebRTC event handlers (with useCallback to maintain references) + const handleCameraAdded = useCallback((source: UnifiedCameraSource) => { + console.log("📹 Camera added in teleoperation:", source.name); + setWebrtcSources(prev => { + // Remove any existing source with same ID or deviceId to prevent duplicates + const filtered = prev.filter(s => s.id !== source.id && s.deviceId !== source.deviceId); + return [...filtered, source]; + }); + }, []); + + const handleCameraRemoved = useCallback((sourceId: string) => { + console.log("🗑️ Camera removed in teleoperation:", sourceId); + setWebrtcSources(prev => prev.filter(s => s.id !== sourceId)); + + // Remove video element ref + const videoElement = videoRefs.current.get(sourceId); + if (videoElement) { + videoElement.srcObject = null; + videoRefs.current.delete(sourceId); + } + }, []); + + const handleCameraConnected = useCallback((sourceId: string, stream: MediaStream) => { + console.log("✅ Camera connected in teleoperation:", sourceId); + + // Update source status + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'connected', stream } + : source + )); + + // Attach stream to video element + setTimeout(() => { + const videoElement = videoRefs.current.get(sourceId); + if (videoElement && stream) { + console.log(`🔗 Attaching stream to video element for ${sourceId}`); + videoElement.srcObject = stream; + videoElement.play().catch(console.error); + } + }, 100); + }, []); + + const handleCameraDisconnected = useCallback((sourceId: string) => { + console.log("🔌 Camera disconnected in teleoperation:", sourceId); + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'disconnected', stream: undefined } + : source + )); + }, []); + + // Initialize WebRTC connection and load cameras + useEffect(() => { + console.log("🔄 WebRTCVisualizerPanel useEffect triggered"); + + const initializeWebRTC = async () => { + try { + console.log("🚀 Initializing WebRTC for teleoperation..."); + console.log("📊 Current WebRTC manager state:", { + isConnected: webRTCManager.isConnectedToSignaling(), + cameraCount: webRTCManager.getAllCameras().length, + cameras: webRTCManager.getAllCameras().map(c => ({ id: c.id, name: c.name, status: c.status })), + signalingUrl: webRTCManager.config.signalingUrl + }); + + // Always load existing cameras first, regardless of connection status + const loadExistingCameras = () => { + const existingSources = webRTCManager.getAllCameras(); + console.log("📹 Loading existing WebRTC cameras for teleoperation:", existingSources.length, existingSources.map(s => ({ name: s.name, status: s.status, hasStream: !!s.stream }))); + + if (existingSources.length > 0) { + // Deduplicate by deviceId and ID to prevent duplicates + const uniqueSources = existingSources.reduce((acc, source) => { + const existingIndex = acc.findIndex(s => s.id === source.id || s.deviceId === source.deviceId); + if (existingIndex === -1) { + acc.push(source); + } else { + // Keep the most recent one (higher timestamp or connected status) + if (source.status === 'connected' && acc[existingIndex].status !== 'connected') { + acc[existingIndex] = source; + } + } + return acc; + }, [] as UnifiedCameraSource[]); + + console.log("📹 Deduplicated sources:", uniqueSources.length); + setWebrtcSources(uniqueSources); + + // Setup video streams for existing cameras (use deduplicated sources) + uniqueSources.forEach(source => { + console.log(`🎥 Setting up stream for existing camera: ${source.name}, status: ${source.status}, hasStream: ${!!source.stream}`); + if (source.stream && source.status === 'connected') { + // Use a slightly longer timeout to ensure DOM is ready + setTimeout(() => { + const videoElement = videoRefs.current.get(source.id); + if (videoElement && source.stream) { + console.log(`🔗 Attaching stream to video for ${source.name}`); + videoElement.srcObject = source.stream; + videoElement.play().catch(console.error); + } else { + console.log(`⚠️ Video element or stream not available for ${source.name}`, { + hasElement: !!videoElement, + hasStream: !!source.stream + }); + } + }, 200); + } + }); + } else { + console.log("📹 No existing cameras found in WebRTC manager"); + } + }; + + // Setup event listeners + webRTCManager.on('camera-added', handleCameraAdded); + webRTCManager.on('camera-removed', handleCameraRemoved); + webRTCManager.on('camera-connected', handleCameraConnected); + webRTCManager.on('camera-disconnected', handleCameraDisconnected); + webRTCManager.on('connected', () => { + console.log("✅ Connected to WebRTC signaling server in teleoperation"); + setIsConnectedToSignaling(true); + // Reload cameras when we connect + loadExistingCameras(); + fetchSignalingStats(); + }); + webRTCManager.on('disconnected', () => { + console.log("🔌 Disconnected from WebRTC signaling server in teleoperation"); + setIsConnectedToSignaling(false); + }); + + // Load cameras immediately + loadExistingCameras(); + + // Check if already connected + if (webRTCManager.isConnectedToSignaling()) { + console.log("✅ WebRTC already connected"); + setIsConnectedToSignaling(true); + fetchSignalingStats(); + } else { + console.log("🔄 WebRTC not connected, attempting connection..."); + // Configure and connect if not already connected + if (!webRTCManager.config.signalingUrl.includes(baseUrl)) { + webRTCManager.config.signalingUrl = `${baseUrl.replace('http', 'ws')}/ws/webrtc`; + } + await webRTCManager.connect(); + } + + } catch (error) { + console.error("❌ Failed to initialize WebRTC for teleoperation:", error); + } + }; + + initializeWebRTC(); + + // Cleanup event listeners on unmount + return () => { + // Don't remove ALL listeners, just remove the specific ones we added + // to avoid breaking other components that might be listening + webRTCManager.off('camera-added', handleCameraAdded); + webRTCManager.off('camera-removed', handleCameraRemoved); + webRTCManager.off('camera-connected', handleCameraConnected); + webRTCManager.off('camera-disconnected', handleCameraDisconnected); + }; + }, [baseUrl, handleCameraAdded, handleCameraRemoved, handleCameraConnected, handleCameraDisconnected]); + + // Additional effect to ensure cameras are always loaded on mount + useEffect(() => { + console.log("🔄 Additional effect: checking for cameras on component mount"); + const existingSources = webRTCManager.getAllCameras(); + console.log("📊 Found cameras in additional effect:", existingSources.length, existingSources.map(s => s.name)); + + if (existingSources.length > 0) { + setWebrtcSources([...existingSources]); + console.log("✅ Set cameras in additional effect"); + } + }, []); // Empty dependency array - runs only on mount + + const fetchSignalingStats = async () => { + try { + const response = await fetch(`${baseUrl}/webrtc/status`); + const data = await response.json(); + setSignalingStats(data.stats); + } catch (error) { + console.error("Error fetching signaling stats:", error); + } + }; + + // Get available cameras (only connected ones for display) + const getAvailableCameras = () => { + return webrtcSources.filter(source => source.status === 'connected'); + }; + + // Get responsive layout classes based on number of cameras + const getCameraLayoutClasses = () => { + const cameraCount = getAvailableCameras().length; + + if (cameraCount === 0) return "lg:w-80"; + if (cameraCount === 1) return "lg:w-80"; // 1 camera: single column, full width + if (cameraCount === 2) return "lg:w-80"; // 2 cameras: single column, stacked + if (cameraCount === 3) return "lg:w-80"; // 3 cameras: single column, full height + if (cameraCount === 4) return "lg:w-96"; // 4 cameras: 2x2 grid + if (cameraCount <= 6) return "lg:w-[32rem]"; // 5-6 cameras: 3x2 grid + + return "lg:w-[36rem]"; // 7+ cameras: wider grid + }; + + // Get grid classes for camera layout + const getCameraGridClasses = () => { + const cameraCount = getAvailableCameras().length; + + if (cameraCount === 0) return ""; + if (cameraCount === 1) return "flex flex-col gap-3"; // 1 camera: single column + if (cameraCount === 2) return "flex flex-col gap-3"; // 2 cameras: stacked + if (cameraCount === 3) return "flex flex-col gap-3"; // 3 cameras: single column + if (cameraCount === 4) return "grid grid-cols-2 gap-3"; // 4 cameras: 2x2 grid + if (cameraCount <= 6) return "grid grid-cols-3 gap-2"; // 5-6 cameras: 3x2 grid + + return "grid grid-cols-3 gap-2"; // 7+ cameras: 3 columns + }; + + const availableCameras = getAvailableCameras(); + + return ( +
+
+
+ + +
+

Teleoperation

+ +
+
+ + +
+
+ +
+ {availableCameras.length > 0 ? ( +
+ {availableCameras.map((source) => ( + { + if (el) { + videoRefs.current.set(source.id, el); + // If stream is already available, attach it + if (source.stream) { + el.srcObject = source.stream; + el.play().catch(console.error); + } + } else { + videoRefs.current.delete(source.id); + } + }} + /> + ))} +
+ ) : ( +
+ + + No cameras available + + + Configure cameras in settings + +
+ )} +
+
+ ); +}; + +// WebRTC Camera Display Component +interface WebRTCCameraDisplayProps { + source: UnifiedCameraSource; + videoRef: (el: HTMLVideoElement | null) => void; +} + +const WebRTCCameraDisplay: React.FC = ({ + source, + videoRef, +}) => { + const [isPlaying, setIsPlaying] = useState(false); + + const getStatusColor = () => { + switch (source.status) { + case 'connected': return 'bg-green-500'; + case 'connecting': return 'bg-yellow-500 animate-pulse'; + case 'disconnected': return 'bg-red-500'; + case 'error': return 'bg-red-600'; + default: return 'bg-gray-500'; + } + }; + + const handleVideoPlay = () => { + setIsPlaying(true); + }; + + const handleVideoError = () => { + setIsPlaying(false); + console.error(`Video playback error for camera ${source.name}`); + }; + + return ( +
+
+
+ {source.stream && source.status === 'connected' ? ( + <> +
+ +
+ + {source.name} {/* Consistent user-given name */} + + + {source.id.substring(0, 8)}... + + + {source.width}x{source.height} @ {source.fps}fps + +
+
+
+ ); +}; + +export default WebRTCVisualizerPanel; \ No newline at end of file diff --git a/src/pages/Landing.tsx b/src/pages/Landing.tsx index ee19bd5..20c51b7 100644 --- a/src/pages/Landing.tsx +++ b/src/pages/Landing.tsx @@ -12,6 +12,7 @@ import NgrokConfigModal from "@/components/landing/NgrokConfigModal"; import { Action } from "@/components/landing/types"; import UsageInstructionsModal from "@/components/landing/UsageInstructionsModal"; import DirectFollowerModal from "@/components/landing/DirectFollowerModal"; +import { CameraConfig } from "@/components/recording/CameraConfiguration"; import { useApi } from "@/contexts/ApiContext"; const Landing = () => { @@ -52,6 +53,10 @@ const Landing = () => { ); const [directFollowerConfig, setDirectFollowerConfig] = useState(""); + // Camera state for all modals + const [teleoperationCameras, setTeleoperationCameras] = useState([]); + const [recordingCameras, setRecordingCameras] = useState([]); + const navigate = useNavigate(); const { toast } = useToast(); @@ -291,11 +296,18 @@ const Landing = () => { handler: handleTeleoperationClick, color: "bg-yellow-500 hover:bg-yellow-600", }, + { + title: "Record Dataset", + description: "Record episodes for training data.", + handler: handleRecordingClick, + color: "bg-red-500 hover:bg-red-600", + }, { title: "Direct Follower Control", - description: "Train a model on your datasets.", + description: "Control robot arm with mouse movements.", handler: handleDirectFollowerClick, color: "bg-blue-500 hover:bg-blue-600", + isWorkInProgress: true, }, { title: "Calibration", @@ -304,12 +316,6 @@ const Landing = () => { color: "bg-indigo-500 hover:bg-indigo-600", isWorkInProgress: true, }, - { - title: "Record Dataset", - description: "Record episodes for training data.", - handler: handleRecordingClick, - color: "bg-red-500 hover:bg-red-600", - }, { title: "Training", description: "Train a model on your datasets.", @@ -367,6 +373,8 @@ const Landing = () => { setFollowerConfig={setFollowerConfig} leaderConfigs={leaderConfigs} followerConfigs={followerConfigs} + cameras={teleoperationCameras} + setCameras={setTeleoperationCameras} isLoadingConfigs={isLoadingConfigs} onStart={handleStartTeleoperation} /> @@ -390,6 +398,8 @@ const Landing = () => { setSingleTask={setSingleTask} numEpisodes={numEpisodes} setNumEpisodes={setNumEpisodes} + cameras={recordingCameras} + setCameras={setRecordingCameras} isLoadingConfigs={isLoadingConfigs} onStart={handleStartRecording} /> diff --git a/src/pages/Teleoperation.tsx b/src/pages/Teleoperation.tsx index ba7bf20..25e9243 100644 --- a/src/pages/Teleoperation.tsx +++ b/src/pages/Teleoperation.tsx @@ -1,6 +1,6 @@ import React from "react"; import { useNavigate } from "react-router-dom"; -import VisualizerPanel from "@/components/control/VisualizerPanel"; +import WebRTCVisualizerPanel from "@/components/webrtc/WebRTCVisualizerPanel"; import { useToast } from "@/hooks/use-toast"; import { useApi } from "@/contexts/ApiContext"; @@ -53,9 +53,9 @@ const TeleoperationPage = () => { }; return ( -
-
- +
+
+
); diff --git a/src/types/webrtc.ts b/src/types/webrtc.ts new file mode 100644 index 0000000..fb4b461 --- /dev/null +++ b/src/types/webrtc.ts @@ -0,0 +1,117 @@ +// WebRTC unified camera system types + +export interface WebRTCConfiguration { + iceServers: RTCIceServer[]; + iceTransportPolicy?: RTCIceTransportPolicy; + bundlePolicy?: RTCBundlePolicy; +} + +export interface UnifiedCameraSource { + id: string; // device_id for local, peer_id for remote + type: 'local' | 'remote'; + name: string; // user-given name + stream?: MediaStream; + peerConnection?: RTCPeerConnection; + status: 'connecting' | 'connected' | 'disconnected' | 'error'; + + // Local camera specific + deviceId?: string; // Browser device ID + + // Remote camera specific + peerId?: string; // Remote peer identifier + + // Stream configuration + width: number; + height: number; + fps: number; + + // Metadata + lastSeen: string; + errorCount: number; + qualityStats?: RTCStatsReport; +} + +export interface SignalingMessage { + type: 'offer' | 'answer' | 'ice-candidate' | 'camera-list' | 'camera-request' | 'error'; + sourceId: string; // Camera source ID + targetId?: string; // Target peer (for remote) + payload: any; // Message-specific data + timestamp: number; +} + +export interface CameraOfferMessage extends SignalingMessage { + type: 'offer'; + payload: { + sdp: RTCSessionDescriptionInit; + cameraInfo: { + name: string; + width: number; + height: number; + fps: number; + }; + }; +} + +export interface CameraAnswerMessage extends SignalingMessage { + type: 'answer'; + payload: { + sdp: RTCSessionDescriptionInit; + }; +} + +export interface ICECandidateMessage extends SignalingMessage { + type: 'ice-candidate'; + payload: { + candidate: RTCIceCandidateInit; + }; +} + +export interface CameraListMessage extends SignalingMessage { + type: 'camera-list'; + payload: { + cameras: Array<{ + id: string; + name: string; + type: 'local' | 'remote'; + available: boolean; + }>; + }; +} + +export interface WebRTCManagerEvents { + 'camera-added': (source: UnifiedCameraSource) => void; + 'camera-removed': (sourceId: string) => void; + 'camera-connected': (sourceId: string, stream: MediaStream) => void; + 'camera-disconnected': (sourceId: string) => void; + 'camera-error': (sourceId: string, error: Error) => void; + 'stats-updated': (sourceId: string, stats: RTCStatsReport) => void; +} + +export interface WebRTCManagerConfig { + signalingUrl: string; // WebSocket signaling server + rtcConfiguration: WebRTCConfiguration; + autoReconnect: boolean; + statsInterval: number; // Stats collection interval (ms) + maxReconnectAttempts: number; +} + +// Default WebRTC configuration +export const DEFAULT_RTC_CONFIG: WebRTCConfiguration = { + iceServers: [ + { urls: 'stun:stun.l.google.com:19302' }, + { urls: 'stun:stun1.l.google.com:19302' }, + // Add TURN servers here if needed for production + ], + iceTransportPolicy: 'all', + bundlePolicy: 'balanced' +}; + +// Camera constraints for different quality levels +export const CAMERA_CONSTRAINTS = { + low: { width: 320, height: 240, fps: 15 }, + medium: { width: 640, height: 480, fps: 30 }, + high: { width: 1280, height: 720, fps: 30 }, + ultra: { width: 1920, height: 1080, fps: 30 } +} as const; + +export type CameraQuality = keyof typeof CAMERA_CONSTRAINTS; \ No newline at end of file diff --git a/src/utils/cameraUtils.ts b/src/utils/cameraUtils.ts new file mode 100644 index 0000000..6294139 --- /dev/null +++ b/src/utils/cameraUtils.ts @@ -0,0 +1,149 @@ +// Robust camera management utilities +import { CameraConfig } from "@/components/recording/CameraConfiguration"; + +export interface RobustCameraConfig { + // Stable identifiers + hash: string; // Unique hash based on device_id + name + device_id: string; // Stable hardware identifier + user_name: string; // User-given name + + // Configuration + width: number; + height: number; + fps: number; + + // Metadata (auto-updated) + last_detected_index?: number; // Last detected index (can change) + last_seen: string; // Timestamp of last detection + is_available: boolean; // Currently available +} + +/** + * Create a stable hash from device_id and user name + * This hash will be the primary identifier for camera configs + */ +export const createCameraHash = (deviceId: string, userName: string): string => { + // Create a stable hash using device_id + user_name + const combined = `${deviceId}|${userName}`; + + // Simple but stable hash implementation + let hash = 0; + for (let i = 0; i < combined.length; i++) { + const char = combined.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + + // Convert to base36 for shorter, readable string + return Math.abs(hash).toString(36).padStart(8, '0'); +}; + +/** + * Convert legacy CameraConfig to robust format + */ +export const convertToRobustConfig = (legacyConfig: CameraConfig): RobustCameraConfig => { + const hash = createCameraHash(legacyConfig.device_id, legacyConfig.name); + + return { + hash, + device_id: legacyConfig.device_id, + user_name: legacyConfig.name, + width: legacyConfig.width, + height: legacyConfig.height, + fps: legacyConfig.fps || 30, + last_detected_index: legacyConfig.camera_index, + last_seen: new Date().toISOString(), + is_available: true, + }; +}; + +/** + * Convert robust config back to legacy format for compatibility + */ +export const convertFromRobustConfig = (robustConfig: RobustCameraConfig): CameraConfig => { + return { + id: robustConfig.hash, + name: robustConfig.user_name, + type: "browser", + camera_index: robustConfig.last_detected_index || 0, + device_id: robustConfig.device_id, + width: robustConfig.width, + height: robustConfig.height, + fps: robustConfig.fps, + }; +}; + +/** + * Validate if a camera config is still valid by checking device availability + */ +export const validateCameraConfig = async (config: RobustCameraConfig): Promise => { + try { + // Check if device_id still exists in browser + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + return videoDevices.some(device => device.deviceId === config.device_id); + } catch (error) { + console.error("Error validating camera config:", error); + return false; + } +}; + +/** + * Find current device index for a device_id + */ +export const findCurrentDeviceIndex = async (deviceId: string): Promise => { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + const index = videoDevices.findIndex(device => device.deviceId === deviceId); + return index; + } catch (error) { + console.error("Error finding device index:", error); + return -1; + } +}; + +/** + * Clean up camera configs - mark unavailable ones + */ +export const cleanupCameraConfigs = async (configs: RobustCameraConfig[]): Promise => { + const cleanedConfigs = []; + + for (const config of configs) { + const isValid = await validateCameraConfig(config); + const currentIndex = await findCurrentDeviceIndex(config.device_id); + + cleanedConfigs.push({ + ...config, + is_available: isValid, + last_detected_index: currentIndex !== -1 ? currentIndex : config.last_detected_index, + last_seen: isValid ? new Date().toISOString() : config.last_seen, + }); + } + + return cleanedConfigs; +}; + +/** + * Get display name for camera (consistent between preview and streaming) + */ +export const getCameraDisplayName = (config: RobustCameraConfig): string => { + return config.user_name; +}; + +/** + * Get camera streaming identifier (consistent between preview and streaming) + */ +export const getCameraStreamingId = (config: RobustCameraConfig): string => { + // Use hash as consistent identifier for both preview and streaming + return config.hash; +}; + +/** + * Sort cameras consistently (by user name, alphabetical) + */ +export const sortCamerasConsistently = (cameras: RobustCameraConfig[]): RobustCameraConfig[] => { + return cameras.sort((a, b) => a.user_name.localeCompare(b.user_name)); +}; \ No newline at end of file diff --git a/src/utils/webrtc/BrowserEventEmitter.ts b/src/utils/webrtc/BrowserEventEmitter.ts new file mode 100644 index 0000000..70aa326 --- /dev/null +++ b/src/utils/webrtc/BrowserEventEmitter.ts @@ -0,0 +1,61 @@ +// Browser-compatible EventEmitter implementation + +export class BrowserEventEmitter { + private events: Map = new Map(); + + on(event: string, listener: Function): this { + if (!this.events.has(event)) { + this.events.set(event, []); + } + this.events.get(event)!.push(listener); + return this; + } + + off(event: string, listener: Function): this { + const listeners = this.events.get(event); + if (listeners) { + const index = listeners.indexOf(listener); + if (index !== -1) { + listeners.splice(index, 1); + } + } + return this; + } + + emit(event: string, ...args: any[]): boolean { + const listeners = this.events.get(event); + if (listeners && listeners.length > 0) { + listeners.forEach(listener => { + try { + listener(...args); + } catch (error) { + console.error(`Error in event listener for ${event}:`, error); + } + }); + return true; + } + return false; + } + + removeAllListeners(event?: string | string[]): this { + if (event) { + if (Array.isArray(event)) { + event.forEach(e => this.events.delete(e)); + } else { + this.events.delete(event); + } + } else { + this.events.clear(); + } + return this; + } + + listenerCount(event: string): number { + const listeners = this.events.get(event); + return listeners ? listeners.length : 0; + } + + eventNames(): string[] { + return Array.from(this.events.keys()); + } +} \ No newline at end of file diff --git a/src/utils/webrtc/WebRTCManager.ts b/src/utils/webrtc/WebRTCManager.ts new file mode 100644 index 0000000..c8da5ff --- /dev/null +++ b/src/utils/webrtc/WebRTCManager.ts @@ -0,0 +1,395 @@ +// Unified WebRTC Camera Manager +import { BrowserEventEmitter } from './BrowserEventEmitter'; +import { + UnifiedCameraSource, + SignalingMessage, + WebRTCManagerConfig, + WebRTCManagerEvents, + DEFAULT_RTC_CONFIG, + CAMERA_CONSTRAINTS, + CameraQuality +} from '@/types/webrtc'; + +export class WebRTCManager extends BrowserEventEmitter { + public config: WebRTCManagerConfig; + private signalingSocket: WebSocket | null = null; + private sources = new Map(); + private isConnected = false; + private reconnectAttempts = 0; + private statsIntervals = new Map(); + + constructor(config: Partial) { + super(); + this.config = { + signalingUrl: config.signalingUrl || 'ws://localhost:8000/ws/webrtc', + rtcConfiguration: config.rtcConfiguration || DEFAULT_RTC_CONFIG, + autoReconnect: config.autoReconnect ?? true, + statsInterval: config.statsInterval || 5000, + maxReconnectAttempts: config.maxReconnectAttempts || 5, + }; + } + + // ==================== Connection Management ==================== + + async connect(): Promise { + try { + console.log('🔗 Connecting to WebRTC signaling server:', this.config.signalingUrl); + + this.signalingSocket = new WebSocket(this.config.signalingUrl); + + this.signalingSocket.onopen = () => { + console.log('✅ WebRTC signaling connected'); + this.isConnected = true; + this.reconnectAttempts = 0; + this.emit('connected'); + }; + + this.signalingSocket.onmessage = (event) => { + this.handleSignalingMessage(JSON.parse(event.data)); + }; + + this.signalingSocket.onclose = () => { + console.log('🔌 WebRTC signaling disconnected'); + this.isConnected = false; + this.emit('disconnected'); + + if (this.config.autoReconnect && this.reconnectAttempts < this.config.maxReconnectAttempts) { + this.reconnectAttempts++; + console.log(`🔄 Reconnecting attempt ${this.reconnectAttempts}/${this.config.maxReconnectAttempts}`); + setTimeout(() => this.connect(), 2000 * this.reconnectAttempts); + } + }; + + this.signalingSocket.onerror = (error) => { + console.error('❌ WebRTC signaling error:', error); + this.emit('error', error); + }; + + } catch (error) { + console.error('Failed to connect to signaling server:', error); + throw error; + } + } + + disconnect(): void { + if (this.signalingSocket) { + this.signalingSocket.close(); + this.signalingSocket = null; + } + + // Close all peer connections + this.sources.forEach(source => { + this.removeCamera(source.id); + }); + + this.isConnected = false; + } + + // ==================== Local Camera Management ==================== + + async addLocalCamera(deviceId: string, name: string, quality: CameraQuality = 'medium'): Promise { + // Check if camera with this deviceId already exists + const existingSource = Array.from(this.sources.values()).find(source => source.deviceId === deviceId); + if (existingSource) { + console.log(`⚠️ Camera with deviceId ${deviceId} already exists: ${existingSource.name}`); + return existingSource.id; + } + + const sourceId = `local_${deviceId}_${Date.now()}`; + + console.log(`🎥 Adding local camera: ${name} (${deviceId})`); + + try { + // Get camera stream + const constraints = CAMERA_CONSTRAINTS[quality]; + const stream = await navigator.mediaDevices.getUserMedia({ + video: { + deviceId: { exact: deviceId }, + width: { ideal: constraints.width }, + height: { ideal: constraints.height }, + frameRate: { ideal: constraints.fps } + } + }); + + // Create peer connection for local streaming + const peerConnection = new RTCPeerConnection(this.config.rtcConfiguration); + + // Add stream to peer connection + stream.getTracks().forEach(track => { + peerConnection.addTrack(track, stream); + }); + + // Setup peer connection event handlers + this.setupPeerConnectionHandlers(peerConnection, sourceId); + + // Create camera source + const source: UnifiedCameraSource = { + id: sourceId, + type: 'local', + name, + deviceId, + stream, + peerConnection, + status: 'connected', + width: constraints.width, + height: constraints.height, + fps: constraints.fps, + lastSeen: new Date().toISOString(), + errorCount: 0 + }; + + this.sources.set(sourceId, source); + this.startStatsCollection(sourceId); + + console.log(`✅ Local camera added: ${name} (${sourceId})`); + this.emit('camera-added', source); + this.emit('camera-connected', sourceId, stream); + + return sourceId; + + } catch (error) { + console.error(`❌ Failed to add local camera ${name}:`, error); + throw error; + } + } + + // ==================== Remote Camera Management (Future) ==================== + + async addRemoteCamera(peerId: string, name: string): Promise { + const sourceId = `remote_${peerId}_${Date.now()}`; + + console.log(`📡 Adding remote camera: ${name} (${peerId})`); + + // This will be implemented for remote cameras + // For now, return placeholder + throw new Error('Remote cameras not implemented yet'); + } + + // ==================== Camera Operations ==================== + + async updateCamera(sourceId: string, updates: Partial): Promise { + const source = this.sources.get(sourceId); + if (!source) { + throw new Error(`Camera ${sourceId} not found for update`); + } + + console.log(`🔧 Updating camera: ${source.name} (${sourceId})`, updates); + + // Create updated source + const updatedSource = { ...source, ...updates }; + + // If updating stream-related properties, update the peer connection + if (updates.stream && source.peerConnection) { + const sender = source.peerConnection.getSenders().find(s => + s.track && s.track.kind === 'video' + ); + if (sender && updates.stream.getVideoTracks()[0]) { + await sender.replaceTrack(updates.stream.getVideoTracks()[0]); + } + } + + // Update in sources map + this.sources.set(sourceId, updatedSource); + + // Emit update event + this.emit('camera-updated', sourceId, updatedSource); + } + + removeCamera(sourceId: string): void { + const source = this.sources.get(sourceId); + if (!source) { + console.warn(`Camera ${sourceId} not found for removal`); + return; + } + + console.log(`🗑️ Removing camera: ${source.name} (${sourceId})`); + + // Stop stats collection + this.stopStatsCollection(sourceId); + + // Close peer connection + if (source.peerConnection) { + source.peerConnection.close(); + } + + // Stop media stream + if (source.stream) { + source.stream.getTracks().forEach(track => track.stop()); + } + + this.sources.delete(sourceId); + this.emit('camera-removed', sourceId); + } + + getCamera(sourceId: string): UnifiedCameraSource | undefined { + return this.sources.get(sourceId); + } + + getAllCameras(): UnifiedCameraSource[] { + return Array.from(this.sources.values()); + } + + getLocalCameras(): UnifiedCameraSource[] { + return this.getAllCameras().filter(source => source.type === 'local'); + } + + getRemoteCameras(): UnifiedCameraSource[] { + return this.getAllCameras().filter(source => source.type === 'remote'); + } + + // ==================== Streaming ==================== + + getStream(sourceId: string): MediaStream | null { + const source = this.sources.get(sourceId); + return source?.stream || null; + } + + async createStreamingOffer(sourceId: string): Promise { + const source = this.sources.get(sourceId); + if (!source || !source.peerConnection) { + throw new Error(`Camera ${sourceId} not found or not connected`); + } + + const offer = await source.peerConnection.createOffer(); + await source.peerConnection.setLocalDescription(offer); + + return offer; + } + + async handleStreamingAnswer(sourceId: string, answer: RTCSessionDescriptionInit): Promise { + const source = this.sources.get(sourceId); + if (!source || !source.peerConnection) { + throw new Error(`Camera ${sourceId} not found or not connected`); + } + + await source.peerConnection.setRemoteDescription(answer); + } + + async addIceCandidate(sourceId: string, candidate: RTCIceCandidateInit): Promise { + const source = this.sources.get(sourceId); + if (!source || !source.peerConnection) { + console.warn(`Cannot add ICE candidate: Camera ${sourceId} not found`); + return; + } + + await source.peerConnection.addIceCandidate(candidate); + } + + // ==================== Private Methods ==================== + + private setupPeerConnectionHandlers(peerConnection: RTCPeerConnection, sourceId: string): void { + peerConnection.onicecandidate = (event) => { + if (event.candidate) { + this.sendSignalingMessage({ + type: 'ice-candidate', + sourceId, + payload: { candidate: event.candidate }, + timestamp: Date.now() + }); + } + }; + + peerConnection.onconnectionstatechange = () => { + const source = this.sources.get(sourceId); + if (source) { + console.log(`🔗 Connection state changed for ${sourceId}:`, peerConnection.connectionState); + + switch (peerConnection.connectionState) { + case 'connected': + source.status = 'connected'; + this.emit('camera-connected', sourceId, source.stream!); + break; + case 'disconnected': + case 'failed': + source.status = 'disconnected'; + this.emit('camera-disconnected', sourceId); + break; + case 'connecting': + source.status = 'connecting'; + break; + } + } + }; + + peerConnection.onicecandidateerror = (event) => { + console.error(`ICE candidate error for ${sourceId}:`, event); + const source = this.sources.get(sourceId); + if (source) { + source.errorCount++; + this.emit('camera-error', sourceId, new Error(`ICE candidate error: ${event.errorText}`)); + } + }; + } + + private handleSignalingMessage(message: SignalingMessage): void { + console.log('📨 Received signaling message:', message.type, 'for source:', message.sourceId); + + switch (message.type) { + case 'offer': + // Handle incoming offers (for remote cameras) + break; + case 'answer': + this.handleStreamingAnswer(message.sourceId, message.payload.sdp); + break; + case 'ice-candidate': + this.addIceCandidate(message.sourceId, message.payload.candidate); + break; + case 'error': + console.error('Signaling error:', message.payload); + this.emit('camera-error', message.sourceId, new Error(message.payload.error)); + break; + } + } + + private sendSignalingMessage(message: SignalingMessage): void { + if (this.signalingSocket && this.isConnected) { + this.signalingSocket.send(JSON.stringify(message)); + } else { + console.warn('Cannot send signaling message: not connected'); + } + } + + private startStatsCollection(sourceId: string): void { + const source = this.sources.get(sourceId); + if (!source || !source.peerConnection) return; + + const interval = setInterval(async () => { + try { + const stats = await source.peerConnection!.getStats(); + source.qualityStats = stats; + this.emit('stats-updated', sourceId, stats); + } catch (error) { + console.error(`Failed to collect stats for ${sourceId}:`, error); + } + }, this.config.statsInterval); + + this.statsIntervals.set(sourceId, interval); + } + + private stopStatsCollection(sourceId: string): void { + const interval = this.statsIntervals.get(sourceId); + if (interval) { + clearInterval(interval); + this.statsIntervals.delete(sourceId); + } + } + + // ==================== Utility Methods ==================== + + isConnectedToSignaling(): boolean { + return this.isConnected; + } + + getConnectionStats(): { total: number; connected: number; local: number; remote: number } { + const all = this.getAllCameras(); + return { + total: all.length, + connected: all.filter(s => s.status === 'connected').length, + local: all.filter(s => s.type === 'local').length, + remote: all.filter(s => s.type === 'remote').length + }; + } +} + +// Export singleton instance +export const webRTCManager = new WebRTCManager({}); \ No newline at end of file From bef8996376a21f1481886f20cd21890bbed0bf64 Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Wed, 25 Jun 2025 13:38:33 +0200 Subject: [PATCH 4/4] cameras working --- .gitignore | 1 + package-lock.json | 447 +++++++++++++- package.json | 3 + src/components/UrdfViewer.tsx | 2 +- src/components/landing/NgrokConfigModal.tsx | 71 ++- src/components/landing/RecordingModal.tsx | 34 -- .../recording/RecordingCameraPanel.tsx | 354 +++++++++++ .../webrtc/WebRTCCameraConfiguration.tsx | 550 +++++++++++++++++- .../webrtc/WebRTCVisualizerPanel.tsx | 28 +- src/pages/Recording.tsx | 94 +-- src/types/webrtc.ts | 5 +- src/utils/webrtc/WebRTCManager.ts | 223 ++++++- 12 files changed, 1653 insertions(+), 159 deletions(-) create mode 100644 .gitignore create mode 100644 src/components/recording/RecordingCameraPanel.tsx diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/package-lock.json b/package-lock.json index 05edd5f..d812454 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "@react-three/drei": "^9.122.0", "@react-three/fiber": "^8.18.0", "@tanstack/react-query": "^5.56.2", + "@types/qrcode": "^1.5.5", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.0.0", @@ -48,6 +49,7 @@ "jszip": "^3.10.1", "lucide-react": "^0.462.0", "next-themes": "^0.3.0", + "qrcode": "^1.5.4", "react": "^18.3.1", "react-day-picker": "^8.10.1", "react-dom": "^18.3.1", @@ -55,6 +57,7 @@ "react-resizable-panels": "^2.1.3", "react-router-dom": "^6.26.2", "recharts": "^2.12.7", + "socket.io-client": "^4.8.1", "sonner": "^1.5.0", "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", @@ -2823,6 +2826,12 @@ "win32" ] }, + "node_modules/@socket.io/component-emitter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", + "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==", + "license": "MIT" + }, "node_modules/@swc/core": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.12.1.tgz", @@ -3184,7 +3193,6 @@ "version": "22.15.31", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.31.tgz", "integrity": "sha512-jnVe5ULKl6tijxUhvQeNbQG/84fHfg+yMak02cT8QVhBx/F05rAVxCGBYYTh2EKz22D6JF5ktXuNwdx7b9iEGw==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -3202,6 +3210,15 @@ "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", "license": "MIT" }, + "node_modules/@types/qrcode": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@types/qrcode/-/qrcode-1.5.5.tgz", + "integrity": "sha512-CdfBi/e3Qk+3Z/fXYShipBT13OJ2fDO2Q2w5CIP5anLTLIndQG9z6P1cnm+8zCWSpm5dnxMFd/uREtb0EXuQzg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/react": { "version": "18.3.23", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.23.tgz", @@ -3839,6 +3856,15 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/camelcase-css": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", @@ -3943,6 +3969,72 @@ "url": "https://polar.sh/cva" } }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -4207,6 +4299,15 @@ } } }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/decimal.js-light": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", @@ -4241,6 +4342,12 @@ "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", "license": "Apache-2.0" }, + "node_modules/dijkstrajs": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/dijkstrajs/-/dijkstrajs-1.0.3.tgz", + "integrity": "sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==", + "license": "MIT" + }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -4310,6 +4417,45 @@ "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "license": "MIT" }, + "node_modules/engine.io-client": { + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.3.tgz", + "integrity": "sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1", + "engine.io-parser": "~5.2.1", + "ws": "~8.17.1", + "xmlhttprequest-ssl": "~2.1.1" + } + }, + "node_modules/engine.io-client/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/engine.io-parser": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/esbuild": { "version": "0.25.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.5.tgz", @@ -4757,6 +4903,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-nonce": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", @@ -5367,7 +5522,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/mz": { @@ -5510,6 +5664,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -5539,7 +5702,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5612,6 +5774,15 @@ "node": ">= 6" } }, + "node_modules/pngjs": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", + "integrity": "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==", + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/postcss": { "version": "8.5.5", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.5.tgz", @@ -5822,6 +5993,23 @@ "node": ">=6" } }, + "node_modules/qrcode": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.4.tgz", + "integrity": "sha512-1ca71Zgiu6ORjHqFBDpnSMTR2ReToX4l1Au1VFLyVeBTFavzQnv5JxMFr3ukHVKpSrSA2MCk0lNJSykjUfz7Zg==", + "license": "MIT", + "dependencies": { + "dijkstrajs": "^1.0.1", + "pngjs": "^5.0.0", + "yargs": "^15.3.1" + }, + "bin": { + "qrcode": "bin/qrcode" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -6171,6 +6359,15 @@ "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "license": "MIT" }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -6180,6 +6377,12 @@ "node": ">=0.10.0" } }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "license": "ISC" + }, "node_modules/resolve": { "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", @@ -6318,6 +6521,12 @@ "node": ">=10" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "license": "ISC" + }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", @@ -6357,6 +6566,68 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/socket.io-client": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.1.tgz", + "integrity": "sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.2", + "engine.io-client": "~6.6.1", + "socket.io-parser": "~4.2.4" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-client/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socket.io-parser": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz", + "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-parser/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/sonner": { "version": "1.7.4", "resolved": "https://registry.npmjs.org/sonner/-/sonner-1.7.4.tgz", @@ -6869,7 +7140,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/update-browserslist-db": { @@ -7540,6 +7810,12 @@ "node": ">= 8" } }, + "node_modules/which-module": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz", + "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", + "license": "ISC" + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -7638,6 +7914,41 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xmlhttprequest-ssl": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz", + "integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "license": "ISC" + }, "node_modules/yaml": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", @@ -7650,6 +7961,134 @@ "node": ">= 14.6" } }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "license": "MIT", + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "license": "ISC", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yargs/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index d8b5178..e4e7ada 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@react-three/drei": "^9.122.0", "@react-three/fiber": "^8.18.0", "@tanstack/react-query": "^5.56.2", + "@types/qrcode": "^1.5.5", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.0.0", @@ -51,6 +52,7 @@ "jszip": "^3.10.1", "lucide-react": "^0.462.0", "next-themes": "^0.3.0", + "qrcode": "^1.5.4", "react": "^18.3.1", "react-day-picker": "^8.10.1", "react-dom": "^18.3.1", @@ -58,6 +60,7 @@ "react-resizable-panels": "^2.1.3", "react-router-dom": "^6.26.2", "recharts": "^2.12.7", + "socket.io-client": "^4.8.1", "sonner": "^1.5.0", "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", diff --git a/src/components/UrdfViewer.tsx b/src/components/UrdfViewer.tsx index adf1b90..f0fb372 100644 --- a/src/components/UrdfViewer.tsx +++ b/src/components/UrdfViewer.tsx @@ -45,7 +45,7 @@ const UrdfViewer: React.FC = () => { // Real-time joint updates via WebSocket const { isConnected: isWebSocketConnected } = useRealTimeJoints({ viewerRef, - enabled: isDefaultModel, // Only enable WebSocket for default model + enabled: true, // Always enable WebSocket for real-time robot data }); // Add state for custom URDF path diff --git a/src/components/landing/NgrokConfigModal.tsx b/src/components/landing/NgrokConfigModal.tsx index 9ceefa2..f1bc3b2 100644 --- a/src/components/landing/NgrokConfigModal.tsx +++ b/src/components/landing/NgrokConfigModal.tsx @@ -16,11 +16,15 @@ import { useToast } from "@/hooks/use-toast"; interface NgrokConfigModalProps { open: boolean; onOpenChange: (open: boolean) => void; + onSuccess?: () => void | Promise; + isForExternalCamera?: boolean; } const NgrokConfigModal: React.FC = ({ open, onOpenChange, + onSuccess, + isForExternalCamera = false, }) => { const { ngrokUrl, @@ -36,9 +40,24 @@ const NgrokConfigModal: React.FC = ({ const handleSave = async () => { if (!inputUrl.trim()) { resetToLocalhost(); + + // Clear external URL configuration in backend + try { + await fetch("http://localhost:8000/api/config/external-url", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ external_url: "" }), + }); + console.log("✅ Backend external URL configuration cleared"); + } catch (error) { + console.warn("⚠️ Failed to clear backend external URL:", error); + } + toast({ title: "Ngrok Disabled", - description: "Switched back to localhost mode.", + description: "Switched back to localhost mode for QR codes too.", }); onOpenChange(false); return; @@ -64,10 +83,31 @@ const NgrokConfigModal: React.FC = ({ if (testResponse.ok) { setNgrokUrl(cleanUrl); + + // Send ngrok URL to backend for QR code generation + try { + await fetchWithHeaders(`${cleanUrl}/api/config/external-url`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ external_url: cleanUrl }), + }); + console.log("✅ Backend notified of ngrok URL for QR codes"); + } catch (backendError) { + console.warn("⚠️ Failed to notify backend of ngrok URL:", backendError); + } + toast({ title: "Ngrok Configured Successfully", - description: `Connected to ${cleanUrl}. All API calls will now use this URL.`, + description: `Connected to ${cleanUrl}. All API calls and QR codes will now use this URL.`, }); + + // Call success callback if provided (for external camera flow) + if (onSuccess) { + await onSuccess(); + } + onOpenChange(false); } else { throw new Error(`Server responded with status ${testResponse.status}`); @@ -84,12 +124,27 @@ const NgrokConfigModal: React.FC = ({ } }; - const handleReset = () => { + const handleReset = async () => { resetToLocalhost(); setInputUrl(""); + + // Clear external URL configuration in backend + try { + await fetch("http://localhost:8000/api/config/external-url", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ external_url: "" }), + }); + console.log("✅ Backend external URL configuration cleared"); + } catch (error) { + console.warn("⚠️ Failed to clear backend external URL:", error); + } + toast({ title: "Reset to Localhost", - description: "All API calls will now use localhost:8000.", + description: "All API calls and QR codes will now use localhost:8000.", }); onOpenChange(false); }; @@ -102,11 +157,13 @@ const NgrokConfigModal: React.FC = ({
- Ngrok Configuration + {isForExternalCamera ? "Ngrok Required for External Camera" : "Ngrok Configuration"} - Configure ngrok tunnel for external access and phone camera - features. + {isForExternalCamera + ? "External phone cameras require ngrok for HTTPS connectivity. Please configure ngrok to continue." + : "Configure ngrok tunnel for external access and phone camera features." + } diff --git a/src/components/landing/RecordingModal.tsx b/src/components/landing/RecordingModal.tsx index e37cac2..b44b331 100644 --- a/src/components/landing/RecordingModal.tsx +++ b/src/components/landing/RecordingModal.tsx @@ -16,13 +16,11 @@ import { DialogTitle, DialogDescription, } from "@/components/ui/dialog"; -import { QrCode } from "lucide-react"; import PortDetectionModal from "@/components/ui/PortDetectionModal"; import PortDetectionButton from "@/components/ui/PortDetectionButton"; import WebRTCCameraConfiguration, { CameraConfig, } from "@/components/webrtc/WebRTCCameraConfiguration"; -import QrCodeModal from "@/components/recording/QrCodeModal"; import { useApi } from "@/contexts/ApiContext"; import { useAutoSave } from "@/hooks/useAutoSave"; interface RecordingModalProps { @@ -81,8 +79,6 @@ const RecordingModal: React.FC = ({ const [detectionRobotType, setDetectionRobotType] = useState< "leader" | "follower" >("leader"); - const [showQrCodeModal, setShowQrCodeModal] = useState(false); - const [sessionId, setSessionId] = useState(""); const handlePortDetection = (robotType: "leader" | "follower") => { setDetectionRobotType(robotType); @@ -171,15 +167,6 @@ const RecordingModal: React.FC = ({ } }, [open, setLeaderPort, setFollowerPort, setLeaderConfig, setFollowerConfig, leaderConfigs, followerConfigs, baseUrl, fetchWithHeaders]); - const handleQrCodeClick = () => { - // Generate a session ID for this recording session - const newSessionId = `recording_${Date.now()}_${Math.random() - .toString(36) - .substr(2, 9)}`; - setSessionId(newSessionId); - setShowQrCodeModal(true); - }; - return ( <> @@ -200,22 +187,6 @@ const RecordingModal: React.FC = ({ recording. -
-

- Need an extra angle? -

-

- Add your phone as a secondary camera. -

- -
@@ -427,11 +398,6 @@ const RecordingModal: React.FC = ({ onPortDetected={handlePortDetected} /> - ); diff --git a/src/components/recording/RecordingCameraPanel.tsx b/src/components/recording/RecordingCameraPanel.tsx new file mode 100644 index 0000000..46bc868 --- /dev/null +++ b/src/components/recording/RecordingCameraPanel.tsx @@ -0,0 +1,354 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; +import { + Camera, + VideoOff, + Wifi, + WifiOff, + Activity +} from "lucide-react"; +import { cn } from "@/lib/utils"; +import { webRTCManager } from "@/utils/webrtc/WebRTCManager"; +import { UnifiedCameraSource } from "@/types/webrtc"; +import { useApi } from "@/contexts/ApiContext"; + +interface RecordingCameraPanelProps { + className?: string; +} + +const RecordingCameraPanel: React.FC = ({ + className, +}) => { + const { baseUrl } = useApi(); + const [webrtcSources, setWebrtcSources] = useState([]); + const [isConnectedToSignaling, setIsConnectedToSignaling] = useState(false); + + // Video element refs for WebRTC streams + const videoRefs = useRef>(new Map()); + + // WebRTC event handlers (with useCallback to maintain references) + const handleCameraAdded = useCallback((source: UnifiedCameraSource) => { + console.log("📹 Camera added in recording:", source.name); + setWebrtcSources(prev => { + // Remove any existing source with same ID or deviceId to prevent duplicates + const filtered = prev.filter(s => s.id !== source.id && s.deviceId !== source.deviceId); + return [...filtered, source]; + }); + }, []); + + const handleCameraRemoved = useCallback((sourceId: string) => { + console.log("🗑️ Camera removed in recording:", sourceId); + setWebrtcSources(prev => prev.filter(s => s.id !== sourceId)); + + // Remove video element ref + const videoElement = videoRefs.current.get(sourceId); + if (videoElement) { + videoElement.srcObject = null; + videoRefs.current.delete(sourceId); + } + }, []); + + const handleCameraConnected = useCallback((sourceId: string, stream: MediaStream) => { + console.log("✅ Camera connected in recording:", sourceId); + + // Update source status + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'connected', stream } + : source + )); + + // Attach stream to video element + setTimeout(() => { + const videoElement = videoRefs.current.get(sourceId); + if (videoElement && stream) { + console.log(`🔗 Attaching stream to video element for ${sourceId}`); + videoElement.srcObject = stream; + videoElement.play().catch(console.error); + } + }, 100); + }, []); + + const handleCameraDisconnected = useCallback((sourceId: string) => { + console.log("🔌 Camera disconnected in recording:", sourceId); + setWebrtcSources(prev => prev.map(source => + source.id === sourceId + ? { ...source, status: 'disconnected', stream: undefined } + : source + )); + }, []); + + // Initialize WebRTC connection and load cameras + useEffect(() => { + console.log("🔄 RecordingCameraPanel useEffect triggered"); + + const initializeWebRTC = async () => { + try { + console.log("🚀 Initializing WebRTC for recording..."); + + // Always load existing cameras first, regardless of connection status + const loadExistingCameras = () => { + const existingSources = webRTCManager.getAllCameras(); + console.log("📹 Loading existing WebRTC cameras for recording:", existingSources.length, existingSources.map(s => ({ name: s.name, status: s.status, hasStream: !!s.stream }))); + + if (existingSources.length > 0) { + // Deduplicate by deviceId and ID to prevent duplicates + const uniqueSources = existingSources.reduce((acc, source) => { + const existingIndex = acc.findIndex(s => s.id === source.id || s.deviceId === source.deviceId); + if (existingIndex === -1) { + acc.push(source); + } else { + // Keep the most recent one (higher timestamp or connected status) + if (source.status === 'connected' && acc[existingIndex].status !== 'connected') { + acc[existingIndex] = source; + } + } + return acc; + }, [] as UnifiedCameraSource[]); + + console.log("📹 Deduplicated sources:", uniqueSources.length); + setWebrtcSources(uniqueSources); + + // Setup video streams for existing cameras (use deduplicated sources) + uniqueSources.forEach(source => { + console.log(`🎥 Setting up stream for existing camera: ${source.name}, status: ${source.status}, hasStream: ${!!source.stream}`); + if (source.stream && source.status === 'connected') { + // Use a slightly longer timeout to ensure DOM is ready + setTimeout(() => { + const videoElement = videoRefs.current.get(source.id); + if (videoElement && source.stream) { + console.log(`🔗 Attaching stream to video for ${source.name}`); + videoElement.srcObject = source.stream; + videoElement.play().catch(console.error); + } else { + console.log(`⚠️ Video element or stream not available for ${source.name}`, { + hasElement: !!videoElement, + hasStream: !!source.stream + }); + } + }, 200); + } + }); + } else { + console.log("📹 No existing cameras found in WebRTC manager"); + } + }; + + // Setup event listeners + webRTCManager.on('camera-added', handleCameraAdded); + webRTCManager.on('camera-removed', handleCameraRemoved); + webRTCManager.on('camera-connected', handleCameraConnected); + webRTCManager.on('camera-disconnected', handleCameraDisconnected); + webRTCManager.on('connected', () => { + console.log("✅ Connected to WebRTC signaling server in recording"); + setIsConnectedToSignaling(true); + // Reload cameras when we connect + loadExistingCameras(); + }); + webRTCManager.on('disconnected', () => { + console.log("🔌 Disconnected from WebRTC signaling server in recording"); + setIsConnectedToSignaling(false); + }); + + // Load cameras immediately + loadExistingCameras(); + + // Check if already connected + if (webRTCManager.isConnectedToSignaling()) { + console.log("✅ WebRTC already connected"); + setIsConnectedToSignaling(true); + } else { + console.log("🔄 WebRTC not connected, attempting connection..."); + // Configure and connect if not already connected + if (!webRTCManager.config.signalingUrl.includes(baseUrl)) { + webRTCManager.config.signalingUrl = baseUrl.replace('http', 'ws') + '/ws/webrtc'; + } + await webRTCManager.connect(); + } + + } catch (error) { + console.error("❌ Failed to initialize WebRTC for recording:", error); + } + }; + + initializeWebRTC(); + + // Cleanup event listeners on unmount + return () => { + webRTCManager.off('camera-added', handleCameraAdded); + webRTCManager.off('camera-removed', handleCameraRemoved); + webRTCManager.off('camera-connected', handleCameraConnected); + webRTCManager.off('camera-disconnected', handleCameraDisconnected); + }; + }, [baseUrl, handleCameraAdded, handleCameraRemoved, handleCameraConnected, handleCameraDisconnected]); + + // Get available cameras (only connected ones for display) + const getAvailableCameras = () => { + return webrtcSources.filter(source => source.status === 'connected'); + }; + + // Get grid classes for camera layout (optimized for recording mode) + const getCameraGridClasses = () => { + const cameraCount = getAvailableCameras().length; + + if (cameraCount === 0) return ""; + if (cameraCount === 1) return "flex flex-col gap-2"; // 1 camera: single column + if (cameraCount === 2) return "flex flex-col gap-2"; // 2 cameras: stacked + if (cameraCount >= 3) return "grid grid-cols-1 gap-2"; // 3+ cameras: single column grid + }; + + const availableCameras = getAvailableCameras(); + + return ( +
+ {availableCameras.length > 0 ? ( +
+ {availableCameras.map((source) => ( + { + if (el) { + videoRefs.current.set(source.id, el); + } else { + videoRefs.current.delete(source.id); + } + }} + /> + ))} +
+ ) : ( +
+ + + No cameras available + + + Configure cameras in settings + +
+ )} +
+ ); +}; + +// Camera Display Component optimized for recording +interface RecordingCameraDisplayProps { + source: UnifiedCameraSource; + videoRef: (el: HTMLVideoElement | null) => void; +} + +const RecordingCameraDisplay: React.FC = ({ + source, + videoRef, +}) => { + const [isPlaying, setIsPlaying] = useState(false); + const videoElementRef = useRef(null); + + // Handle video ref assignment and stream management + const handleVideoRef = useCallback((el: HTMLVideoElement | null) => { + videoElementRef.current = el; + videoRef(el); + }, [videoRef]); + + // Use useEffect to manage srcObject changes to prevent flashing + useEffect(() => { + const videoElement = videoElementRef.current; + if (videoElement && source.stream && source.status === 'connected') { + // Only update srcObject if it's different to prevent flashing + if (videoElement.srcObject !== source.stream) { + console.log(`🔗 Updating srcObject for ${source.name}`); + videoElement.srcObject = source.stream; + videoElement.play().catch(console.error); + } + } else if (videoElement && (!source.stream || source.status !== 'connected')) { + // Clear srcObject when stream is not available + if (videoElement.srcObject) { + console.log(`🔗 Clearing srcObject for ${source.name}`); + videoElement.srcObject = null; + } + } + }, [source.stream, source.status, source.name]); + + const getStatusColor = () => { + switch (source.status) { + case 'connected': return 'bg-green-500'; + case 'connecting': return 'bg-yellow-500 animate-pulse'; + case 'disconnected': return 'bg-red-500'; + case 'error': return 'bg-red-600'; + default: return 'bg-gray-500'; + } + }; + + const handleVideoPlay = () => { + setIsPlaying(true); + }; + + const handleVideoError = () => { + setIsPlaying(false); + console.error(`Video playback error for camera ${source.name}`); + }; + + // Check if this is a remote camera (external) waiting for connection + const isRemoteCamera = source.type === 'remote'; + const shouldShowQRMessage = isRemoteCamera && !source.stream && source.status === 'connecting'; + + return ( +
+
+ {source.stream && source.status === 'connected' ? ( + <> +
+ +
+ + {source.name} + + + {source.id.substring(0, 8)}... + +
+
+ ); +}; + +export default RecordingCameraPanel; \ No newline at end of file diff --git a/src/components/webrtc/WebRTCCameraConfiguration.tsx b/src/components/webrtc/WebRTCCameraConfiguration.tsx index eced054..c350b6a 100644 --- a/src/components/webrtc/WebRTCCameraConfiguration.tsx +++ b/src/components/webrtc/WebRTCCameraConfiguration.tsx @@ -18,12 +18,16 @@ import { RefreshCw, Wifi, WifiOff, - Activity + Activity, + Smartphone, + QrCode, + Globe } from "lucide-react"; import { useToast } from "@/hooks/use-toast"; import { webRTCManager } from "@/utils/webrtc/WebRTCManager"; import { UnifiedCameraSource, CameraQuality, CAMERA_CONSTRAINTS } from "@/types/webrtc"; import { useApi } from "@/contexts/ApiContext"; +import NgrokConfigModal from "@/components/landing/NgrokConfigModal"; // Legacy interface for compatibility export interface CameraConfig { @@ -57,7 +61,7 @@ const WebRTCCameraConfiguration: React.FC = ({ releaseStreamsRef, loadSavedCameras = true, }) => { - const { baseUrl, fetchWithHeaders } = useApi(); + const { baseUrl, fetchWithHeaders, isNgrokEnabled } = useApi(); const { toast } = useToast(); // WebRTC state @@ -69,8 +73,16 @@ const WebRTCCameraConfiguration: React.FC = ({ const [detectedCameras, setDetectedCameras] = useState([]); const [selectedCameraIndex, setSelectedCameraIndex] = useState(""); const [cameraName, setCameraName] = useState(""); - // Removed selectedQuality - will be configurable per camera after adding const [isLoadingCameras, setIsLoadingCameras] = useState(false); + + // External camera state + const [isGeneratingQR, setIsGeneratingQR] = useState(false); + const [externalSessionQrUrls, setExternalSessionQrUrls] = useState>(new Map()); + const [reconnectingSessions, setReconnectingSessions] = useState>(new Set()); + + // Ngrok modal state for external cameras + const [showNgrokModalForCamera, setShowNgrokModalForCamera] = useState(false); + const [pendingCameraName, setPendingCameraName] = useState(""); // WebRTC video elements refs const videoRefs = useRef>(new Map()); @@ -103,13 +115,23 @@ const WebRTCCameraConfiguration: React.FC = ({ : source )); + // Clean up reconnecting state for this session + const connectedSource = webrtcSources.find(s => s.id === sourceId); + if (connectedSource?.deviceId) { + setReconnectingSessions(prev => { + const newSet = new Set(prev); + newSet.delete(connectedSource.deviceId!); + return newSet; + }); + } + // Attach stream to video element const videoElement = videoRefs.current.get(sourceId); if (videoElement && stream) { videoElement.srcObject = stream; videoElement.play().catch(console.error); } - }, []); + }, [webrtcSources]); const handleCameraDisconnected = useCallback((sourceId: string) => { console.log("🔌 Camera disconnected:", sourceId); @@ -129,6 +151,30 @@ const WebRTCCameraConfiguration: React.FC = ({ }); }, [toast]); + const handleSessionCreated = useCallback((sessionId: string, qrUrl: string, sessionInfo?: any) => { + console.log("🌐 Session created with QR URL:", { sessionId, qrUrl, sessionInfo }); + console.log("🌐 Current externalSessionQrUrls before update:", externalSessionQrUrls); + + // Always save the QR URL if we have one + if (qrUrl) { + setExternalSessionQrUrls(prev => { + const newMap = new Map(prev.set(sessionId, qrUrl)); + console.log("🌐 Updated externalSessionQrUrls:", newMap); + return newMap; + }); + } + + // If session existed and device was connected, mark as reconnecting + if (sessionInfo?.sessionExisted && sessionInfo?.deviceConnected) { + console.log("🔄 Session existed with connected device - marking as reconnecting"); + setReconnectingSessions(prev => { + const newSet = new Set(prev.add(sessionId)); + console.log("🔄 Updated reconnectingSessions:", newSet); + return newSet; + }); + } + }, [externalSessionQrUrls]); + const handleCameraUpdated = useCallback((sourceId: string, updatedSource: UnifiedCameraSource) => { console.log("🔄 Camera updated in WebRTC manager:", sourceId); setWebrtcSources(prev => prev.map(s => @@ -136,6 +182,44 @@ const WebRTCCameraConfiguration: React.FC = ({ )); }, []); + // Helper function to request QR URL for cameras that don't have it + const requestQRUrlForExistingCamera = useCallback(async (deviceId: string, name: string, retryCount = 0) => { + const maxRetries = 3; + + // Check connection state + if (!webRTCManager.isConnectedToSignaling()) { + console.log(`❌ Cannot request QR URL for ${deviceId}: not connected to signaling server`); + + // If not connected and we have retries left, wait and try again + if (retryCount < maxRetries) { + console.log(`⏳ Waiting 3 seconds before retry ${retryCount + 1}/${maxRetries} for ${deviceId}`); + setTimeout(() => { + requestQRUrlForExistingCamera(deviceId, name, retryCount + 1); + }, 3000); + } else { + console.log(`❌ Max retries reached for ${deviceId}, giving up`); + } + return; + } + + try { + console.log(`🔄 Checking if session exists for ${deviceId} (attempt ${retryCount + 1})`); + // Check if session already exists and just needs to be restored + await webRTCManager.addRemoteCamera(deviceId, name, "medium"); + console.log(`✅ Successfully requested session info for ${deviceId}`); + } catch (error) { + console.error(`❌ Failed to request QR URL for ${deviceId}:`, error); + + // Retry if we have attempts left + if (retryCount < maxRetries) { + console.log(`⏳ Retrying QR URL request for ${deviceId} in 2 seconds...`); + setTimeout(() => { + requestQRUrlForExistingCamera(deviceId, name, retryCount + 1); + }, 2000); + } + } + }, []); + const fetchSignalingStats = useCallback(async () => { try { const response = await fetchWithHeaders(`${baseUrl}/webrtc/status`); @@ -150,7 +234,26 @@ const WebRTCCameraConfiguration: React.FC = ({ console.log("✅ Connected to WebRTC signaling server"); setIsConnectedToSignaling(true); fetchSignalingStats(); - }, [fetchSignalingStats]); + + // If we have external cameras without QR URLs, try to request them now + const externalCamerasWithoutQR = webrtcSources.filter(camera => + camera.type === 'remote' && + (camera.deviceId?.startsWith('external_') || camera.deviceId?.startsWith('phone_')) && + camera.deviceId && !externalSessionQrUrls.get(camera.deviceId) + ); + + if (externalCamerasWithoutQR.length > 0) { + console.log(`🔄 Connection established, requesting QR URLs for ${externalCamerasWithoutQR.length} external cameras`); + setTimeout(() => { + externalCamerasWithoutQR.forEach(camera => { + if (camera.deviceId) { + console.log(`🔄 Requesting QR URL for ${camera.deviceId} after connection`); + requestQRUrlForExistingCamera(camera.deviceId, camera.name, 0); + } + }); + }, 1000); + } + }, [fetchSignalingStats, webrtcSources, externalSessionQrUrls, requestQRUrlForExistingCamera]); const handleWebRTCDisconnected = useCallback(() => { console.log("🔌 Disconnected from WebRTC signaling server"); @@ -164,7 +267,10 @@ const WebRTCCameraConfiguration: React.FC = ({ console.log("🚀 Initializing WebRTC Manager..."); // Configure WebRTC manager - webRTCManager.config.signalingUrl = `${baseUrl.replace('http', 'ws')}/ws/webrtc`; + const signalingUrl = baseUrl.replace('http', 'ws') + '/ws/webrtc'; + console.log('🔧 Configuring WebRTC signaling URL:', signalingUrl); + console.log('🔧 Base URL:', baseUrl); + webRTCManager.config.signalingUrl = signalingUrl; // Setup event listeners webRTCManager.on('camera-added', handleCameraAdded); @@ -173,6 +279,7 @@ const WebRTCCameraConfiguration: React.FC = ({ webRTCManager.on('camera-disconnected', handleCameraDisconnected); webRTCManager.on('camera-error', handleCameraError); webRTCManager.on('camera-updated', handleCameraUpdated); + webRTCManager.on('session-created', handleSessionCreated); webRTCManager.on('connected', handleWebRTCConnected); webRTCManager.on('disconnected', handleWebRTCDisconnected); @@ -180,6 +287,7 @@ const WebRTCCameraConfiguration: React.FC = ({ if (!webRTCManager.isConnectedToSignaling()) { console.log("🔗 Connecting to signaling server..."); await webRTCManager.connect(); + console.log("✅ Successfully connected to signaling server"); } else { console.log("✅ Already connected to signaling server"); // Set the connected state even if already connected @@ -187,6 +295,9 @@ const WebRTCCameraConfiguration: React.FC = ({ fetchSignalingStats(); } + // Wait a bit for connection to stabilize before loading cameras + await new Promise(resolve => setTimeout(resolve, 1000)); + // Load saved cameras if requested and no cameras exist yet if (loadSavedCameras && webRTCManager.getAllCameras().length === 0) { console.log("📂 No existing cameras, loading saved configurations..."); @@ -197,6 +308,24 @@ const WebRTCCameraConfiguration: React.FC = ({ const existingCameras = webRTCManager.getAllCameras(); console.log("📹 Loading existing cameras into state:", existingCameras.length); setWebrtcSources([...existingCameras]); + + // For existing external cameras, try to restore QR URLs + const externalCameras = existingCameras.filter(camera => + camera.type === 'remote' && + (camera.deviceId?.startsWith('external_') || camera.deviceId?.startsWith('phone_')) + ); + + if (externalCameras.length > 0) { + console.log("🔄 Found existing external cameras, requesting QR URLs..."); + setTimeout(() => { + externalCameras.forEach(camera => { + if (camera.deviceId && !externalSessionQrUrls.get(camera.deviceId)) { + console.log(`🔄 Requesting QR URL for existing camera: ${camera.deviceId}`); + requestQRUrlForExistingCamera(camera.deviceId, camera.name); + } + }); + }, 2000); + } } } catch (error) { @@ -220,6 +349,7 @@ const WebRTCCameraConfiguration: React.FC = ({ webRTCManager.off('camera-disconnected', handleCameraDisconnected); webRTCManager.off('camera-error', handleCameraError); webRTCManager.off('camera-updated', handleCameraUpdated); + webRTCManager.off('session-created', handleSessionCreated); // Also remove our connection listeners specifically webRTCManager.off('connected', handleWebRTCConnected); @@ -227,7 +357,7 @@ const WebRTCCameraConfiguration: React.FC = ({ // Don't disconnect WebRTC manager as teleoperation might need it }; - }, [baseUrl, loadSavedCameras, handleCameraAdded, handleCameraRemoved, handleCameraConnected, handleCameraDisconnected, handleCameraError, handleCameraUpdated, handleWebRTCConnected, handleWebRTCDisconnected]); + }, [baseUrl, loadSavedCameras, handleCameraAdded, handleCameraRemoved, handleCameraConnected, handleCameraDisconnected, handleCameraError, handleCameraUpdated, handleSessionCreated, handleWebRTCConnected, handleWebRTCDisconnected, requestQRUrlForExistingCamera]); // Sync WebRTC sources to parent component const syncToParent = useCallback(() => { @@ -303,6 +433,16 @@ const WebRTCCameraConfiguration: React.FC = ({ // Add camera to WebRTC system const addCamera = async () => { + // Check if "Add External" option is selected + if (selectedCameraIndex === "external_phone") { + await addExternalCamera(); + } else { + await addLocalCamera(); + } + }; + + // Add local camera + const addLocalCamera = async () => { if (!selectedCameraIndex || !cameraName.trim()) { toast({ title: "Missing Information", @@ -347,7 +487,7 @@ const WebRTCCameraConfiguration: React.FC = ({ } try { - console.log(`🆕 Adding WebRTC camera: ${cameraName} (${selectedCamera.deviceId})`); + console.log(`🆕 Adding WebRTC local camera: ${cameraName} (${selectedCamera.deviceId})`); const sourceId = await webRTCManager.addLocalCamera( selectedCamera.deviceId, @@ -368,7 +508,7 @@ const WebRTCCameraConfiguration: React.FC = ({ }); } catch (error) { - console.error("❌ Failed to add camera:", error); + console.error("❌ Failed to add local camera:", error); toast({ title: "Camera Add Failed", description: `Could not add camera: ${error.message}`, @@ -377,6 +517,115 @@ const WebRTCCameraConfiguration: React.FC = ({ } }; + // Add external camera + const addExternalCamera = async () => { + if (!cameraName.trim()) { + toast({ + title: "Missing Information", + description: "Please provide a name for the external camera.", + variant: "destructive", + }); + return; + } + + // Check if ngrok is configured before proceeding + if (!isNgrokEnabled) { + console.log("🌐 Ngrok not configured, opening ngrok modal for external camera"); + setPendingCameraName(cameraName); + setShowNgrokModalForCamera(true); + return; + } + + if (!isConnectedToSignaling) { + toast({ + title: "Not Connected", + description: "Not connected to WebRTC signaling server.", + variant: "destructive", + }); + return; + } + + await performAddExternalCamera(cameraName); + }; + + // Core logic for adding external camera (can be called from ngrok success callback) + const performAddExternalCamera = async (camName: string) => { + setIsGeneratingQR(true); + + try { + console.log(`🆕 Adding external camera: ${camName}`); + + // Generate unique session ID for external camera connection + const sessionId = `external_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + + // Create a remote camera source + const sourceId = await webRTCManager.addRemoteCamera( + sessionId, + camName.trim(), + "medium" + ); + + // Save to backend + await saveCameraToBackend(sourceId, camName, sessionId, "medium"); + + // Reset form + setCameraName(""); + setSelectedCameraIndex(""); + + toast({ + title: "External Camera Created", + description: `${camName} QR code generated. Scan with your device to connect.`, + }); + + } catch (error) { + console.error("❌ Failed to add external camera:", error); + toast({ + title: "External Camera Add Failed", + description: `Could not create external camera: ${error.message}`, + variant: "destructive", + }); + } finally { + setIsGeneratingQR(false); + } + }; + + // Handle successful ngrok configuration for camera creation + const handleNgrokConfiguredForCamera = async () => { + console.log("✅ Ngrok configured successfully, proceeding with camera creation"); + setShowNgrokModalForCamera(false); + + if (pendingCameraName) { + await performAddExternalCamera(pendingCameraName); + setPendingCameraName(""); + } + }; + + // Handle ngrok modal cancellation for camera creation + const handleNgrokCancelledForCamera = () => { + console.log("❌ Ngrok configuration cancelled, not adding camera"); + setShowNgrokModalForCamera(false); + setPendingCameraName(""); + // Don't add the camera - user cancelled the required ngrok setup + }; + + // Get local network IP for QR code generation + const getLocalNetworkIP = async (): Promise => { + try { + // Try to get it from the current URL if it's not localhost + const currentHost = window.location.hostname; + if (currentHost !== 'localhost' && currentHost !== '127.0.0.1') { + return currentHost; + } + + // Otherwise, try to detect local IP (this is a simplified approach) + // In production, the backend should provide this information + return currentHost; + } catch (error) { + console.error("Failed to get local IP:", error); + return window.location.hostname; + } + }; + // Update camera settings (resolution, FPS) const updateCameraSettings = async (sourceId: string, updates: Partial) => { const source = webrtcSources.find(s => s.id === sourceId); @@ -544,9 +793,30 @@ const WebRTCCameraConfiguration: React.FC = ({ if ((config as any).type === "webrtc" && (config as any).device_id && (config as any).source_id) { try { const quality = (config as any).quality || "medium"; + const deviceId = (config as any).device_id; console.log(`🔄 Restoring WebRTC camera: ${name}`); - await webRTCManager.addLocalCamera((config as any).device_id, name, quality); + // Check if this is an external camera (remote) or local camera + if (deviceId.startsWith('external_') || deviceId.startsWith('phone_')) { + // This is a remote external camera (legacy phone_ prefix supported) + console.log(`🔄 Restoring external camera: ${name} with deviceId: ${deviceId}`); + console.log(`🔄 Current externalSessionQrUrls:`, externalSessionQrUrls); + await webRTCManager.addRemoteCamera(deviceId, name, quality); + console.log(`🔄 After addRemoteCamera, externalSessionQrUrls:`, externalSessionQrUrls); + + // Wait a bit and check if QR URL was received, if not request it again + setTimeout(() => { + if (!externalSessionQrUrls.get(deviceId)) { + console.log(`⚠️ QR URL not received for ${deviceId}, requesting again...`); + requestQRUrlForExistingCamera(deviceId, name, 0); + } else { + console.log(`✅ QR URL already available for ${deviceId}`); + } + }, 3000); + } else { + // This is a local camera + await webRTCManager.addLocalCamera(deviceId, name, quality); + } } catch (error) { console.error(`❌ Failed to restore camera ${name}:`, error); } @@ -611,7 +881,7 @@ const WebRTCCameraConfiguration: React.FC = ({
{/* Add Camera Section */} - {detectedCameras.length > 0 && isConnectedToSignaling && ( + {(detectedCameras.length > 0 || isConnectedToSignaling) && (

Add Camera

@@ -643,6 +913,16 @@ const WebRTCCameraConfiguration: React.FC = ({ ); })} + {/* Add External Camera option at the end */} + +
+ + Add External +
+
@@ -654,7 +934,7 @@ const WebRTCCameraConfiguration: React.FC = ({ setCameraName(e.target.value)} - placeholder="e.g., workspace_cam" + placeholder={selectedCameraIndex === "external_phone" ? "e.g., external_cam" : "e.g., workspace_cam"} className="bg-gray-800 border-gray-700 text-white" />
@@ -663,10 +943,26 @@ const WebRTCCameraConfiguration: React.FC = ({
@@ -685,6 +981,8 @@ const WebRTCCameraConfiguration: React.FC = ({ removeCamera(source.id)} onUpdateSource={(updates) => { updateCameraSettings(source.id, updates); @@ -714,6 +1012,18 @@ const WebRTCCameraConfiguration: React.FC = ({

)} + + {/* Ngrok Configuration Modal for External Cameras */} + { + if (!open) { + handleNgrokCancelledForCamera(); + } + }} + onSuccess={handleNgrokConfiguredForCamera} + isForExternalCamera={true} + />
); }; @@ -724,6 +1034,8 @@ interface WebRTCCameraPreviewProps { onRemove: () => void; onUpdateSource: (updates: Partial) => void; videoRef: (el: HTMLVideoElement | null) => void; + externalSessionQrUrls?: Map; + reconnectingSessions?: Set; } const WebRTCCameraPreview: React.FC = ({ @@ -731,7 +1043,60 @@ const WebRTCCameraPreview: React.FC = ({ onRemove, onUpdateSource, videoRef, + externalSessionQrUrls, + reconnectingSessions, }) => { + const localVideoRef = useRef(null); + const [forceShowQR, setForceShowQR] = useState(false); + + // Handle video stream assignment separately to avoid re-renders causing flashing + useEffect(() => { + if (localVideoRef.current && source.stream && source.status === 'connected') { + const videoElement = localVideoRef.current; + + // Only set srcObject if it's different from current + if (videoElement.srcObject !== source.stream) { + console.log(`🎥 Setting srcObject for ${source.name}`); + videoElement.srcObject = source.stream; + videoElement.play().catch(e => { + // Only log if it's not the "interrupted by new load" error, which is expected + if (e.name !== 'AbortError') { + console.error(`Video play error for ${source.name}:`, e); + } + }); + } + } else if (localVideoRef.current && !source.stream) { + // Clear srcObject if no stream + localVideoRef.current.srcObject = null; + } + }, [source.stream, source.status, source.name]); + + // Reset forceShowQR when camera connects successfully + useEffect(() => { + if (source.status === 'connected' && forceShowQR) { + setForceShowQR(false); + } + }, [source.status, forceShowQR]); + + // Check if this is an external camera waiting for connection + const isExternalCamera = source.type === 'remote' && !source.stream; + + // Logic for showing QR vs waiting for reconnection + const qrUrlValue = externalSessionQrUrls?.get(source.deviceId || ''); + const isReconnecting = reconnectingSessions?.has(source.deviceId || '') || false; + const hasValidQrUrl = !!qrUrlValue; + + // Auto-trigger ngrok modal if QR is forced but no URL available (ngrok not configured) + useEffect(() => { + if (forceShowQR && isExternalCamera && !qrUrlValue) { + console.log("🌐 QR forced but no URL available - ngrok needs configuration"); + // Reset forceShowQR and trigger ngrok modal + setForceShowQR(false); + // Here we could trigger the ngrok modal, but we need access to those functions + // For now, let's add a note that ngrok needs to be configured + } + }, [forceShowQR, isExternalCamera, qrUrlValue]); + const getStatusColor = () => { switch (source.status) { case 'connected': return 'text-green-400'; @@ -751,6 +1116,10 @@ const WebRTCCameraPreview: React.FC = ({ default: return ; } }; + + // Show QR for new external cameras, when forced, or when we have a valid QR URL but device hasn't connected yet + // Don't show QR only when it's a reconnection scenario (unless forced) + const shouldShowQR = isExternalCamera && source.status === 'connecting' && (!isReconnecting || forceShowQR); return (
@@ -760,12 +1129,8 @@ const WebRTCCameraPreview: React.FC = ({ <>
+ ) : shouldShowQR ? ( +
+ +
+

Scan QR or access URL from device

+
+ { + const qrUrl = externalSessionQrUrls?.get(source.deviceId || ''); + console.log(`🔍 Getting QR URL for deviceId '${source.deviceId}' from map:`, qrUrl); + console.log(`🔍 Current externalSessionQrUrls keys:`, Array.from(externalSessionQrUrls?.keys() || [])); + return qrUrl; + })()} + /> +
+ {!qrUrlValue && forceShowQR && ( +

+ ⚠️ ngrok must be configured to generate QR codes +

+ )} + {isReconnecting && forceShowQR && ( + + )} +
+
) : (
- - - {source.status === 'connecting' ? 'Connecting...' : 'No Stream'} - + {isExternalCamera ? ( + <> + + + {source.status === 'connecting' ? ( + isReconnecting ? 'Waiting for device reconnection...' : 'Waiting for device...' + ) : 'Device disconnected'} + + {isReconnecting && source.status === 'connecting' && !forceShowQR && ( +
+

Refresh the camera page on your device to reconnect

+ +
+ )} + + ) : ( + <> + + + {source.status === 'connecting' ? 'Connecting...' : 'No Stream'} + + + )}
)}
@@ -853,4 +1273,86 @@ const WebRTCCameraPreview: React.FC = ({ ); }; +// QR Code Component using qrcode library +interface QRCodePlaceholderProps { + sessionId: string; + qrUrl?: string; +} + +const QRCodePlaceholder: React.FC = ({ sessionId, qrUrl }) => { + const [qrDataUrl, setQrDataUrl] = useState(""); + const [isLoading, setIsLoading] = useState(true); + + console.log(`🔍 QRCodePlaceholder render - sessionId: ${sessionId}, qrUrl: ${qrUrl}`); + + // Only use provided URL from ngrok, don't generate local HTTP URLs + const finalQrUrl = qrUrl; + + useEffect(() => { + const generateQR = async () => { + try { + setIsLoading(true); + if (!finalQrUrl) { + setQrDataUrl(""); + setIsLoading(false); + return; + } + const QRCode = await import('qrcode'); + const dataUrl = await QRCode.toDataURL(finalQrUrl, { + width: 128, + margin: 1, + color: { + dark: '#000000', + light: '#FFFFFF' + } + }); + setQrDataUrl(dataUrl); + } catch (error) { + console.error('Failed to generate QR code:', error); + } finally { + setIsLoading(false); + } + }; + + generateQR(); + }, [finalQrUrl]); + + if (isLoading) { + return ( +
+ +

Generating QR...

+
+ ); + } + + return ( +
+ {qrDataUrl ? ( + QR Code for phone camera connection + ) : ( +
+ +

+ Configure
ngrok first +

+
+ )} +

+ {qrUrl ? ( + + {qrUrl.length > 30 ? `${qrUrl.substring(0, 30)}...` : qrUrl} + + ) : ( + Requires ngrok URL + )} +

+
+ ); +}; + export default WebRTCCameraConfiguration; \ No newline at end of file diff --git a/src/components/webrtc/WebRTCVisualizerPanel.tsx b/src/components/webrtc/WebRTCVisualizerPanel.tsx index 9a9c01d..7b1f809 100644 --- a/src/components/webrtc/WebRTCVisualizerPanel.tsx +++ b/src/components/webrtc/WebRTCVisualizerPanel.tsx @@ -169,7 +169,7 @@ const WebRTCVisualizerPanel: React.FC = ({ console.log("🔄 WebRTC not connected, attempting connection..."); // Configure and connect if not already connected if (!webRTCManager.config.signalingUrl.includes(baseUrl)) { - webRTCManager.config.signalingUrl = `${baseUrl.replace('http', 'ws')}/ws/webrtc`; + webRTCManager.config.signalingUrl = baseUrl.replace('http', 'ws') + '/ws/webrtc'; } await webRTCManager.connect(); } @@ -238,7 +238,7 @@ const WebRTCVisualizerPanel: React.FC = ({ const cameraCount = getAvailableCameras().length; if (cameraCount === 0) return ""; - if (cameraCount === 1) return "flex flex-col gap-3"; // 1 camera: single column + if (cameraCount === 1) return "flex flex-col gap-3 items-center"; // 1 camera: centered if (cameraCount === 2) return "flex flex-col gap-3"; // 2 cameras: stacked if (cameraCount === 3) return "flex flex-col gap-3"; // 3 cameras: single column if (cameraCount === 4) return "grid grid-cols-2 gap-3"; // 4 cameras: 2x2 grid @@ -284,6 +284,7 @@ const WebRTCVisualizerPanel: React.FC = ({ { if (el) { videoRefs.current.set(source.id, el); @@ -319,11 +320,13 @@ const WebRTCVisualizerPanel: React.FC = ({ interface WebRTCCameraDisplayProps { source: UnifiedCameraSource; videoRef: (el: HTMLVideoElement | null) => void; + isSingleCamera?: boolean; } const WebRTCCameraDisplay: React.FC = ({ source, videoRef, + isSingleCamera = false, }) => { const [isPlaying, setIsPlaying] = useState(false); @@ -346,8 +349,15 @@ const WebRTCCameraDisplay: React.FC = ({ console.error(`Video playback error for camera ${source.name}`); }; + // Check if this is a remote camera (phone) waiting for connection + const isRemoteCamera = source.type === 'remote'; + const shouldShowQRMessage = isRemoteCamera && !source.stream && source.status === 'connecting'; + return ( -
+
{source.stream && source.status === 'connected' ? ( @@ -357,7 +367,10 @@ const WebRTCCameraDisplay: React.FC = ({ autoPlay muted playsInline - className="w-full h-full object-cover" + className={cn( + "w-full h-full", + isSingleCamera ? "object-contain" : "object-cover" + )} onPlay={handleVideoPlay} onError={handleVideoError} /> @@ -377,6 +390,13 @@ const WebRTCCameraDisplay: React.FC = ({
+ ) : shouldShowQRMessage ? ( +
+ + + Scan QR in camera config + +
) : (
diff --git a/src/pages/Recording.tsx b/src/pages/Recording.tsx index 7a5ccf2..3213aea 100644 --- a/src/pages/Recording.tsx +++ b/src/pages/Recording.tsx @@ -5,7 +5,7 @@ import { useToast } from "@/hooks/use-toast"; import { ArrowLeft, Square, SkipForward, RotateCcw, Play } from "lucide-react"; import UrdfViewer from "@/components/UrdfViewer"; import UrdfProcessorInitializer from "@/components/UrdfProcessorInitializer"; -import PhoneCameraFeed from "@/components/recording/PhoneCameraFeed"; +import RecordingCameraPanel from "@/components/recording/RecordingCameraPanel"; import { useApi } from "@/contexts/ApiContext"; interface RecordingConfig { @@ -56,10 +56,8 @@ const Recording = () => { ); const [recordingSessionStarted, setRecordingSessionStarted] = useState(false); - // QR Code and camera states + // QR Code and camera states (legacy - kept for potential future use) const [showQrModal, setShowQrModal] = useState(false); - const [sessionId, setSessionId] = useState(""); - const [phoneCameraConnected, setPhoneCameraConnected] = useState(false); // Redirect if no config provided useEffect(() => { @@ -128,50 +126,6 @@ const Recording = () => { }; }, [recordingSessionStarted, recordingConfig, navigate, toast]); - // Generate session ID when component loads - useEffect(() => { - const newSessionId = `session_${Date.now()}_${Math.random() - .toString(36) - .substr(2, 9)}`; - setSessionId(newSessionId); - }, []); - - // Listen for phone camera connections - useEffect(() => { - if (!sessionId) return; - - const connectToPhoneCameraWS = () => { - const ws = new WebSocket(`${wsBaseUrl}/ws/camera/${sessionId}`); - - ws.onopen = () => { - console.log("Phone camera WebSocket connected"); - }; - - ws.onmessage = (event) => { - if (event.data === "camera_connected" && !phoneCameraConnected) { - setPhoneCameraConnected(true); - toast({ - title: "Phone Camera Connected!", - description: "New camera feed detected and connected successfully.", - }); - } - }; - - ws.onclose = () => { - console.log("Phone camera WebSocket disconnected"); - setPhoneCameraConnected(false); - }; - - ws.onerror = (error) => { - console.error("Phone camera WebSocket error:", error); - }; - - return ws; - }; - - const ws = connectToPhoneCameraWS(); - return () => ws.close(); - }, [sessionId, phoneCameraConnected, toast]); const formatTime = (seconds: number): string => { const mins = Math.floor(seconds / 60); @@ -472,9 +426,9 @@ const Recording = () => {
{/* Status and Controls */} -
- {/* Recording Status - takes up 3 columns */} -
+
+ {/* Recording Status - full width */} +
{/* Status header */}
@@ -616,25 +570,29 @@ const Recording = () => {
- {/* Phone Camera Feed - takes up 1 column */} - {phoneCameraConnected && ( -
-

- Phone Camera -

- -
- )}
- {/* URDF Viewer Section */} -
-

- Robot Visualizer -

-
- - + {/* Robot Visualizer and Camera Section */} +
+ {/* Robot Visualizer - flex-1 */} +
+

+ Robot Visualizer +

+
+ + +
+
+ + {/* Camera Panel - fixed width like teleoperation */} +
+

+ Camera Visualizer +

+
+ +
diff --git a/src/types/webrtc.ts b/src/types/webrtc.ts index fb4b461..ab43f97 100644 --- a/src/types/webrtc.ts +++ b/src/types/webrtc.ts @@ -32,7 +32,7 @@ export interface UnifiedCameraSource { } export interface SignalingMessage { - type: 'offer' | 'answer' | 'ice-candidate' | 'camera-list' | 'camera-request' | 'error'; + type: 'offer' | 'answer' | 'ice-candidate' | 'camera-list' | 'camera-request' | 'error' | 'create-session'; sourceId: string; // Camera source ID targetId?: string; // Target peer (for remote) payload: any; // Message-specific data @@ -84,7 +84,10 @@ export interface WebRTCManagerEvents { 'camera-connected': (sourceId: string, stream: MediaStream) => void; 'camera-disconnected': (sourceId: string) => void; 'camera-error': (sourceId: string, error: Error) => void; + 'camera-updated': (sourceId: string, updatedSource: UnifiedCameraSource) => void; 'stats-updated': (sourceId: string, stats: RTCStatsReport) => void; + 'connected': () => void; + 'disconnected': () => void; } export interface WebRTCManagerConfig { diff --git a/src/utils/webrtc/WebRTCManager.ts b/src/utils/webrtc/WebRTCManager.ts index c8da5ff..ef75d87 100644 --- a/src/utils/webrtc/WebRTCManager.ts +++ b/src/utils/webrtc/WebRTCManager.ts @@ -12,7 +12,7 @@ import { export class WebRTCManager extends BrowserEventEmitter { public config: WebRTCManagerConfig; - private signalingSocket: WebSocket | null = null; + private signalingSocket: any = null; // Socket.IO client private sources = new Map(); private isConnected = false; private reconnectAttempts = 0; @@ -35,6 +35,7 @@ export class WebRTCManager extends BrowserEventEmitter { try { console.log('🔗 Connecting to WebRTC signaling server:', this.config.signalingUrl); + // Use native WebSocket instead of Socket.IO this.signalingSocket = new WebSocket(this.config.signalingUrl); this.signalingSocket.onopen = () => { @@ -44,10 +45,6 @@ export class WebRTCManager extends BrowserEventEmitter { this.emit('connected'); }; - this.signalingSocket.onmessage = (event) => { - this.handleSignalingMessage(JSON.parse(event.data)); - }; - this.signalingSocket.onclose = () => { console.log('🔌 WebRTC signaling disconnected'); this.isConnected = false; @@ -60,10 +57,19 @@ export class WebRTCManager extends BrowserEventEmitter { } }; - this.signalingSocket.onerror = (error) => { + this.signalingSocket.onerror = (error: any) => { console.error('❌ WebRTC signaling error:', error); this.emit('error', error); }; + + this.signalingSocket.onmessage = (event) => { + try { + const message = JSON.parse(event.data); + this.handleSignalingMessage(message); + } catch (error) { + console.error('❌ Failed to parse signaling message:', error); + } + }; } catch (error) { console.error('Failed to connect to signaling server:', error); @@ -153,16 +159,91 @@ export class WebRTCManager extends BrowserEventEmitter { } } - // ==================== Remote Camera Management (Future) ==================== + // ==================== Remote Camera Management ==================== - async addRemoteCamera(peerId: string, name: string): Promise { - const sourceId = `remote_${peerId}_${Date.now()}`; + async addRemoteCamera(sessionId: string, name: string, quality: CameraQuality = 'medium'): Promise { + // Check if a camera with this sessionId already exists + const existingSource = Array.from(this.sources.values()).find(source => + source.type === 'remote' && source.deviceId === sessionId + ); + + if (existingSource) { + console.log(`📡 Remote camera already exists: ${name} (${sessionId}), updating instead`); + // Update existing camera properties if needed + existingSource.name = name; + const constraints = CAMERA_CONSTRAINTS[quality]; + existingSource.width = constraints.width; + existingSource.height = constraints.height; + existingSource.fps = constraints.fps; + + // Emit camera-updated event + this.emit('camera-updated', existingSource.id, existingSource); + + // Re-request QR URL for existing session + if (this.signalingSocket && this.isConnected) { + console.log(`🔄 Re-requesting QR URL for existing session: ${sessionId}`); + this.sendSignalingMessage({ + type: 'create-session', + sourceId: sessionId, + payload: { sessionId, name }, + timestamp: Date.now() + }); + } + + return existingSource.id; + } + + const sourceId = `remote_${sessionId}_${Date.now()}`; - console.log(`📡 Adding remote camera: ${name} (${peerId})`); + console.log(`📡 Adding new remote camera: ${name} (${sessionId})`); - // This will be implemented for remote cameras - // For now, return placeholder - throw new Error('Remote cameras not implemented yet'); + try { + // Create peer connection for remote camera + const peerConnection = new RTCPeerConnection(this.config.rtcConfiguration); + + // Setup peer connection event handlers + this.setupPeerConnectionHandlers(peerConnection, sourceId); + + // Get constraints for the quality level + const constraints = CAMERA_CONSTRAINTS[quality]; + + // Create camera source for remote camera (will get stream later from phone) + const source: UnifiedCameraSource = { + id: sourceId, + type: 'remote', + name, + deviceId: sessionId, // Use session ID as device identifier + stream: undefined, // Will be set when phone connects + peerConnection, + status: 'connecting', + width: constraints.width, + height: constraints.height, + fps: constraints.fps, + lastSeen: new Date().toISOString(), + errorCount: 0 + }; + + this.sources.set(sourceId, source); + + console.log(`✅ Remote camera created: ${name} (${sourceId}), waiting for phone connection`); + this.emit('camera-added', source); + + // Send signaling message to create session via WebSocket + if (this.signalingSocket && this.isConnected) { + this.sendSignalingMessage({ + type: 'create-session', + sourceId: sessionId, + payload: { sessionId, name }, + timestamp: Date.now() + }); + } + + return sourceId; + + } catch (error) { + console.error(`❌ Failed to add remote camera ${name}:`, error); + throw error; + } } // ==================== Camera Operations ==================== @@ -256,6 +337,74 @@ export class WebRTCManager extends BrowserEventEmitter { return offer; } + async handleIncomingOffer(sourceId: string, offer: RTCSessionDescriptionInit): Promise { + console.log('📥 Handling incoming offer from mobile phone:', sourceId); + console.log('📥 Offer details:', offer); + console.log('🔧 Current sources:', Array.from(this.sources.keys())); + + // Find the remote camera source for this session + let source = this.sources.get(sourceId); + + if (!source) { + // If source doesn't exist, find by device ID (session ID) + for (const [, src] of this.sources.entries()) { + if (src.deviceId === sourceId && src.type === 'remote') { + source = src; + break; + } + } + } + + if (!source) { + console.warn(`⚠️ No remote camera source found for session: ${sourceId}`); + return; + } + + try { + // Create peer connection if it doesn't exist + if (!source.peerConnection) { + source.peerConnection = new RTCPeerConnection(this.config.rtcConfiguration); + this.setupPeerConnectionHandlers(source.peerConnection, sourceId); + } + + // Set remote description (the offer from mobile phone) + await source.peerConnection.setRemoteDescription(offer); + console.log('✅ Set remote description from mobile phone'); + + // Create answer + const answer = await source.peerConnection.createAnswer(); + await source.peerConnection.setLocalDescription(answer); + console.log('✅ Created answer for mobile phone'); + + // Send answer back to mobile phone + if (this.signalingSocket && this.isConnected) { + const answerMessage = { + type: 'answer', + sourceId: sourceId, + payload: answer, + timestamp: Date.now() + }; + + console.log('📤 Sending answer to mobile phone:', answerMessage); + this.signalingSocket.send(JSON.stringify(answerMessage)); + console.log('✅ Answer sent successfully'); + } else { + console.error('❌ Cannot send answer: WebSocket not connected'); + console.log('🔧 Socket state:', this.signalingSocket ? 'exists' : 'null'); + console.log('🔧 Is connected:', this.isConnected); + } + + // Update source status + source.status = 'connecting'; + this.emit('camera-updated', source.id, source); + + } catch (error) { + console.error('❌ Error handling incoming offer:', error); + source.status = 'error'; + this.emit('camera-updated', source.id, source); + } + } + async handleStreamingAnswer(sourceId: string, answer: RTCSessionDescriptionInit): Promise { const source = this.sources.get(sourceId); if (!source || !source.peerConnection) { @@ -311,6 +460,19 @@ export class WebRTCManager extends BrowserEventEmitter { } }; + // Handle incoming video tracks from mobile phone + peerConnection.ontrack = (event) => { + console.log('📹 Received video track from mobile phone:', sourceId); + const source = this.sources.get(sourceId); + if (source && event.streams[0]) { + source.stream = event.streams[0]; + source.status = 'connected'; + console.log('✅ Video stream assigned to camera source:', sourceId); + this.emit('camera-connected', sourceId, source.stream); + this.emit('camera-updated', sourceId, source); + } + }; + peerConnection.onicecandidateerror = (event) => { console.error(`ICE candidate error for ${sourceId}:`, event); const source = this.sources.get(sourceId); @@ -321,15 +483,44 @@ export class WebRTCManager extends BrowserEventEmitter { }; } - private handleSignalingMessage(message: SignalingMessage): void { + private handleSignalingMessage(message: any): void { console.log('📨 Received signaling message:', message.type, 'for source:', message.sourceId); switch (message.type) { + case 'session-created': + console.log('✅ Session created:', message); + // Handle session creation confirmation + const sessionId = message.sourceId; + const qrCodeUrl = message.payload?.qrCodeUrl; + + // Update remote camera with QR code info + this.sources.forEach(source => { + if (source.deviceId === sessionId && source.type === 'remote') { + source.status = 'connecting'; + this.emit('camera-updated', source.id, source); + } + }); + + // Emit session created event with QR URL and session info + this.emit('session-created', sessionId, qrCodeUrl, message.payload); + break; + case 'phone-connected': + case 'external-device-connected': + console.log('🌐 External device connected to session:', message.sourceId); + // Update the corresponding camera source status + this.sources.forEach(source => { + if (source.deviceId === message.sourceId) { + source.status = 'connected'; + this.emit('camera-connected', source.id, source.stream || new MediaStream()); + } + }); + break; case 'offer': // Handle incoming offers (for remote cameras) + this.handleIncomingOffer(message.sourceId, message.payload); break; case 'answer': - this.handleStreamingAnswer(message.sourceId, message.payload.sdp); + this.handleStreamingAnswer(message.sourceId, message.payload); break; case 'ice-candidate': this.addIceCandidate(message.sourceId, message.payload.candidate); @@ -363,7 +554,7 @@ export class WebRTCManager extends BrowserEventEmitter { } }, this.config.statsInterval); - this.statsIntervals.set(sourceId, interval); + this.statsIntervals.set(sourceId, interval as unknown as number); } private stopStatsCollection(sourceId: string): void {