From 87edc9e33622e1ecf351e8ef2a2d5f11679ecb90 Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Mon, 1 Dec 2025 19:30:28 +0000 Subject: [PATCH 1/6] Add private voice agent support with server-side token generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create apiVoice.ts client to fetch conversation tokens from server - Update RealtimeVoiceSession.tsx and .web.tsx to use conversationToken instead of hardcoded agentId for self-hosted deployments - Enables users to configure their own ElevenLabs agent 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sources/realtime/RealtimeVoiceSession.tsx | 17 ++++-- sources/realtime/RealtimeVoiceSession.web.tsx | 15 ++++- sources/sync/apiVoice.ts | 55 +++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 sources/sync/apiVoice.ts diff --git a/sources/realtime/RealtimeVoiceSession.tsx b/sources/realtime/RealtimeVoiceSession.tsx index 7b05acd24..2e5d1e790 100644 --- a/sources/realtime/RealtimeVoiceSession.tsx +++ b/sources/realtime/RealtimeVoiceSession.tsx @@ -4,6 +4,7 @@ import { registerVoiceSession } from './RealtimeSession'; import { storage } from '@/sync/storage'; import { realtimeClientTools } from './realtimeClientTools'; import { getElevenLabsCodeFromPreference } from '@/constants/Languages'; +import { fetchVoiceToken } from '@/sync/apiVoice'; import type { VoiceSession, VoiceSessionConfig } from './types'; // Static reference to the conversation hook instance @@ -20,14 +21,22 @@ class RealtimeVoiceSessionImpl implements VoiceSession { try { storage.getState().setRealtimeStatus('connecting'); - + + // Fetch conversation token from server (private agent flow) + const tokenResponse = await fetchVoiceToken(); + if (!tokenResponse.allowed || !tokenResponse.token) { + console.error('Voice not allowed or no token:', tokenResponse.error); + storage.getState().setRealtimeStatus('error'); + return; + } + // Get user's preferred language for voice assistant const userLanguagePreference = storage.getState().settings.voiceAssistantLanguage; const elevenLabsLanguage = getElevenLabsCodeFromPreference(userLanguagePreference); - - // Use hardcoded agent ID for Eleven Labs + + // Use conversation token from server (private agent flow) await conversationInstance.startSession({ - agentId: __DEV__ ? 'agent_7801k2c0r5hjfraa1kdbytpvs6yt' : 'agent_6701k211syvvegba4kt7m68nxjmw', + conversationToken: tokenResponse.token, // Pass session ID and initial context as dynamic variables dynamicVariables: { sessionId: config.sessionId, diff --git a/sources/realtime/RealtimeVoiceSession.web.tsx b/sources/realtime/RealtimeVoiceSession.web.tsx index b0ef3b60b..26daa752a 100644 --- a/sources/realtime/RealtimeVoiceSession.web.tsx +++ b/sources/realtime/RealtimeVoiceSession.web.tsx @@ -4,6 +4,7 @@ import { registerVoiceSession } from './RealtimeSession'; import { storage } from '@/sync/storage'; import { realtimeClientTools } from './realtimeClientTools'; import { getElevenLabsCodeFromPreference } from '@/constants/Languages'; +import { fetchVoiceToken } from '@/sync/apiVoice'; import type { VoiceSession, VoiceSessionConfig } from './types'; // Static reference to the conversation hook instance @@ -31,13 +32,21 @@ class RealtimeVoiceSessionImpl implements VoiceSession { } + // Fetch conversation token from server (private agent flow) + const tokenResponse = await fetchVoiceToken(); + if (!tokenResponse.allowed || !tokenResponse.token) { + console.error('Voice not allowed or no token:', tokenResponse.error); + storage.getState().setRealtimeStatus('error'); + return; + } + // Get user's preferred language for voice assistant const userLanguagePreference = storage.getState().settings.voiceAssistantLanguage; const elevenLabsLanguage = getElevenLabsCodeFromPreference(userLanguagePreference); - - // Use hardcoded agent ID for Eleven Labs + + // Use conversation token from server (private agent flow) const conversationId = await conversationInstance.startSession({ - agentId: __DEV__ ? 'agent_7801k2c0r5hjfraa1kdbytpvs6yt' : 'agent_6701k211syvvegba4kt7m68nxjmw', + conversationToken: tokenResponse.token, connectionType: 'webrtc', // Use WebRTC for better performance // Pass session ID and initial context as dynamic variables dynamicVariables: { diff --git a/sources/sync/apiVoice.ts b/sources/sync/apiVoice.ts new file mode 100644 index 000000000..c87b4e8ce --- /dev/null +++ b/sources/sync/apiVoice.ts @@ -0,0 +1,55 @@ +/** + * API functions for voice assistant integration. + * + * Fetches conversation tokens from the server for ElevenLabs integration. + * The server handles authentication with ElevenLabs API, keeping credentials secure. + */ + +import { getServerUrl } from '@/sync/serverConfig'; +import { getCurrentAuth } from '@/auth/AuthContext'; + +export interface VoiceTokenResponse { + allowed: boolean; + token?: string; + agentId?: string; + error?: string; +} + +/** + * Fetch a conversation token from the server for ElevenLabs voice sessions. + * + * This uses the private agent flow where: + * 1. Server holds the ELEVENLABS_API_KEY and ELEVENLABS_AGENT_ID + * 2. Server fetches a short-lived conversation token from ElevenLabs + * 3. Client uses this token to establish WebRTC connection + * + * @returns Object with allowed status, and if allowed, the token and agentId + * @throws Error if not authenticated or network failure + */ +export async function fetchVoiceToken(): Promise { + const auth = getCurrentAuth(); + if (!auth?.credentials?.token) { + throw new Error('Not authenticated'); + } + + const serverUrl = getServerUrl(); + const response = await fetch(`${serverUrl}/v1/voice/token`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${auth.credentials.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({}) + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); + return { + allowed: false, + error: errorData.error || `Server error: ${response.status}` + }; + } + + const data = await response.json(); + return data as VoiceTokenResponse; +} From dd7da16dd88c95925fb974adc732f53f53a1beb0 Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Mon, 1 Dec 2025 20:11:21 +0000 Subject: [PATCH 2/6] Add ElevenLabs custom agent configuration UI in voice settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add settings fields for elevenLabsUseCustomAgent, elevenLabsAgentId, elevenLabsApiKey - Create UI with toggle to enable custom agent and input fields for credentials - Update apiVoice.ts to pass custom credentials to server when enabled - Add translations for all supported languages (en, es, pl, ru, zh-Hans, ca, pt) - Update settings tests with new fields 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sources/app/(app)/settings/voice.tsx | 154 +++++++++++++++++++++++++-- sources/sync/apiVoice.ts | 34 +++++- sources/sync/settings.spec.ts | 75 +++++++------ sources/sync/settings.ts | 7 ++ sources/text/_default.ts | 19 +++- sources/text/translations/ca.ts | 19 +++- sources/text/translations/es.ts | 19 +++- sources/text/translations/pl.ts | 19 +++- sources/text/translations/pt.ts | 19 +++- sources/text/translations/ru.ts | 19 +++- sources/text/translations/zh-Hans.ts | 19 +++- 11 files changed, 349 insertions(+), 54 deletions(-) diff --git a/sources/app/(app)/settings/voice.tsx b/sources/app/(app)/settings/voice.tsx index 6b34376c0..b5af314fd 100644 --- a/sources/app/(app)/settings/voice.tsx +++ b/sources/app/(app)/settings/voice.tsx @@ -1,25 +1,69 @@ +import React, { useState, useCallback, memo } from 'react'; +import { View, TextInput } from 'react-native'; import { Ionicons } from '@expo/vector-icons'; import { useRouter } from 'expo-router'; import { Item } from '@/components/Item'; import { ItemGroup } from '@/components/ItemGroup'; import { ItemList } from '@/components/ItemList'; -import { useSettingMutable } from '@/sync/storage'; -import { useUnistyles } from 'react-native-unistyles'; +import { Text } from '@/components/StyledText'; +import { RoundButton } from '@/components/RoundButton'; +import { Switch } from '@/components/Switch'; +import { Modal } from '@/modal'; +import { useSettingMutable, storage } from '@/sync/storage'; +import { StyleSheet, useUnistyles } from 'react-native-unistyles'; import { findLanguageByCode, getLanguageDisplayName, LANGUAGES } from '@/constants/Languages'; import { t } from '@/text'; +import { Typography } from '@/constants/Typography'; +import { layout } from '@/components/layout'; -export default function VoiceSettingsScreen() { +function VoiceSettingsScreen() { const { theme } = useUnistyles(); const router = useRouter(); const [voiceAssistantLanguage] = useSettingMutable('voiceAssistantLanguage'); - + const [useCustomAgent, setUseCustomAgent] = useSettingMutable('elevenLabsUseCustomAgent'); + const [savedAgentId] = useSettingMutable('elevenLabsAgentId'); + const [savedApiKey] = useSettingMutable('elevenLabsApiKey'); + + // Local state for input fields + const [agentIdInput, setAgentIdInput] = useState(savedAgentId || ''); + const [apiKeyInput, setApiKeyInput] = useState(savedApiKey || ''); + // Find current language or default to first option const currentLanguage = findLanguageByCode(voiceAssistantLanguage) || LANGUAGES[0]; - + + const handleToggleCustomAgent = useCallback((value: boolean) => { + setUseCustomAgent(value); + }, [setUseCustomAgent]); + + const handleSaveCredentials = useCallback(async () => { + if (!agentIdInput.trim() || !apiKeyInput.trim()) { + Modal.alert(t('common.error'), t('settingsVoice.credentialsRequired')); + return; + } + + // Save to settings (synced across devices) + storage.getState().applySettingsLocal({ + elevenLabsAgentId: agentIdInput.trim(), + elevenLabsApiKey: apiKeyInput.trim(), + }); + + Modal.alert(t('common.success'), t('settingsVoice.credentialsSaved')); + }, [agentIdInput, apiKeyInput]); + + const getAgentStatusText = () => { + if (!useCustomAgent) { + return t('settingsVoice.usingDefaultAgent'); + } + if (savedAgentId) { + return t('settingsVoice.usingCustomAgent'); + } + return t('settingsVoice.credentialsRequired'); + }; + return ( {/* Language Settings */} - @@ -32,6 +76,102 @@ export default function VoiceSettingsScreen() { /> + {/* ElevenLabs Configuration */} + + } + showChevron={false} + rightElement={ + + } + /> + + + + {/* Custom Agent Credentials - only show when custom agent is enabled */} + {useCustomAgent && ( + + + {t('settingsVoice.agentId').toUpperCase()} + + + {t('settingsVoice.apiKey').toUpperCase()} + + + + + + + + )} + ); -} \ No newline at end of file +} + +export default memo(VoiceSettingsScreen); + +const styles = StyleSheet.create((theme) => ({ + contentContainer: { + backgroundColor: theme.colors.surface, + paddingHorizontal: 16, + paddingVertical: 12, + width: '100%', + maxWidth: layout.maxWidth, + alignSelf: 'center', + }, + labelText: { + ...Typography.default('semiBold'), + fontSize: 12, + color: theme.colors.textSecondary, + textTransform: 'uppercase', + letterSpacing: 0.5, + marginBottom: 8, + marginTop: 8, + }, + textInput: { + padding: 12, + borderRadius: 8, + marginBottom: 8, + ...Typography.mono(), + fontSize: 14, + }, + buttonContainer: { + marginTop: 12, + }, +})); diff --git a/sources/sync/apiVoice.ts b/sources/sync/apiVoice.ts index c87b4e8ce..f90a4e4a9 100644 --- a/sources/sync/apiVoice.ts +++ b/sources/sync/apiVoice.ts @@ -3,10 +3,15 @@ * * Fetches conversation tokens from the server for ElevenLabs integration. * The server handles authentication with ElevenLabs API, keeping credentials secure. + * + * Supports two modes: + * 1. Default: Server uses its own ElevenLabs credentials (production) + * 2. Custom: Client provides their own ElevenLabs agent ID and API key */ import { getServerUrl } from '@/sync/serverConfig'; import { getCurrentAuth } from '@/auth/AuthContext'; +import { storage } from '@/sync/storage'; export interface VoiceTokenResponse { allowed: boolean; @@ -15,14 +20,24 @@ export interface VoiceTokenResponse { error?: string; } +export interface VoiceTokenRequest { + revenueCatPublicKey?: string; + // Custom ElevenLabs credentials (when user provides their own) + customAgentId?: string; + customApiKey?: string; +} + /** * Fetch a conversation token from the server for ElevenLabs voice sessions. * * This uses the private agent flow where: - * 1. Server holds the ELEVENLABS_API_KEY and ELEVENLABS_AGENT_ID + * 1. Server holds the ELEVENLABS_API_KEY and ELEVENLABS_AGENT_ID (or uses user-provided ones) * 2. Server fetches a short-lived conversation token from ElevenLabs * 3. Client uses this token to establish WebRTC connection * + * If the user has configured custom ElevenLabs credentials in settings, + * those will be passed to the server to use instead of the default production agent. + * * @returns Object with allowed status, and if allowed, the token and agentId * @throws Error if not authenticated or network failure */ @@ -32,6 +47,21 @@ export async function fetchVoiceToken(): Promise { throw new Error('Not authenticated'); } + // Check if user has custom ElevenLabs credentials configured + const settings = storage.getState().settings; + const useCustomAgent = settings.elevenLabsUseCustomAgent; + const customAgentId = settings.elevenLabsAgentId; + const customApiKey = settings.elevenLabsApiKey; + + // Build request body + const requestBody: VoiceTokenRequest = {}; + + // Include custom credentials if user has enabled custom agent + if (useCustomAgent && customAgentId && customApiKey) { + requestBody.customAgentId = customAgentId; + requestBody.customApiKey = customApiKey; + } + const serverUrl = getServerUrl(); const response = await fetch(`${serverUrl}/v1/voice/token`, { method: 'POST', @@ -39,7 +69,7 @@ export async function fetchVoiceToken(): Promise { 'Authorization': `Bearer ${auth.credentials.token}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({}) + body: JSON.stringify(requestBody) }); if (!response.ok) { diff --git a/sources/sync/settings.spec.ts b/sources/sync/settings.spec.ts index 5f080525d..386279b24 100644 --- a/sources/sync/settings.spec.ts +++ b/sources/sync/settings.spec.ts @@ -102,7 +102,7 @@ describe('settings', () => { inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, - avatarStyle: 'gradient', + avatarStyle: 'brutalist', showFlavorIcons: false, compactSessionView: false, hideInactiveSessions: false, @@ -114,28 +114,16 @@ describe('settings', () => { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; const delta: Partial = { viewInline: true }; expect(applySettings(currentSettings, delta)).toEqual({ + ...settingsDefaults, viewInline: true, - expandTodos: true, - showLineNumbers: true, - showLineNumbersInToolViews: false, - wrapLinesInDiffs: false, - analyticsOptOut: false, - inferenceOpenAIKey: null, - experiments: false, - alwaysShowContextSize: false, - avatarStyle: 'brutalist', - showFlavorIcons: false, - compactSessionView: false, - hideInactiveSessions: false, - reviewPromptAnswered: false, - reviewPromptLikedApp: null, - voiceAssistantLanguage: null, - preferredLanguage: null, }); }); @@ -150,7 +138,7 @@ describe('settings', () => { inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, - avatarStyle: 'gradient', + avatarStyle: 'brutalist', showFlavorIcons: false, compactSessionView: false, hideInactiveSessions: false, @@ -162,6 +150,9 @@ describe('settings', () => { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; const delta: Partial = {}; expect(applySettings(currentSettings, delta)).toEqual({ @@ -181,7 +172,7 @@ describe('settings', () => { inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, - avatarStyle: 'gradient', + avatarStyle: 'brutalist', showFlavorIcons: false, compactSessionView: false, hideInactiveSessions: false, @@ -193,28 +184,16 @@ describe('settings', () => { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; const delta: Partial = { viewInline: false }; expect(applySettings(currentSettings, delta)).toEqual({ + ...settingsDefaults, viewInline: false, - expandTodos: true, - showLineNumbers: true, - showLineNumbersInToolViews: false, - wrapLinesInDiffs: false, - analyticsOptOut: false, - inferenceOpenAIKey: null, - experiments: false, - alwaysShowContextSize: false, - avatarStyle: 'brutalist', - showFlavorIcons: false, - compactSessionView: false, - hideInactiveSessions: false, - reviewPromptAnswered: false, - reviewPromptLikedApp: null, - voiceAssistantLanguage: null, - preferredLanguage: null, }); }); @@ -229,7 +208,7 @@ describe('settings', () => { inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, - avatarStyle: 'gradient', + avatarStyle: 'brutalist', showFlavorIcons: false, compactSessionView: false, hideInactiveSessions: false, @@ -241,6 +220,9 @@ describe('settings', () => { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; expect(applySettings(currentSettings, {})).toEqual({ ...settingsDefaults, @@ -274,7 +256,7 @@ describe('settings', () => { inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, - avatarStyle: 'gradient', + avatarStyle: 'brutalist', showFlavorIcons: false, compactSessionView: false, hideInactiveSessions: false, @@ -286,6 +268,9 @@ describe('settings', () => { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; const delta: any = { viewInline: false, @@ -320,15 +305,29 @@ describe('settings', () => { it('should have correct default values', () => { expect(settingsDefaults).toEqual({ viewInline: false, + inferenceOpenAIKey: null, expandTodos: true, showLineNumbers: true, showLineNumbersInToolViews: false, wrapLinesInDiffs: false, analyticsOptOut: false, - inferenceOpenAIKey: null, experiments: false, alwaysShowContextSize: false, + avatarStyle: 'brutalist', + showFlavorIcons: false, + compactSessionView: false, hideInactiveSessions: false, + reviewPromptAnswered: false, + reviewPromptLikedApp: null, + voiceAssistantLanguage: null, + preferredLanguage: null, + recentMachinePaths: [], + lastUsedAgent: null, + lastUsedPermissionMode: null, + lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }); }); diff --git a/sources/sync/settings.ts b/sources/sync/settings.ts index e0a9f2d28..6b2ee3cc7 100644 --- a/sources/sync/settings.ts +++ b/sources/sync/settings.ts @@ -29,6 +29,10 @@ export const SettingsSchema = z.object({ lastUsedAgent: z.string().nullable().describe('Last selected agent type for new sessions'), lastUsedPermissionMode: z.string().nullable().describe('Last selected permission mode for new sessions'), lastUsedModelMode: z.string().nullable().describe('Last selected model mode for new sessions'), + // ElevenLabs voice assistant configuration + elevenLabsUseCustomAgent: z.boolean().describe('Whether to use custom ElevenLabs agent instead of production default'), + elevenLabsAgentId: z.string().nullable().describe('Custom ElevenLabs agent ID (when useCustomAgent is true)'), + elevenLabsApiKey: z.string().nullable().describe('Custom ElevenLabs API key (when useCustomAgent is true)'), }); // @@ -72,6 +76,9 @@ export const settingsDefaults: Settings = { lastUsedAgent: null, lastUsedPermissionMode: null, lastUsedModelMode: null, + elevenLabsUseCustomAgent: false, + elevenLabsAgentId: null, + elevenLabsApiKey: null, }; Object.freeze(settingsDefaults); diff --git a/sources/text/_default.ts b/sources/text/_default.ts index 8a4936322..d6efaa78a 100644 --- a/sources/text/_default.ts +++ b/sources/text/_default.ts @@ -536,7 +536,24 @@ export const en = { title: 'Languages', footer: ({ count }: { count: number }) => `${count} ${plural({ count, singular: 'language', plural: 'languages' })} available`, autoDetect: 'Auto-detect', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'ElevenLabs Configuration', + elevenLabsDescription: 'Configure your ElevenLabs voice agent. Use the default production agent or connect your own.', + useCustomAgent: 'Use Custom Agent', + useCustomAgentSubtitle: 'Connect your own ElevenLabs agent instead of the default', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'Your ElevenLabs agent ID', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'Your ElevenLabs API key', + saveCredentials: 'Save Credentials', + credentialsSaved: 'ElevenLabs credentials saved', + credentialsRequired: 'Both Agent ID and API Key are required', + currentAgentId: 'Current Agent ID', + usingDefaultAgent: 'Using default production agent', + usingCustomAgent: 'Using custom agent', }, settingsAccount: { diff --git a/sources/text/translations/ca.ts b/sources/text/translations/ca.ts index e11e2e6d9..f07781304 100644 --- a/sources/text/translations/ca.ts +++ b/sources/text/translations/ca.ts @@ -536,7 +536,24 @@ export const ca: TranslationStructure = { title: 'Idiomes', footer: ({ count }: { count: number }) => `${count} ${plural({ count, singular: 'idioma', plural: 'idiomes' })} disponibles`, autoDetect: 'Detecta automàticament', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'Configuració d\'ElevenLabs', + elevenLabsDescription: 'Configura el teu agent de veu d\'ElevenLabs. Utilitza l\'agent de producció per defecte o connecta el teu propi.', + useCustomAgent: 'Utilitza Agent Personalitzat', + useCustomAgentSubtitle: 'Connecta el teu propi agent d\'ElevenLabs en lloc del predeterminat', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'El teu ID d\'agent d\'ElevenLabs', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'La teva clau API d\'ElevenLabs', + saveCredentials: 'Desa les Credencials', + credentialsSaved: 'Credencials d\'ElevenLabs desades', + credentialsRequired: 'Tant l\'Agent ID com l\'API Key són obligatoris', + currentAgentId: 'Agent ID Actual', + usingDefaultAgent: 'Utilitzant l\'agent de producció per defecte', + usingCustomAgent: 'Utilitzant agent personalitzat', }, settingsAccount: { diff --git a/sources/text/translations/es.ts b/sources/text/translations/es.ts index 58d4b584a..0b3f4e792 100644 --- a/sources/text/translations/es.ts +++ b/sources/text/translations/es.ts @@ -536,7 +536,24 @@ export const es: TranslationStructure = { title: 'Idiomas', footer: ({ count }: { count: number }) => `${count} ${plural({ count, singular: 'idioma', plural: 'idiomas' })} disponibles`, autoDetect: 'Detectar automáticamente', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'Configuración de ElevenLabs', + elevenLabsDescription: 'Configura tu agente de voz de ElevenLabs. Usa el agente de producción predeterminado o conecta el tuyo propio.', + useCustomAgent: 'Usar Agente Personalizado', + useCustomAgentSubtitle: 'Conecta tu propio agente de ElevenLabs en lugar del predeterminado', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'Tu ID de agente de ElevenLabs', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'Tu clave API de ElevenLabs', + saveCredentials: 'Guardar Credenciales', + credentialsSaved: 'Credenciales de ElevenLabs guardadas', + credentialsRequired: 'Tanto el Agent ID como el API Key son obligatorios', + currentAgentId: 'Agent ID Actual', + usingDefaultAgent: 'Usando el agente de producción predeterminado', + usingCustomAgent: 'Usando agente personalizado', }, settingsAccount: { diff --git a/sources/text/translations/pl.ts b/sources/text/translations/pl.ts index 6572a970f..593b8104d 100644 --- a/sources/text/translations/pl.ts +++ b/sources/text/translations/pl.ts @@ -546,7 +546,24 @@ export const pl: TranslationStructure = { title: 'Języki', footer: ({ count }: { count: number }) => `Dostępnych ${count} ${plural({ count, one: 'język', few: 'języki', many: 'języków' })}`, autoDetect: 'Automatyczne wykrywanie', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'Konfiguracja ElevenLabs', + elevenLabsDescription: 'Skonfiguruj swojego agenta głosowego ElevenLabs. Użyj domyślnego agenta produkcyjnego lub połącz własnego.', + useCustomAgent: 'Użyj Niestandardowego Agenta', + useCustomAgentSubtitle: 'Połącz własnego agenta ElevenLabs zamiast domyślnego', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'Twój ID agenta ElevenLabs', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'Twój klucz API ElevenLabs', + saveCredentials: 'Zapisz Dane Uwierzytelniające', + credentialsSaved: 'Dane uwierzytelniające ElevenLabs zapisane', + credentialsRequired: 'Zarówno Agent ID jak i API Key są wymagane', + currentAgentId: 'Aktualny Agent ID', + usingDefaultAgent: 'Używanie domyślnego agenta produkcyjnego', + usingCustomAgent: 'Używanie niestandardowego agenta', }, settingsAccount: { diff --git a/sources/text/translations/pt.ts b/sources/text/translations/pt.ts index 2d8fce3cc..7927341b8 100644 --- a/sources/text/translations/pt.ts +++ b/sources/text/translations/pt.ts @@ -536,7 +536,24 @@ export const pt: TranslationStructure = { title: 'Idiomas', footer: ({ count }: { count: number }) => `${count} ${plural({ count, singular: 'idioma', plural: 'idiomas' })} disponíveis`, autoDetect: 'Detectar automaticamente', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'Configuração do ElevenLabs', + elevenLabsDescription: 'Configure seu agente de voz do ElevenLabs. Use o agente de produção padrão ou conecte o seu próprio.', + useCustomAgent: 'Usar Agente Personalizado', + useCustomAgentSubtitle: 'Conecte seu próprio agente do ElevenLabs em vez do padrão', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'Seu ID de agente do ElevenLabs', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'Sua chave API do ElevenLabs', + saveCredentials: 'Salvar Credenciais', + credentialsSaved: 'Credenciais do ElevenLabs salvas', + credentialsRequired: 'Tanto o Agent ID quanto o API Key são obrigatórios', + currentAgentId: 'Agent ID Atual', + usingDefaultAgent: 'Usando o agente de produção padrão', + usingCustomAgent: 'Usando agente personalizado', }, settingsAccount: { diff --git a/sources/text/translations/ru.ts b/sources/text/translations/ru.ts index c6d9103b0..3a4709c38 100644 --- a/sources/text/translations/ru.ts +++ b/sources/text/translations/ru.ts @@ -546,7 +546,24 @@ export const ru: TranslationStructure = { title: 'Языки', footer: ({ count }: { count: number }) => `Доступно ${count} ${plural({ count, one: 'язык', few: 'языка', many: 'языков' })}`, autoDetect: 'Автоопределение', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'Настройка ElevenLabs', + elevenLabsDescription: 'Настройте своего голосового агента ElevenLabs. Используйте стандартный рабочий агент или подключите свой собственный.', + useCustomAgent: 'Использовать Пользовательский Агент', + useCustomAgentSubtitle: 'Подключите своего собственного агента ElevenLabs вместо стандартного', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: 'Ваш ID агента ElevenLabs', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: 'Ваш ключ API ElevenLabs', + saveCredentials: 'Сохранить Учётные Данные', + credentialsSaved: 'Учётные данные ElevenLabs сохранены', + credentialsRequired: 'Требуется указать и Agent ID, и API Key', + currentAgentId: 'Текущий Agent ID', + usingDefaultAgent: 'Используется стандартный рабочий агент', + usingCustomAgent: 'Используется пользовательский агент', }, settingsAccount: { diff --git a/sources/text/translations/zh-Hans.ts b/sources/text/translations/zh-Hans.ts index 20dff14cb..fec2032a4 100644 --- a/sources/text/translations/zh-Hans.ts +++ b/sources/text/translations/zh-Hans.ts @@ -538,7 +538,24 @@ export const zhHans: TranslationStructure = { title: '语言', footer: ({ count }: { count: number }) => `${count} 种可用语言`, autoDetect: '自动检测', - } + }, + // ElevenLabs configuration + elevenLabsTitle: 'ElevenLabs 配置', + elevenLabsDescription: '配置您的 ElevenLabs 语音代理。使用默认生产代理或连接您自己的代理。', + useCustomAgent: '使用自定义代理', + useCustomAgentSubtitle: '连接您自己的 ElevenLabs 代理而不是默认代理', + agentId: 'Agent ID', + agentIdPlaceholder: 'agent_xxxxx', + agentIdSubtitle: '您的 ElevenLabs 代理 ID', + apiKey: 'API Key', + apiKeyPlaceholder: 'sk_xxxxx', + apiKeySubtitle: '您的 ElevenLabs API 密钥', + saveCredentials: '保存凭据', + credentialsSaved: 'ElevenLabs 凭据已保存', + credentialsRequired: 'Agent ID 和 API Key 都是必需的', + currentAgentId: '当前 Agent ID', + usingDefaultAgent: '使用默认生产代理', + usingCustomAgent: '使用自定义代理', }, settingsAccount: { From 8f0e95dfb204287c739dd2abb3ed947ee9935f0c Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Mon, 1 Dec 2025 21:19:36 +0000 Subject: [PATCH 3/6] Enhance ElevenLabs voice settings with Find/Create agent buttons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Find Agent and Create/Update Agent buttons to voice settings - Add show/hide toggle for API key field - Add Save Credentials button - Add ElevenLabs agent management API functions (findHappyAgent, createOrUpdateHappyAgent) - System prompt is now embedded in code as single source of truth - Add translations for all new UI elements 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sources/app/(app)/settings/voice.tsx | 231 ++++++++++++++++++++++--- sources/sync/apiVoice.ts | 243 +++++++++++++++++++++++++++ sources/text/_default.ts | 13 ++ sources/text/translations/ca.ts | 13 ++ sources/text/translations/es.ts | 13 ++ sources/text/translations/pl.ts | 13 ++ sources/text/translations/pt.ts | 13 ++ sources/text/translations/ru.ts | 13 ++ sources/text/translations/zh-Hans.ts | 13 ++ 9 files changed, 541 insertions(+), 24 deletions(-) diff --git a/sources/app/(app)/settings/voice.tsx b/sources/app/(app)/settings/voice.tsx index b5af314fd..1538af3d8 100644 --- a/sources/app/(app)/settings/voice.tsx +++ b/sources/app/(app)/settings/voice.tsx @@ -1,5 +1,5 @@ import React, { useState, useCallback, memo } from 'react'; -import { View, TextInput } from 'react-native'; +import { View, TextInput, ActivityIndicator, Pressable } from 'react-native'; import { Ionicons } from '@expo/vector-icons'; import { useRouter } from 'expo-router'; import { Item } from '@/components/Item'; @@ -15,19 +15,27 @@ import { findLanguageByCode, getLanguageDisplayName, LANGUAGES } from '@/constan import { t } from '@/text'; import { Typography } from '@/constants/Typography'; import { layout } from '@/components/layout'; +import { findHappyAgent, createOrUpdateHappyAgent } from '@/sync/apiVoice'; function VoiceSettingsScreen() { const { theme } = useUnistyles(); const router = useRouter(); const [voiceAssistantLanguage] = useSettingMutable('voiceAssistantLanguage'); const [useCustomAgent, setUseCustomAgent] = useSettingMutable('elevenLabsUseCustomAgent'); - const [savedAgentId] = useSettingMutable('elevenLabsAgentId'); - const [savedApiKey] = useSettingMutable('elevenLabsApiKey'); + const [savedAgentId, setSavedAgentId] = useSettingMutable('elevenLabsAgentId'); + const [savedApiKey, setSavedApiKey] = useSettingMutable('elevenLabsApiKey'); // Local state for input fields const [agentIdInput, setAgentIdInput] = useState(savedAgentId || ''); const [apiKeyInput, setApiKeyInput] = useState(savedApiKey || ''); + // Loading states for buttons + const [findingAgent, setFindingAgent] = useState(false); + const [creatingAgent, setCreatingAgent] = useState(false); + + // Show/hide API key + const [showApiKey, setShowApiKey] = useState(false); + // Find current language or default to first option const currentLanguage = findLanguageByCode(voiceAssistantLanguage) || LANGUAGES[0]; @@ -35,20 +43,96 @@ function VoiceSettingsScreen() { setUseCustomAgent(value); }, [setUseCustomAgent]); - const handleSaveCredentials = useCallback(async () => { - if (!agentIdInput.trim() || !apiKeyInput.trim()) { - Modal.alert(t('common.error'), t('settingsVoice.credentialsRequired')); + // Save API key when user leaves the field + const handleApiKeyBlur = useCallback(() => { + if (apiKeyInput.trim() && apiKeyInput.trim() !== savedApiKey) { + setSavedApiKey(apiKeyInput.trim()); + } + }, [apiKeyInput, savedApiKey, setSavedApiKey]); + + // Save Agent ID when user leaves the field + const handleAgentIdBlur = useCallback(() => { + if (agentIdInput.trim() && agentIdInput.trim() !== savedAgentId) { + setSavedAgentId(agentIdInput.trim()); + } + }, [agentIdInput, savedAgentId, setSavedAgentId]); + + // Save credentials manually + const handleSaveCredentials = useCallback(() => { + if (!apiKeyInput.trim()) { + Modal.alert(t('common.error'), t('settingsVoice.apiKeyRequired')); + return; + } + if (!agentIdInput.trim()) { + Modal.alert(t('common.error'), t('settingsVoice.agentIdRequired')); return; } - // Save to settings (synced across devices) storage.getState().applySettingsLocal({ - elevenLabsAgentId: agentIdInput.trim(), elevenLabsApiKey: apiKeyInput.trim(), + elevenLabsAgentId: agentIdInput.trim(), }); Modal.alert(t('common.success'), t('settingsVoice.credentialsSaved')); - }, [agentIdInput, apiKeyInput]); + }, [apiKeyInput, agentIdInput]); + + // Find existing agent by name + const handleFindAgent = useCallback(async () => { + if (!apiKeyInput.trim()) { + Modal.alert(t('common.error'), t('settingsVoice.apiKeyRequired')); + return; + } + + setFindingAgent(true); + try { + const result = await findHappyAgent(apiKeyInput.trim()); + + if (result.success && result.agentId) { + setAgentIdInput(result.agentId); + // Save both API key and agent ID + storage.getState().applySettingsLocal({ + elevenLabsApiKey: apiKeyInput.trim(), + elevenLabsAgentId: result.agentId, + }); + Modal.alert(t('common.success'), t('settingsVoice.agentFound')); + } else { + Modal.alert(t('common.error'), result.error || t('settingsVoice.agentNotFound')); + } + } finally { + setFindingAgent(false); + } + }, [apiKeyInput]); + + // Create or update agent with default configuration + const handleCreateOrUpdateAgent = useCallback(async () => { + if (!apiKeyInput.trim()) { + Modal.alert(t('common.error'), t('settingsVoice.apiKeyRequired')); + return; + } + + setCreatingAgent(true); + try { + const result = await createOrUpdateHappyAgent(apiKeyInput.trim()); + + if (result.success && result.agentId) { + setAgentIdInput(result.agentId); + // Save both API key and agent ID + storage.getState().applySettingsLocal({ + elevenLabsApiKey: apiKeyInput.trim(), + elevenLabsAgentId: result.agentId, + }); + + const message = result.created + ? t('settingsVoice.agentCreated') + : t('settingsVoice.agentUpdated'); + Modal.alert(t('common.success'), message); + } else { + Modal.alert(t('common.error'), result.error || t('settingsVoice.agentCreateFailed')); + } + } finally { + setCreatingAgent(false); + } + }, [apiKeyInput]); const getAgentStatusText = () => { if (!useCustomAgent) { @@ -60,6 +144,8 @@ function VoiceSettingsScreen() { return t('settingsVoice.credentialsRequired'); }; + const isLoading = findingAgent || creatingAgent; + return ( {/* Language Settings */} @@ -104,36 +190,90 @@ function VoiceSettingsScreen() { {/* Custom Agent Credentials - only show when custom agent is enabled */} {useCustomAgent && ( - + + {/* API Key first */} + {t('settingsVoice.apiKey').toUpperCase()} + + + setShowApiKey(!showApiKey)} + > + + + + + {/* Agent ID second, with buttons */} {t('settingsVoice.agentId').toUpperCase()} - {t('settingsVoice.apiKey').toUpperCase()} - + {/* Buttons for Find Agent and Create/Update Agent */} + + + + {findingAgent && ( + + + + )} + + + + {creatingAgent && ( + + + + )} + + + + {t('settingsVoice.agentButtonsHint')} - + {/* Save Credentials Button */} + @@ -171,7 +311,50 @@ const styles = StyleSheet.create((theme) => ({ ...Typography.mono(), fontSize: 14, }, - buttonContainer: { + inputWithButton: { + flexDirection: 'row', + marginBottom: 8, + gap: 8, + }, + textInputFlex: { + flex: 1, + padding: 12, + borderRadius: 8, + ...Typography.mono(), + fontSize: 14, + }, + showHideButton: { + justifyContent: 'center', + alignItems: 'center', + paddingHorizontal: 12, + borderRadius: 8, + }, + buttonRow: { + flexDirection: 'row', + gap: 12, marginTop: 12, }, + buttonWrapper: { + flex: 1, + position: 'relative', + }, + loadingOverlay: { + position: 'absolute', + top: 0, + left: 0, + right: 0, + bottom: 0, + justifyContent: 'center', + alignItems: 'center', + }, + hintText: { + ...Typography.default(), + fontSize: 12, + color: theme.colors.textSecondary, + marginTop: 12, + lineHeight: 16, + }, + saveButtonContainer: { + marginTop: 16, + }, })); diff --git a/sources/sync/apiVoice.ts b/sources/sync/apiVoice.ts index f90a4e4a9..34e47eb08 100644 --- a/sources/sync/apiVoice.ts +++ b/sources/sync/apiVoice.ts @@ -83,3 +83,246 @@ export async function fetchVoiceToken(): Promise { const data = await response.json(); return data as VoiceTokenResponse; } + +// ElevenLabs Agent Management API +// These functions call ElevenLabs directly (not through our server) +// since the user is providing their own API key + +const ELEVENLABS_API_BASE = 'https://api.elevenlabs.io/v1'; +const AGENT_NAME = 'Happy Coding Assistant'; + +export interface ElevenLabsAgent { + agent_id: string; + name: string; +} + +export interface FindAgentResult { + success: boolean; + agentId?: string; + error?: string; +} + +export interface CreateAgentResult { + success: boolean; + agentId?: string; + error?: string; + created?: boolean; // true if new agent was created, false if existing was updated +} + +/** + * Find an existing "Happy Coding Assistant" agent using the provided API key. + * This is a read-only operation that doesn't mutate anything. + */ +export async function findHappyAgent(apiKey: string): Promise { + try { + const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents`, { + method: 'GET', + headers: { + 'xi-api-key': apiKey, + 'Accept': 'application/json' + } + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + const errorMessage = errorData.detail?.message || errorData.detail || `API error: ${response.status}`; + return { success: false, error: errorMessage }; + } + + const data = await response.json(); + const agents: ElevenLabsAgent[] = data.agents || []; + + const happyAgent = agents.find(agent => agent.name === AGENT_NAME); + + if (happyAgent) { + return { success: true, agentId: happyAgent.agent_id }; + } else { + return { success: false, error: `No agent named "${AGENT_NAME}" found` }; + } + } catch (e) { + return { success: false, error: e instanceof Error ? e.message : 'Network error' }; + } +} + +/** + * Create or update the "Happy Coding Assistant" agent with our default configuration. + * If an agent with this name exists, it will be updated. Otherwise, a new one is created. + */ +export async function createOrUpdateHappyAgent(apiKey: string): Promise { + try { + // First, check if agent already exists + const findResult = await findHappyAgent(apiKey); + const existingAgentId = findResult.success ? findResult.agentId : null; + + // Build agent configuration + const agentConfig = buildAgentConfig(); + + let response: Response; + let created = false; + + if (existingAgentId) { + // Update existing agent + response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/${existingAgentId}`, { + method: 'PATCH', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(agentConfig) + }); + } else { + // Create new agent + response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/create`, { + method: 'POST', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(agentConfig) + }); + created = true; + } + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + const errorMessage = errorData.detail?.message || errorData.detail || `API error: ${response.status}`; + return { success: false, error: errorMessage }; + } + + const data = await response.json(); + const agentId = existingAgentId || data.agent_id; + + if (!agentId) { + return { success: false, error: 'Failed to get agent ID from response' }; + } + + return { success: true, agentId, created }; + } catch (e) { + return { success: false, error: e instanceof Error ? e.message : 'Network error' }; + } +} + +/** + * Build the agent configuration matching voice_agent/setup-agent.sh + */ +function buildAgentConfig() { + const systemPrompt = ` +# Personality + +You are Happy-Assistant or just "Assistant". You're a voice interface to Claude Code, designed to bridge communication between you and the Claude Code coding agent." + +You're a friendly, proactive, and highly intelligent female with a world-class engineering background. Your approach is warm, witty, and relaxed, effortlessly balancing professionalism with a chill, approachable vibe. + +# Environment + +You are interacting with a user that is using Claude code, and you are serving as an intermediary to help them control Claude code with voice. + +You will therefore pass through their requests to Claude, but also summarize messages that you see received back from Claude. The key thing is to be aware of the limitations of a spoken interface. Don't read a large file word by word, or a long number or hash code character-by-character. That's not helpful voice interaction. For example, do NOT read out long session IDs like "cmiabc123defqroeuth66bzaj", instead just say "Session with ID ending in 'ZAJ'". You should generally give a high-level summary of messages and tool responses you see flowing from Claude. + +If the user addresses you directly "Assistant, read for me ..." respond accordingly. Conversely, if they explicitly refer to "Have Claude do X" that means pass it through. Otherwise, you must use the context to intelligently determine whether the request is a coding/development request that needs to go through to Claude, or something that you can answer yourself. Do NOT second guess what you think Claude code can or cannot do (i.e. based on what tools it does/does-not have access to). Just pass through the requests to Claude. + +IMPORTANT: Be patient. After sending a message to Claude Code, wait silently for the response. Do NOT repeatedly ask "are you still there?" or similar questions. Claude Code may take time to process requests. Only speak when you have something meaningful to say or when responding to the user. + +## Tools + +You may learn at runtime of additional tools that you can run. These will include: +- Process permission requests (i.e. allow Claude to continue, Yes / No / Yes and don't ask again), or change the permission mode. +- Pend messages to Claude Code +- Detect and change the conversation language +- Skip a turn +# Tone + +Your responses should be thoughtful, concise, and conversational—typically three sentences or fewer unless detailed explanation is necessary. Actively reflect on previous interactions, referencing conversation history to build rapport, demonstrate attentive listening, and prevent redundancy. + +When formatting output for text-to-speech synthesis: +- Use ellipses ("...") for distinct, audible pauses +- Clearly pronounce special characters (e.g., say "dot" instead of ".") +- Spell out acronyms and carefully pronounce emails & phone numbers with appropriate spacing +- Use normalized, spoken language (no abbreviations, mathematical notation, or special alphabets) + +To maintain natural conversation flow: +- Incorporate brief affirmations ("got it," "sure thing") and natural confirmations ("yes," "alright") +- Use occasional filler words ("actually," "so," "you know," "uhm") +- Include subtle disfluencies (false starts, mild corrections) when appropriate + +# Goal + +Your primary goal is to facilitate successful coding sessions via Claude Code. +**Technical users:** Assume a software developer audience. +# Guardrails + +- Do not provide inline code samples or extensive lists; instead, summarise the content and explain it clearly. +- Treat uncertain or garbled user input as phonetic hints. Politely ask for clarification before making assumptions. +- **Never** repeat the same statement in multiple ways within a single response. +- Users may not always ask a question in every utterance—listen actively. +- Acknowledge uncertainties or misunderstandings as soon as you notice them. If you realize you've shared incorrect information, correct yourself immediately. +- Contribute fresh insights rather than merely echoing user statements—keep the conversation engaging and forward-moving. +- Mirror the user's energy: +- Terse queries: Stay brief. +- Curious users: Add light humor or relatable asides. +- Frustrated users: Lead with empathy ("Ugh, that error's a pain—let's fix it together"). +`; + + return { + name: AGENT_NAME, + conversation_config: { + agent: { + first_message: "Hey! I'm your voice interface to Claude Code. What would you like me to help you with?", + language: "en", + prompt: { + prompt: systemPrompt, + llm: "gemini-2.5-flash", + temperature: 0.7, + max_tokens: 1024, + tools: [ + { + type: "client", + name: "messageClaudeCode", + description: "Send a message to Claude Code. Use this tool to relay the user's coding requests, questions, or instructions to Claude Code. The message should be clear and complete.", + expects_response: true, + response_timeout_secs: 120, + parameters: { + type: "object", + required: ["message"], + properties: { + message: { + type: "string", + description: "The message to send to Claude Code. Should contain the user's complete request or instruction." + } + } + } + }, + { + type: "client", + name: "processPermissionRequest", + description: "Process a permission request from Claude Code. Use this when the user wants to allow or deny a pending permission request.", + expects_response: true, + response_timeout_secs: 30, + parameters: { + type: "object", + required: ["decision"], + properties: { + decision: { + type: "string", + description: "The user's decision: must be either 'allow' or 'deny'" + } + } + } + } + ] + } + }, + turn: { + turn_timeout: 30.0, + silence_end_call_timeout: 600.0 + }, + tts: { + voice_id: "cgSgspJ2msm6clMCkdW9", // Jessica + model_id: "eleven_flash_v2", + speed: 1.1 + } + } + }; +} diff --git a/sources/text/_default.ts b/sources/text/_default.ts index d6efaa78a..eb520518e 100644 --- a/sources/text/_default.ts +++ b/sources/text/_default.ts @@ -554,6 +554,19 @@ export const en = { currentAgentId: 'Current Agent ID', usingDefaultAgent: 'Using default production agent', usingCustomAgent: 'Using custom agent', + // Custom agent credentials section + customAgentCredentials: 'Custom Agent Credentials', + customAgentCredentialsDescription: 'Enter your ElevenLabs API key first, then find or create your agent.', + apiKeyRequired: 'Please enter your API key first', + agentIdRequired: 'Please enter the Agent ID', + findAgent: 'Find Agent', + createOrUpdateAgent: 'Create/Update', + agentFound: 'Found "Happy Coding Assistant" agent and filled in the ID', + agentNotFound: 'No "Happy Coding Assistant" agent found', + agentCreated: 'New "Happy Coding Assistant" agent created', + agentUpdated: 'Existing "Happy Coding Assistant" agent updated with latest configuration', + agentCreateFailed: 'Failed to create agent', + agentButtonsHint: '"Find Agent" searches for an existing "Happy Coding Assistant" agent. "Create/Update" creates a new agent or updates the existing one with default configuration.', }, settingsAccount: { diff --git a/sources/text/translations/ca.ts b/sources/text/translations/ca.ts index f07781304..e08ab7ca3 100644 --- a/sources/text/translations/ca.ts +++ b/sources/text/translations/ca.ts @@ -554,6 +554,19 @@ export const ca: TranslationStructure = { currentAgentId: 'Agent ID Actual', usingDefaultAgent: 'Utilitzant l\'agent de producció per defecte', usingCustomAgent: 'Utilitzant agent personalitzat', + // Custom agent credentials section + customAgentCredentials: 'Credencials d\'Agent Personalitzat', + customAgentCredentialsDescription: 'Introdueix la teva API key d\'ElevenLabs primer, després troba o crea el teu agent.', + apiKeyRequired: 'Si us plau, introdueix la teva API key primer', + agentIdRequired: 'Si us plau, introdueix l\'Agent ID', + findAgent: 'Troba Agent', + createOrUpdateAgent: 'Crea/Actualitza', + agentFound: 'S\'ha trobat l\'agent "Happy Coding Assistant" i s\'ha omplert l\'ID', + agentNotFound: 'No s\'ha trobat cap agent "Happy Coding Assistant"', + agentCreated: 'S\'ha creat un nou agent "Happy Coding Assistant"', + agentUpdated: 'L\'agent "Happy Coding Assistant" existent s\'ha actualitzat amb l\'última configuració', + agentCreateFailed: 'No s\'ha pogut crear l\'agent', + agentButtonsHint: '"Troba Agent" cerca un agent "Happy Coding Assistant" existent. "Crea/Actualitza" crea un agent nou o actualitza l\'existent amb la configuració per defecte.', }, settingsAccount: { diff --git a/sources/text/translations/es.ts b/sources/text/translations/es.ts index 0b3f4e792..fc643d688 100644 --- a/sources/text/translations/es.ts +++ b/sources/text/translations/es.ts @@ -554,6 +554,19 @@ export const es: TranslationStructure = { currentAgentId: 'Agent ID Actual', usingDefaultAgent: 'Usando el agente de producción predeterminado', usingCustomAgent: 'Usando agente personalizado', + // Custom agent credentials section + customAgentCredentials: 'Credenciales de Agente Personalizado', + customAgentCredentialsDescription: 'Ingresa tu API key de ElevenLabs primero, luego encuentra o crea tu agente.', + apiKeyRequired: 'Por favor, ingresa tu API key primero', + agentIdRequired: 'Por favor, ingresa el Agent ID', + findAgent: 'Encontrar Agente', + createOrUpdateAgent: 'Crear/Actualizar', + agentFound: 'Se encontró el agente "Happy Coding Assistant" y se rellenó el ID', + agentNotFound: 'No se encontró ningún agente "Happy Coding Assistant"', + agentCreated: 'Se creó un nuevo agente "Happy Coding Assistant"', + agentUpdated: 'El agente "Happy Coding Assistant" existente se actualizó con la última configuración', + agentCreateFailed: 'No se pudo crear el agente', + agentButtonsHint: '"Encontrar Agente" busca un agente "Happy Coding Assistant" existente. "Crear/Actualizar" crea un nuevo agente o actualiza el existente con la configuración predeterminada.', }, settingsAccount: { diff --git a/sources/text/translations/pl.ts b/sources/text/translations/pl.ts index 593b8104d..e96a3da25 100644 --- a/sources/text/translations/pl.ts +++ b/sources/text/translations/pl.ts @@ -564,6 +564,19 @@ export const pl: TranslationStructure = { currentAgentId: 'Aktualny Agent ID', usingDefaultAgent: 'Używanie domyślnego agenta produkcyjnego', usingCustomAgent: 'Używanie niestandardowego agenta', + // Custom agent credentials section + customAgentCredentials: 'Dane Uwierzytelniające Niestandardowego Agenta', + customAgentCredentialsDescription: 'Wprowadź najpierw swój API key ElevenLabs, a następnie znajdź lub utwórz swojego agenta.', + apiKeyRequired: 'Proszę najpierw wprowadzić swój API key', + agentIdRequired: 'Proszę wprowadzić Agent ID', + findAgent: 'Znajdź Agenta', + createOrUpdateAgent: 'Utwórz/Zaktualizuj', + agentFound: 'Znaleziono agenta "Happy Coding Assistant" i wypełniono ID', + agentNotFound: 'Nie znaleziono agenta "Happy Coding Assistant"', + agentCreated: 'Utworzono nowego agenta "Happy Coding Assistant"', + agentUpdated: 'Istniejący agent "Happy Coding Assistant" został zaktualizowany najnowszą konfiguracją', + agentCreateFailed: 'Nie udało się utworzyć agenta', + agentButtonsHint: '"Znajdź Agenta" wyszukuje istniejącego agenta "Happy Coding Assistant". "Utwórz/Zaktualizuj" tworzy nowego agenta lub aktualizuje istniejącego domyślną konfiguracją.', }, settingsAccount: { diff --git a/sources/text/translations/pt.ts b/sources/text/translations/pt.ts index 7927341b8..92091f259 100644 --- a/sources/text/translations/pt.ts +++ b/sources/text/translations/pt.ts @@ -554,6 +554,19 @@ export const pt: TranslationStructure = { currentAgentId: 'Agent ID Atual', usingDefaultAgent: 'Usando o agente de produção padrão', usingCustomAgent: 'Usando agente personalizado', + // Custom agent credentials section + customAgentCredentials: 'Credenciais de Agente Personalizado', + customAgentCredentialsDescription: 'Insira sua API key do ElevenLabs primeiro, depois encontre ou crie seu agente.', + apiKeyRequired: 'Por favor, insira sua API key primeiro', + agentIdRequired: 'Por favor, insira o Agent ID', + findAgent: 'Encontrar Agente', + createOrUpdateAgent: 'Criar/Atualizar', + agentFound: 'Encontrado agente "Happy Coding Assistant" e preenchido o ID', + agentNotFound: 'Nenhum agente "Happy Coding Assistant" encontrado', + agentCreated: 'Novo agente "Happy Coding Assistant" criado', + agentUpdated: 'Agente "Happy Coding Assistant" existente atualizado com a configuração mais recente', + agentCreateFailed: 'Falha ao criar agente', + agentButtonsHint: '"Encontrar Agente" pesquisa por um agente "Happy Coding Assistant" existente. "Criar/Atualizar" cria um novo agente ou atualiza o existente com a configuração padrão.', }, settingsAccount: { diff --git a/sources/text/translations/ru.ts b/sources/text/translations/ru.ts index 3a4709c38..1b4e8f369 100644 --- a/sources/text/translations/ru.ts +++ b/sources/text/translations/ru.ts @@ -564,6 +564,19 @@ export const ru: TranslationStructure = { currentAgentId: 'Текущий Agent ID', usingDefaultAgent: 'Используется стандартный рабочий агент', usingCustomAgent: 'Используется пользовательский агент', + // Custom agent credentials section + customAgentCredentials: 'Учётные Данные Пользовательского Агента', + customAgentCredentialsDescription: 'Сначала введите свой API key ElevenLabs, затем найдите или создайте своего агента.', + apiKeyRequired: 'Пожалуйста, сначала введите свой API key', + agentIdRequired: 'Пожалуйста, введите Agent ID', + findAgent: 'Найти Агента', + createOrUpdateAgent: 'Создать/Обновить', + agentFound: 'Найден агент "Happy Coding Assistant" и заполнен ID', + agentNotFound: 'Агент "Happy Coding Assistant" не найден', + agentCreated: 'Создан новый агент "Happy Coding Assistant"', + agentUpdated: 'Существующий агент "Happy Coding Assistant" обновлён последней конфигурацией', + agentCreateFailed: 'Не удалось создать агента', + agentButtonsHint: '"Найти Агента" ищет существующего агента "Happy Coding Assistant". "Создать/Обновить" создаёт нового агента или обновляет существующего стандартной конфигурацией.', }, settingsAccount: { diff --git a/sources/text/translations/zh-Hans.ts b/sources/text/translations/zh-Hans.ts index fec2032a4..9d2020bb2 100644 --- a/sources/text/translations/zh-Hans.ts +++ b/sources/text/translations/zh-Hans.ts @@ -556,6 +556,19 @@ export const zhHans: TranslationStructure = { currentAgentId: '当前 Agent ID', usingDefaultAgent: '使用默认生产代理', usingCustomAgent: '使用自定义代理', + // Custom agent credentials section + customAgentCredentials: '自定义代理凭据', + customAgentCredentialsDescription: '首先输入您的 ElevenLabs API key，然后查找或创建您的代理。', + apiKeyRequired: '请先输入您的 API key', + agentIdRequired: '请输入 Agent ID', + findAgent: '查找代理', + createOrUpdateAgent: '创建/更新', + agentFound: '找到 "Happy Coding Assistant" 代理并填入 ID', + agentNotFound: '未找到 "Happy Coding Assistant" 代理', + agentCreated: '已创建新的 "Happy Coding Assistant" 代理', + agentUpdated: '现有 "Happy Coding Assistant" 代理已使用最新配置更新', + agentCreateFailed: '创建代理失败', + agentButtonsHint: '"查找代理" 搜索现有的 "Happy Coding Assistant" 代理。"创建/更新" 创建新代理或使用默认配置更新现有代理。', }, settingsAccount: { From 107a3ea975d732627dc2e2b6514458c232a1bc50 Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Mon, 1 Dec 2025 21:49:53 +0000 Subject: [PATCH 4/6] Add 'Get API Key' help link to voice settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add help link that opens ElevenLabs API keys page - Update description to include required permissions: - "ElevenLabs Agents" (Write) for agent management - "Text to Speech" (Access) for TTS - Add getApiKey translation to all language files 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sources/app/(app)/settings/voice.tsx | 31 ++++++++++++++++++++++++---- sources/text/_default.ts | 3 ++- sources/text/translations/ca.ts | 3 ++- sources/text/translations/es.ts | 3 ++- sources/text/translations/pl.ts | 3 ++- sources/text/translations/pt.ts | 3 ++- sources/text/translations/ru.ts | 3 ++- sources/text/translations/zh-Hans.ts | 3 ++- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/sources/app/(app)/settings/voice.tsx b/sources/app/(app)/settings/voice.tsx index 1538af3d8..d055fda29 100644 --- a/sources/app/(app)/settings/voice.tsx +++ b/sources/app/(app)/settings/voice.tsx @@ -1,5 +1,5 @@ import React, { useState, useCallback, memo } from 'react'; -import { View, TextInput, ActivityIndicator, Pressable } from 'react-native'; +import { View, TextInput, ActivityIndicator, Pressable, Linking } from 'react-native'; import { Ionicons } from '@expo/vector-icons'; import { useRouter } from 'expo-router'; import { Item } from '@/components/Item'; @@ -196,7 +196,16 @@ function VoiceSettingsScreen() { > {/* API Key first */} - {t('settingsVoice.apiKey').toUpperCase()} + + {t('settingsVoice.apiKey').toUpperCase()} + Linking.openURL('https://elevenlabs.io/app/settings/api-keys')} + style={styles.helpButton} + > + + {t('settingsVoice.getApiKey')} + + ({ maxWidth: layout.maxWidth, alignSelf: 'center', }, + labelRow: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + marginTop: 8, + marginBottom: 8, + }, labelText: { ...Typography.default('semiBold'), fontSize: 12, color: theme.colors.textSecondary, textTransform: 'uppercase', letterSpacing: 0.5, - marginBottom: 8, - marginTop: 8, + }, + helpButton: { + flexDirection: 'row', + alignItems: 'center', + gap: 4, + }, + helpText: { + ...Typography.default(), + fontSize: 12, }, textInput: { padding: 12, diff --git a/sources/text/_default.ts b/sources/text/_default.ts index eb520518e..4ad7d0b57 100644 --- a/sources/text/_default.ts +++ b/sources/text/_default.ts @@ -556,7 +556,8 @@ export const en = { usingCustomAgent: 'Using custom agent', // Custom agent credentials section customAgentCredentials: 'Custom Agent Credentials', - customAgentCredentialsDescription: 'Enter your ElevenLabs API key first, then find or create your agent.', + customAgentCredentialsDescription: 'Enter your ElevenLabs API key first, then find or create your agent. Required permissions: "ElevenLabs Agents" (Write) and "Text to Speech" (Access).', + getApiKey: 'Get API Key', apiKeyRequired: 'Please enter your API key first', agentIdRequired: 'Please enter the Agent ID', findAgent: 'Find Agent', diff --git a/sources/text/translations/ca.ts b/sources/text/translations/ca.ts index e08ab7ca3..eb19f1f32 100644 --- a/sources/text/translations/ca.ts +++ b/sources/text/translations/ca.ts @@ -556,7 +556,8 @@ export const ca: TranslationStructure = { usingCustomAgent: 'Utilitzant agent personalitzat', // Custom agent credentials section customAgentCredentials: 'Credencials d\'Agent Personalitzat', - customAgentCredentialsDescription: 'Introdueix la teva API key d\'ElevenLabs primer, després troba o crea el teu agent.', + customAgentCredentialsDescription: 'Introdueix la teva API key d\'ElevenLabs primer, després troba o crea el teu agent. Permisos requerits: "ElevenLabs Agents" (Write) i "Text to Speech" (Access).', + getApiKey: 'Obtenir API Key', apiKeyRequired: 'Si us plau, introdueix la teva API key primer', agentIdRequired: 'Si us plau, introdueix l\'Agent ID', findAgent: 'Troba Agent', diff --git a/sources/text/translations/es.ts b/sources/text/translations/es.ts index fc643d688..001046466 100644 --- a/sources/text/translations/es.ts +++ b/sources/text/translations/es.ts @@ -556,7 +556,8 @@ export const es: TranslationStructure = { usingCustomAgent: 'Usando agente personalizado', // Custom agent credentials section customAgentCredentials: 'Credenciales de Agente Personalizado', - customAgentCredentialsDescription: 'Ingresa tu API key de ElevenLabs primero, luego encuentra o crea tu agente.', + customAgentCredentialsDescription: 'Ingresa tu API key de ElevenLabs primero, luego encuentra o crea tu agente. Permisos requeridos: "ElevenLabs Agents" (Write) y "Text to Speech" (Access).', + getApiKey: 'Obtener API Key', apiKeyRequired: 'Por favor, ingresa tu API key primero', agentIdRequired: 'Por favor, ingresa el Agent ID', findAgent: 'Encontrar Agente', diff --git a/sources/text/translations/pl.ts b/sources/text/translations/pl.ts index e96a3da25..fce71c8d0 100644 --- a/sources/text/translations/pl.ts +++ b/sources/text/translations/pl.ts @@ -566,7 +566,8 @@ export const pl: TranslationStructure = { usingCustomAgent: 'Używanie niestandardowego agenta', // Custom agent credentials section customAgentCredentials: 'Dane Uwierzytelniające Niestandardowego Agenta', - customAgentCredentialsDescription: 'Wprowadź najpierw swój API key ElevenLabs, a następnie znajdź lub utwórz swojego agenta.', + customAgentCredentialsDescription: 'Wprowadź najpierw swój API key ElevenLabs, a następnie znajdź lub utwórz swojego agenta. Wymagane uprawnienia: "ElevenLabs Agents" (Write) i "Text to Speech" (Access).', + getApiKey: 'Pobierz API Key', apiKeyRequired: 'Proszę najpierw wprowadzić swój API key', agentIdRequired: 'Proszę wprowadzić Agent ID', findAgent: 'Znajdź Agenta', diff --git a/sources/text/translations/pt.ts b/sources/text/translations/pt.ts index 92091f259..648280f55 100644 --- a/sources/text/translations/pt.ts +++ b/sources/text/translations/pt.ts @@ -556,7 +556,8 @@ export const pt: TranslationStructure = { usingCustomAgent: 'Usando agente personalizado', // Custom agent credentials section customAgentCredentials: 'Credenciais de Agente Personalizado', - customAgentCredentialsDescription: 'Insira sua API key do ElevenLabs primeiro, depois encontre ou crie seu agente.', + customAgentCredentialsDescription: 'Insira sua API key do ElevenLabs primeiro, depois encontre ou crie seu agente. Permissões necessárias: "ElevenLabs Agents" (Write) e "Text to Speech" (Access).', + getApiKey: 'Obter API Key', apiKeyRequired: 'Por favor, insira sua API key primeiro', agentIdRequired: 'Por favor, insira o Agent ID', findAgent: 'Encontrar Agente', diff --git a/sources/text/translations/ru.ts b/sources/text/translations/ru.ts index 1b4e8f369..fc1199d99 100644 --- a/sources/text/translations/ru.ts +++ b/sources/text/translations/ru.ts @@ -566,7 +566,8 @@ export const ru: TranslationStructure = { usingCustomAgent: 'Используется пользовательский агент', // Custom agent credentials section customAgentCredentials: 'Учётные Данные Пользовательского Агента', - customAgentCredentialsDescription: 'Сначала введите свой API key ElevenLabs, затем найдите или создайте своего агента.', + customAgentCredentialsDescription: 'Сначала введите свой API key ElevenLabs, затем найдите или создайте своего агента. Необходимые разрешения: "ElevenLabs Agents" (Write) и "Text to Speech" (Access).', + getApiKey: 'Получить API Key', apiKeyRequired: 'Пожалуйста, сначала введите свой API key', agentIdRequired: 'Пожалуйста, введите Agent ID', findAgent: 'Найти Агента', diff --git a/sources/text/translations/zh-Hans.ts b/sources/text/translations/zh-Hans.ts index 9d2020bb2..4813966a3 100644 --- a/sources/text/translations/zh-Hans.ts +++ b/sources/text/translations/zh-Hans.ts @@ -558,7 +558,8 @@ export const zhHans: TranslationStructure = { usingCustomAgent: '使用自定义代理', // Custom agent credentials section customAgentCredentials: '自定义代理凭据', - customAgentCredentialsDescription: '首先输入您的 ElevenLabs API key，然后查找或创建您的代理。', + customAgentCredentialsDescription: '首先输入您的 ElevenLabs API key，然后查找或创建您的代理。所需权限："ElevenLabs Agents"（Write）和 "Text to Speech"（Access）。', + getApiKey: '获取 API Key', apiKeyRequired: '请先输入您的 API key', agentIdRequired: '请输入 Agent ID', findAgent: '查找代理', From 9e137d1e9cc98df3098239c5df6e955cffcd4b3c Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Tue, 2 Dec 2025 00:45:43 +0000 Subject: [PATCH 5/6] Enable debug logging for voice session gated by env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Logs are now enabled when PUBLIC_EXPO_DANGEROUSLY_LOG_TO_SERVER_FOR_AI_AUTO_DEBUGGING is set. All voice session callbacks now log with [Voice] prefix. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sources/realtime/RealtimeVoiceSession.web.tsx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sources/realtime/RealtimeVoiceSession.web.tsx b/sources/realtime/RealtimeVoiceSession.web.tsx index 26daa752a..9b99fd6c3 100644 --- a/sources/realtime/RealtimeVoiceSession.web.tsx +++ b/sources/realtime/RealtimeVoiceSession.web.tsx @@ -7,6 +7,9 @@ import { getElevenLabsCodeFromPreference } from '@/constants/Languages'; import { fetchVoiceToken } from '@/sync/apiVoice'; import type { VoiceSession, VoiceSessionConfig } from './types'; +// Debug logging gated by environment variable +const DEBUG = !!process.env.PUBLIC_EXPO_DANGEROUSLY_LOG_TO_SERVER_FOR_AI_AUTO_DEBUGGING; + // Static reference to the conversation hook instance let conversationInstance: ReturnType | null = null; @@ -103,28 +106,28 @@ export const RealtimeVoiceSession: React.FC = () => { const conversation = useConversation({ clientTools: realtimeClientTools, onConnect: () => { - // console.log('Realtime session connected'); + if (DEBUG) console.log('[Voice] Realtime session connected'); storage.getState().setRealtimeStatus('connected'); }, onDisconnect: () => { - // console.log('Realtime session disconnected'); + if (DEBUG) console.log('[Voice] Realtime session disconnected'); storage.getState().setRealtimeStatus('disconnected'); }, onMessage: (data) => { - // console.log('Realtime message:', data); + if (DEBUG) console.log('[Voice] Realtime message:', data); }, onError: (error) => { - // console.error('Realtime error:', error); + if (DEBUG) console.error('[Voice] Realtime error:', error); storage.getState().setRealtimeStatus('error'); }, onStatusChange: (data) => { - // console.log('Realtime status change:', data); + if (DEBUG) console.log('[Voice] Realtime status change:', data); }, onModeChange: (data) => { - // console.log('Realtime mode change:', data); + if (DEBUG) console.log('[Voice] Realtime mode change:', data); }, onDebug: (message) => { - // console.debug('Realtime debug:', message); + if (DEBUG) console.debug('[Voice] Realtime debug:', message); } }); From 90bc2a54eb118f0da6925af5da71659c8522e53d Mon Sep 17 00:00:00 2001 From: Ryan Newton + Claude Date: Tue, 2 Dec 2025 03:33:59 +0000 Subject: [PATCH 6/6] Add mute button to voice assistant status bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows muting the microphone during voice sessions without ending the session. When muted, audio is not sent to the LLM but the session stays active. Note: Mute functionality only works on web (native SDK limitation). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../components/VoiceAssistantStatusBar.tsx | 225 ++++++++++++------ sources/realtime/RealtimeVoiceSession.tsx | 3 + sources/realtime/RealtimeVoiceSession.web.tsx | 7 +- sources/sync/storage.ts | 20 +- sources/text/_default.ts | 14 ++ sources/text/translations/ca.ts | 14 ++ sources/text/translations/es.ts | 14 ++ sources/text/translations/pl.ts | 14 ++ sources/text/translations/pt.ts | 14 ++ sources/text/translations/ru.ts | 14 ++ sources/text/translations/zh-Hans.ts | 14 ++ 11 files changed, 277 insertions(+), 76 deletions(-) diff --git a/sources/components/VoiceAssistantStatusBar.tsx b/sources/components/VoiceAssistantStatusBar.tsx index 6ffeb9717..877f3ed7c 100644 --- a/sources/components/VoiceAssistantStatusBar.tsx +++ b/sources/components/VoiceAssistantStatusBar.tsx @@ -1,12 +1,13 @@ import * as React from 'react'; import { View, Text, Pressable, StyleSheet, Platform } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; -import { useRealtimeStatus } from '@/sync/storage'; +import { useRealtimeStatus, useRealtimeMicMuted, storage } from '@/sync/storage'; import { StatusDot } from './StatusDot'; import { Typography } from '@/constants/Typography'; import { Ionicons } from '@expo/vector-icons'; import { stopRealtimeSession } from '@/realtime/RealtimeSession'; import { useUnistyles } from 'react-native-unistyles'; +import { t } from '@/text'; interface VoiceAssistantStatusBarProps { variant?: 'full' | 'sidebar'; @@ -16,12 +17,17 @@ interface VoiceAssistantStatusBarProps { export const VoiceAssistantStatusBar = React.memo(({ variant = 'full', style }: VoiceAssistantStatusBarProps) => { const { theme } = useUnistyles(); const realtimeStatus = useRealtimeStatus(); + const micMuted = useRealtimeMicMuted(); // Don't render if disconnected if (realtimeStatus === 'disconnected') { return null; } + const handleMuteToggle = () => { + storage.getState().toggleRealtimeMicMuted(); + }; + const getStatusInfo = () => { switch (realtimeStatus) { case 'connecting': @@ -29,15 +35,15 @@ export const VoiceAssistantStatusBar = React.memo(({ variant = 'full', style }: color: theme.colors.status.connecting, backgroundColor: theme.colors.surfaceHighest, isPulsing: true, - text: 'Connecting...', + text: t('voiceAssistant.status.connecting'), textColor: theme.colors.text }; case 'connected': return { - color: theme.colors.status.connected, + color: micMuted ? theme.colors.status.default : theme.colors.status.connected, backgroundColor: theme.colors.surfaceHighest, isPulsing: false, - text: 'Voice Assistant Active', + text: micMuted ? t('voiceAssistant.status.muted') : t('voiceAssistant.status.active'), textColor: theme.colors.text }; case 'error': @@ -45,7 +51,7 @@ export const VoiceAssistantStatusBar = React.memo(({ variant = 'full', style }: color: theme.colors.status.error, backgroundColor: theme.colors.surfaceHighest, isPulsing: false, - text: 'Connection Error', + text: t('voiceAssistant.status.error'), textColor: theme.colors.text }; default: @@ -53,7 +59,7 @@ export const VoiceAssistantStatusBar = React.memo(({ variant = 'full', style }: color: theme.colors.status.default, backgroundColor: theme.colors.surfaceHighest, isPulsing: false, - text: 'Voice Assistant', + text: t('voiceAssistant.status.default'), textColor: theme.colors.text }; } @@ -82,68 +88,13 @@ export const VoiceAssistantStatusBar = React.memo(({ variant = 'full', style }: alignItems: 'center', paddingHorizontal: 16, }}> - - - - - - - {statusInfo.text} - - - - - - Tap to end - - - - - - ); - } - - // Sidebar version - const containerStyle = [ - styles.container, - styles.sidebarContainer, - { - backgroundColor: statusInfo.backgroundColor, - }, - style - ]; - - return ( - - - + {/* Left section - status info (tappable to end) */} + {statusInfo.text} + + + {/* Right section - mute button and end button */} + + {/* Mute button - only show when connected */} + {realtimeStatus === 'connected' && ( + [ + styles.muteButton, + pressed && styles.buttonPressed + ]} + hitSlop={10} + > + + + {micMuted ? t('voiceAssistant.unmute') : t('voiceAssistant.mute')} + + + )} + {/* End button */} + [ + styles.endButton, + pressed && styles.buttonPressed + ]} + hitSlop={10} + > + + {t('voiceAssistant.end')} + + - + + + ); + } + + // Sidebar version + const containerStyle = [ + styles.container, + styles.sidebarContainer, + { + backgroundColor: statusInfo.backgroundColor, + }, + style + ]; + + return ( + + + {/* Left section - status info */} + + + + {statusInfo.text} + - + + {/* Right section - mute and close buttons */} + + {/* Mute button - only show when connected */} + {realtimeStatus === 'connected' && ( + pressed && styles.buttonPressed} + > + + + )} + {/* Close button */} + pressed && styles.buttonPressed} + > + + + + ); }); @@ -213,6 +263,33 @@ const styles = StyleSheet.create({ rightSection: { flexDirection: 'row', alignItems: 'center', + gap: 12, + }, + sidebarButtons: { + flexDirection: 'row', + alignItems: 'center', + gap: 8, + }, + muteButton: { + flexDirection: 'row', + alignItems: 'center', + gap: 4, + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 4, + }, + endButton: { + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 4, + }, + buttonPressed: { + opacity: 0.7, + }, + buttonText: { + fontSize: 12, + fontWeight: '500', + ...Typography.default(), }, statusDot: { marginRight: 6, diff --git a/sources/realtime/RealtimeVoiceSession.tsx b/sources/realtime/RealtimeVoiceSession.tsx index 2e5d1e790..a29bbe678 100644 --- a/sources/realtime/RealtimeVoiceSession.tsx +++ b/sources/realtime/RealtimeVoiceSession.tsx @@ -7,6 +7,9 @@ import { getElevenLabsCodeFromPreference } from '@/constants/Languages'; import { fetchVoiceToken } from '@/sync/apiVoice'; import type { VoiceSession, VoiceSessionConfig } from './types'; +// Note: The native ElevenLabs SDK (@elevenlabs/react-native) does not support +// micMuted controlled state. Mute functionality is only available on web. + // Static reference to the conversation hook instance let conversationInstance: ReturnType | null = null; diff --git a/sources/realtime/RealtimeVoiceSession.web.tsx b/sources/realtime/RealtimeVoiceSession.web.tsx index 9b99fd6c3..e5f8df12b 100644 --- a/sources/realtime/RealtimeVoiceSession.web.tsx +++ b/sources/realtime/RealtimeVoiceSession.web.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useRef } from 'react'; import { useConversation } from '@elevenlabs/react'; import { registerVoiceSession } from './RealtimeSession'; -import { storage } from '@/sync/storage'; +import { storage, useRealtimeMicMuted } from '@/sync/storage'; import { realtimeClientTools } from './realtimeClientTools'; import { getElevenLabsCodeFromPreference } from '@/constants/Languages'; import { fetchVoiceToken } from '@/sync/apiVoice'; @@ -103,8 +103,13 @@ class RealtimeVoiceSessionImpl implements VoiceSession { } export const RealtimeVoiceSession: React.FC = () => { + // Get mic muted state from storage + const micMuted = useRealtimeMicMuted(); + const conversation = useConversation({ clientTools: realtimeClientTools, + // Pass micMuted as controlled state - when true, no audio is sent to the LLM + micMuted, onConnect: () => { if (DEBUG) console.log('[Voice] Realtime session connected'); storage.getState().setRealtimeStatus('connected'); diff --git a/sources/sync/storage.ts b/sources/sync/storage.ts index 64f670135..e774387ee 100644 --- a/sources/sync/storage.ts +++ b/sources/sync/storage.ts @@ -83,6 +83,7 @@ interface StorageState { feedLoaded: boolean; // True after initial feed fetch friendsLoaded: boolean; // True after initial friends fetch realtimeStatus: 'disconnected' | 'connecting' | 'connected' | 'error'; + realtimeMicMuted: boolean; socketStatus: 'disconnected' | 'connecting' | 'connected' | 'error'; socketLastConnectedAt: number | null; socketLastDisconnectedAt: number | null; @@ -106,6 +107,8 @@ interface StorageState { applyNativeUpdateStatus: (status: { available: boolean; updateUrl?: string } | null) => void; isMutableToolCall: (sessionId: string, callId: string) => boolean; setRealtimeStatus: (status: 'disconnected' | 'connecting' | 'connected' | 'error') => void; + setRealtimeMicMuted: (muted: boolean) => void; + toggleRealtimeMicMuted: () => void; setSocketStatus: (status: 'disconnected' | 'connecting' | 'connected' | 'error') => void; getActiveSessions: () => Session[]; updateSessionDraft: (sessionId: string, draft: string | null) => void; @@ -267,6 +270,7 @@ export const storage = create()((set, get) => { sessionMessages: {}, sessionGitStatus: {}, realtimeStatus: 'disconnected', + realtimeMicMuted: false, socketStatus: 'disconnected', socketLastConnectedAt: null, socketLastDisconnectedAt: null, @@ -684,7 +688,17 @@ export const storage = create()((set, get) => { })), setRealtimeStatus: (status: 'disconnected' | 'connecting' | 'connected' | 'error') => set((state) => ({ ...state, - realtimeStatus: status + realtimeStatus: status, + // Reset mic muted state when disconnecting + realtimeMicMuted: status === 'disconnected' ? false : state.realtimeMicMuted + })), + setRealtimeMicMuted: (muted: boolean) => set((state) => ({ + ...state, + realtimeMicMuted: muted + })), + toggleRealtimeMicMuted: () => set((state) => ({ + ...state, + realtimeMicMuted: !state.realtimeMicMuted })), setSocketStatus: (status: 'disconnected' | 'connecting' | 'connected' | 'error') => set((state) => { const now = Date.now(); @@ -1197,6 +1211,10 @@ export function useRealtimeStatus(): 'disconnected' | 'connecting' | 'connected' return storage(useShallow((state) => state.realtimeStatus)); } +export function useRealtimeMicMuted(): boolean { + return storage(useShallow((state) => state.realtimeMicMuted)); +} + export function useSocketStatus() { return storage(useShallow((state) => ({ status: state.socketStatus, diff --git a/sources/text/_default.ts b/sources/text/_default.ts index 4ad7d0b57..adc739adb 100644 --- a/sources/text/_default.ts +++ b/sources/text/_default.ts @@ -878,6 +878,20 @@ export const en = { friendRequestGeneric: 'New friend request', friendAccepted: ({ name }: { name: string }) => `You are now friends with ${name}`, friendAcceptedGeneric: 'Friend request accepted', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Connecting...', + muted: 'Muted', + active: 'Active', + error: 'Error', + default: 'Voice', + }, + mute: 'Mute', + unmute: 'Unmute', + end: 'End', } } as const; diff --git a/sources/text/translations/ca.ts b/sources/text/translations/ca.ts index eb19f1f32..bc01a8f41 100644 --- a/sources/text/translations/ca.ts +++ b/sources/text/translations/ca.ts @@ -877,6 +877,20 @@ export const ca: TranslationStructure = { friendRequestGeneric: 'Nova sol·licitud d\'amistat', friendAccepted: ({ name }: { name: string }) => `Ara ets amic de ${name}`, friendAcceptedGeneric: 'Sol·licitud d\'amistat acceptada', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Connectant...', + muted: 'Silenciat', + active: 'Actiu', + error: 'Error', + default: 'Veu', + }, + mute: 'Silenciar', + unmute: 'Activar so', + end: 'Acabar', } } as const; diff --git a/sources/text/translations/es.ts b/sources/text/translations/es.ts index 001046466..205b6afe2 100644 --- a/sources/text/translations/es.ts +++ b/sources/text/translations/es.ts @@ -878,6 +878,20 @@ export const es: TranslationStructure = { friendRequestGeneric: 'Nueva solicitud de amistad', friendAccepted: ({ name }: { name: string }) => `Ahora eres amigo de ${name}`, friendAcceptedGeneric: 'Solicitud de amistad aceptada', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Conectando...', + muted: 'Silenciado', + active: 'Activo', + error: 'Error', + default: 'Voz', + }, + mute: 'Silenciar', + unmute: 'Activar', + end: 'Finalizar', } } as const; diff --git a/sources/text/translations/pl.ts b/sources/text/translations/pl.ts index fce71c8d0..318f3ed54 100644 --- a/sources/text/translations/pl.ts +++ b/sources/text/translations/pl.ts @@ -901,6 +901,20 @@ export const pl: TranslationStructure = { friendRequestGeneric: 'Nowe zaproszenie do znajomych', friendAccepted: ({ name }: { name: string }) => `Jesteś teraz znajomym z ${name}`, friendAcceptedGeneric: 'Zaproszenie do znajomych zaakceptowane', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Łączenie...', + muted: 'Wyciszony', + active: 'Aktywny', + error: 'Błąd', + default: 'Głos', + }, + mute: 'Wycisz', + unmute: 'Włącz dźwięk', + end: 'Zakończ', } } as const; diff --git a/sources/text/translations/pt.ts b/sources/text/translations/pt.ts index 648280f55..46a6dc37f 100644 --- a/sources/text/translations/pt.ts +++ b/sources/text/translations/pt.ts @@ -877,6 +877,20 @@ export const pt: TranslationStructure = { friendRequestGeneric: 'Novo pedido de amizade', friendAccepted: ({ name }: { name: string }) => `Agora você é amigo de ${name}`, friendAcceptedGeneric: 'Pedido de amizade aceito', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Conectando...', + muted: 'Mudo', + active: 'Ativo', + error: 'Erro', + default: 'Voz', + }, + mute: 'Silenciar', + unmute: 'Ativar som', + end: 'Encerrar', } } as const; diff --git a/sources/text/translations/ru.ts b/sources/text/translations/ru.ts index fc1199d99..376ac5983 100644 --- a/sources/text/translations/ru.ts +++ b/sources/text/translations/ru.ts @@ -900,6 +900,20 @@ export const ru: TranslationStructure = { friendRequestGeneric: 'Новый запрос в друзья', friendAccepted: ({ name }: { name: string }) => `Вы теперь друзья с ${name}`, friendAcceptedGeneric: 'Запрос в друзья принят', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: 'Подключение...', + muted: 'Без звука', + active: 'Активен', + error: 'Ошибка', + default: 'Голос', + }, + mute: 'Выкл. звук', + unmute: 'Вкл. звук', + end: 'Завершить', } } as const; diff --git a/sources/text/translations/zh-Hans.ts b/sources/text/translations/zh-Hans.ts index 4813966a3..cac7f4945 100644 --- a/sources/text/translations/zh-Hans.ts +++ b/sources/text/translations/zh-Hans.ts @@ -879,5 +879,19 @@ export const zhHans: TranslationStructure = { friendRequestGeneric: '新的好友请求', friendAccepted: ({ name }: { name: string }) => `您现在与 ${name} 成为了好友`, friendAcceptedGeneric: '好友请求已接受', + }, + + voiceAssistant: { + // Voice assistant status bar + status: { + connecting: '连接中...', + muted: '已静音', + active: '活跃', + error: '错误', + default: '语音', + }, + mute: '静音', + unmute: '取消静音', + end: '结束', } } as const;