diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f33051e --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Gemini API (required for voice streaming) +GOOGLE_GENAI_USE_VERTEXAI=FALSE +GOOGLE_API_KEY=your_gemini_api_key + +# Voice model must support Gemini Live API +HOME_AGENT_MODEL=gemini-2.0-flash-live-001 + +# Home Assistant +HOME_ASSISTANT_URL=http://homeassistant.local:8123 +HOME_ASSISTANT_TOKEN=your_long_lived_access_token + +# Voice server +VOICE_SERVER_HOST=0.0.0.0 +VOICE_SERVER_PORT=8000 + +# Optional: local Ollama for text-only dev with doc_agent +OLLAMA_API_BASE=http://localhost:11434 diff --git a/.gitignore b/.gitignore index 2cdfded..391f7cc 100644 --- a/.gitignore +++ b/.gitignore @@ -58,8 +58,12 @@ ENV/ env.bak/ venv.bak/ -# macOS +# macOS / Xcode .DS_Store +*.xcuserstate +xcuserdata/ +DerivedData/ +build/ # VSCode .vscode/ diff --git a/Makefile b/Makefile index 455dbd0..e03fe5e 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,14 @@ test: ## Run all tests --cov-report=term-missing \ --junitxml=junit.xml +.PHONY: voice +voice: ## Run the iPhone-friendly voice server + @PYTHONPATH=src/agents:src python3 -m server.voice_server + +.PHONY: home_web +home_web: ## Run ADK web UI for the home agent + @uv run adk web --reload src/agents/home_agent + .PHONY: web web: ## Run the ADK web demo server @uv run adk web --reload src/agents/ diff --git a/README.md b/README.md index 6f250e5..bfc4367 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,11 @@ make prefect-server - **`doc_agent`**: Technical writing assistant with Google Docs integration - Uses local Ollama models (`gpt-oss:20b`) via LiteLLM - Includes Google Docs tools for creating and formatting documents +- **`home_agent`**: Voice-controlled home assistant for iPhone and browsers + - Uses Gemini Live API for real-time voice conversations + - Pluggable skills (Home Assistant, routines) via `config/skills.yaml` + - Safety guardrails via `config/guardrails.yaml` + - Native iOS app (`ios/`) and web PWA served by the voice WebSocket server ### Workflows - **`pipeline.py`**: Prefect workflow for orchestrating document creation and content generation @@ -45,6 +50,8 @@ make prefect-server ```bash make web # Run ADK web interface +make home_web # Run ADK web UI for home agent only +make voice # Run iPhone-friendly voice server make api_server # Run ADK FastAPI server make prefect-server # Start Prefect server and serve flows make prefect-flows # Serve flows (server must be running) @@ -57,18 +64,52 @@ make check # Lint and type check ``` src/ ├── agents/ -│ └── doc_agent/ # Technical writing assistant -│ ├── agent.py # Agent definition +│ ├── doc_agent/ # Technical writing assistant +│ │ ├── agent.py +│ │ └── tools/ +│ └── home_agent/ # Voice home assistant +│ ├── agent.py +│ ├── config/ # skills.yaml, guardrails.yaml +│ ├── guardrails/ +│ ├── skills/ │ └── tools/ -│ └── google_docs_tool.py # Google Docs integration +├── server/ +│ ├── voice_server.py # WebSocket voice server + PWA +│ └── static/ # Browser web client └── workflows/ - ├── pipeline.py # Main Prefect workflow - ├── serve.py # Flow serving entry point - └── discover.py # Flow discovery utility +ios/ +└── HomeVoiceAgent.xcodeproj # Native iPhone app (SwiftUI) ``` ## Configuration - **Ollama**: Set `OLLAMA_API_BASE` in `.env` or environment - **Google Docs**: Place `credentials.json` in project root (token.json auto-generated) +- **Home Voice Agent**: Copy `.env.example` to `.env` and set: + - `GOOGLE_API_KEY` — Gemini API key (required for voice) + - `HOME_ASSISTANT_URL` and `HOME_ASSISTANT_TOKEN` — Home Assistant access + - Customize skills in `src/agents/home_agent/config/skills.yaml` + - Customize guardrails in `src/agents/home_agent/config/guardrails.yaml` - **Prefect UI**: Available at `http://127.0.0.1:4200` (or port specified by `PREFECT_PORT`) + +## Home Voice Agent (iPhone) + +### Native iOS app (recommended) + +1. Copy `.env.example` to `.env` and add your Gemini + Home Assistant credentials +2. Start the voice server on your Mac: `make voice` +3. Open `ios/HomeVoiceAgent.xcodeproj` in Xcode +4. Set your development team, build to your iPhone +5. In app Settings, enter your server host (e.g. `192.168.1.42:8000`) +6. Tap **Start Voice** and talk to your home + +See `ios/README.md` for full setup and remote access options. + +### Web client (alternative) + +1. Start the voice server: `make voice` +2. On your iPhone (same Wi‑Fi), open `http://:8000` +3. Tap **Start Voice**, allow the microphone, and talk to your home + +For HTTPS/WSS from your phone outside the LAN, put the server behind a reverse proxy +or tunnel (e.g. Cloudflare Tunnel, Tailscale) and use `wss://`. diff --git a/ios/HomeVoiceAgent.xcodeproj/project.pbxproj b/ios/HomeVoiceAgent.xcodeproj/project.pbxproj new file mode 100644 index 0000000..0ff3ee0 --- /dev/null +++ b/ios/HomeVoiceAgent.xcodeproj/project.pbxproj @@ -0,0 +1,314 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + AA0000010000000000000001 /* HomeVoiceAgentApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000001 /* HomeVoiceAgentApp.swift */; }; + AA0000010000000000000002 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000002 /* ContentView.swift */; }; + AA0000010000000000000003 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000003 /* SettingsView.swift */; }; + AA0000010000000000000004 /* ChatViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000004 /* ChatViewModel.swift */; }; + AA0000010000000000000005 /* ChatMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000005 /* ChatMessage.swift */; }; + AA0000010000000000000006 /* ServerSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000006 /* ServerSettings.swift */; }; + AA0000010000000000000007 /* AgentWebSocketClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000007 /* AgentWebSocketClient.swift */; }; + AA0000010000000000000008 /* AudioCaptureManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000008 /* AudioCaptureManager.swift */; }; + AA0000010000000000000009 /* AudioPlaybackManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000020000000000000009 /* AudioPlaybackManager.swift */; }; + AA000001000000000000000A /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = AA000002000000000000000A /* Assets.xcassets */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + AA0000020000000000000001 /* HomeVoiceAgentApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeVoiceAgentApp.swift; sourceTree = ""; }; + AA0000020000000000000002 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; + AA0000020000000000000003 /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; + AA0000020000000000000004 /* ChatViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatViewModel.swift; sourceTree = ""; }; + AA0000020000000000000005 /* ChatMessage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatMessage.swift; sourceTree = ""; }; + AA0000020000000000000006 /* ServerSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerSettings.swift; sourceTree = ""; }; + AA0000020000000000000007 /* AgentWebSocketClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentWebSocketClient.swift; sourceTree = ""; }; + AA0000020000000000000008 /* AudioCaptureManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCaptureManager.swift; sourceTree = ""; }; + AA0000020000000000000009 /* AudioPlaybackManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlaybackManager.swift; sourceTree = ""; }; + AA000002000000000000000A /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + AA000002000000000000000B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + AA0000030000000000000001 /* HomeVoiceAgent.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = HomeVoiceAgent.app; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + AA0000040000000000000001 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + AA0000050000000000000001 = { + isa = PBXGroup; + children = ( + AA0000050000000000000002 /* HomeVoiceAgent */, + AA0000050000000000000003 /* Products */, + ); + sourceTree = ""; + }; + AA0000050000000000000002 /* HomeVoiceAgent */ = { + isa = PBXGroup; + children = ( + AA0000020000000000000001 /* HomeVoiceAgentApp.swift */, + AA0000020000000000000002 /* ContentView.swift */, + AA0000020000000000000003 /* SettingsView.swift */, + AA0000050000000000000004 /* ViewModels */, + AA0000050000000000000005 /* Services */, + AA0000050000000000000006 /* Models */, + AA000002000000000000000A /* Assets.xcassets */, + AA000002000000000000000B /* Info.plist */, + ); + path = HomeVoiceAgent; + sourceTree = ""; + }; + AA0000050000000000000003 /* Products */ = { + isa = PBXGroup; + children = ( + AA0000030000000000000001 /* HomeVoiceAgent.app */, + ); + name = Products; + sourceTree = ""; + }; + AA0000050000000000000004 /* ViewModels */ = { + isa = PBXGroup; + children = ( + AA0000020000000000000004 /* ChatViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + AA0000050000000000000005 /* Services */ = { + isa = PBXGroup; + children = ( + AA0000020000000000000007 /* AgentWebSocketClient.swift */, + AA0000020000000000000008 /* AudioCaptureManager.swift */, + AA0000020000000000000009 /* AudioPlaybackManager.swift */, + ); + path = Services; + sourceTree = ""; + }; + AA0000050000000000000006 /* Models */ = { + isa = PBXGroup; + children = ( + AA0000020000000000000005 /* ChatMessage.swift */, + AA0000020000000000000006 /* ServerSettings.swift */, + ); + path = Models; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + AA0000060000000000000001 /* HomeVoiceAgent */ = { + isa = PBXNativeTarget; + buildConfigurationList = AA0000090000000000000001 /* Build configuration list for PBXNativeTarget "HomeVoiceAgent" */; + buildPhases = ( + AA0000070000000000000001 /* Sources */, + AA0000040000000000000001 /* Frameworks */, + AA0000080000000000000001 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = HomeVoiceAgent; + productName = HomeVoiceAgent; + productReference = AA0000030000000000000001 /* HomeVoiceAgent.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + AA00000A0000000000000001 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 1500; + LastUpgradeCheck = 1500; + TargetAttributes = { + AA0000060000000000000001 = { + CreatedOnToolsVersion = 15.0; + }; + }; + }; + buildConfigurationList = AA0000090000000000000002 /* Build configuration list for PBXProject "HomeVoiceAgent" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = AA0000050000000000000001; + productRefGroup = AA0000050000000000000003 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + AA0000060000000000000001 /* HomeVoiceAgent */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + AA0000080000000000000001 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + AA000001000000000000000A /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + AA0000070000000000000001 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + AA0000010000000000000001 /* HomeVoiceAgentApp.swift in Sources */, + AA0000010000000000000002 /* ContentView.swift in Sources */, + AA0000010000000000000003 /* SettingsView.swift in Sources */, + AA0000010000000000000004 /* ChatViewModel.swift in Sources */, + AA0000010000000000000005 /* ChatMessage.swift in Sources */, + AA0000010000000000000006 /* ServerSettings.swift in Sources */, + AA0000010000000000000007 /* AgentWebSocketClient.swift in Sources */, + AA0000010000000000000008 /* AudioCaptureManager.swift in Sources */, + AA0000010000000000000009 /* AudioPlaybackManager.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + AA00000B0000000000000001 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_DYNAMIC_NO_PIC = NO; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + AA00000B0000000000000002 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_OPTIMIZATION_LEVEL = s; + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + AA00000B0000000000000003 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = NO; + INFOPLIST_FILE = HomeVoiceAgent/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations = UIInterfaceOrientationPortrait; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.agentexperiments.HomeVoiceAgent; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = 1; + }; + name = Debug; + }; + AA00000B0000000000000004 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = NO; + INFOPLIST_FILE = HomeVoiceAgent/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations = UIInterfaceOrientationPortrait; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.agentexperiments.HomeVoiceAgent; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = 1; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + AA0000090000000000000001 /* Build configuration list for PBXNativeTarget "HomeVoiceAgent" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AA00000B0000000000000003 /* Debug */, + AA00000B0000000000000004 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + AA0000090000000000000002 /* Build configuration list for PBXProject "HomeVoiceAgent" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AA00000B0000000000000001 /* Debug */, + AA00000B0000000000000002 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = AA00000A0000000000000001 /* Project object */; +} diff --git a/ios/HomeVoiceAgent.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios/HomeVoiceAgent.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/ios/HomeVoiceAgent.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/ios/HomeVoiceAgent.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ios/HomeVoiceAgent.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..e69de29 diff --git a/ios/HomeVoiceAgent.xcodeproj/xcshareddata/xcschemes/HomeVoiceAgent.xcscheme b/ios/HomeVoiceAgent.xcodeproj/xcshareddata/xcschemes/HomeVoiceAgent.xcscheme new file mode 100644 index 0000000..def9919 --- /dev/null +++ b/ios/HomeVoiceAgent.xcodeproj/xcshareddata/xcschemes/HomeVoiceAgent.xcscheme @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + diff --git a/ios/HomeVoiceAgent/Assets.xcassets/AccentColor.colorset/Contents.json b/ios/HomeVoiceAgent/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 0000000..c5f774e --- /dev/null +++ b/ios/HomeVoiceAgent/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,20 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0.160", + "green" : "0.090", + "red" : "0.060" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios/HomeVoiceAgent/Assets.xcassets/AppIcon.appiconset/Contents.json b/ios/HomeVoiceAgent/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..13613e3 --- /dev/null +++ b/ios/HomeVoiceAgent/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios/HomeVoiceAgent/Assets.xcassets/Contents.json b/ios/HomeVoiceAgent/Assets.xcassets/Contents.json new file mode 100644 index 0000000..73c0059 --- /dev/null +++ b/ios/HomeVoiceAgent/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios/HomeVoiceAgent/ContentView.swift b/ios/HomeVoiceAgent/ContentView.swift new file mode 100644 index 0000000..8ed3358 --- /dev/null +++ b/ios/HomeVoiceAgent/ContentView.swift @@ -0,0 +1,200 @@ +import SwiftUI + +struct ContentView: View { + @EnvironmentObject private var viewModel: ChatViewModel + @State private var showSettings = false + + var body: some View { + NavigationStack { + VStack(spacing: 16) { + statusBar + messageList + composer + voiceControls + } + .padding() + .background(Color(red: 0.06, green: 0.09, blue: 0.16)) + .navigationTitle("Home Voice Agent") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button { + showSettings = true + } label: { + Image(systemName: "gearshape") + } + } + } + .sheet(isPresented: $showSettings) { + SettingsView() + .environmentObject(viewModel) + } + .onAppear { + if viewModel.connectionState == .disconnected { + viewModel.connect() + } + } + } + .preferredColorScheme(.dark) + } + + private var statusBar: some View { + HStack { + Circle() + .fill(statusColor) + .frame(width: 10, height: 10) + Text(viewModel.connectionState.rawValue) + .font(.subheadline) + .foregroundStyle(.secondary) + Spacer() + } + } + + private var statusColor: Color { + switch viewModel.connectionState { + case .connected, .voiceActive: + return .green + case .connecting: + return .yellow + case .disconnected: + return .red + } + } + + private var messageList: some View { + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 12) { + ForEach(viewModel.messages) { message in + MessageBubble(message: message) + .id(message.id) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .onChange(of: viewModel.messages.count) { _, _ in + if let lastID = viewModel.messages.last?.id { + withAnimation { + proxy.scrollTo(lastID, anchor: .bottom) + } + } + } + } + .padding() + .background(Color(red: 0.07, green: 0.1, blue: 0.22)) + .clipShape(RoundedRectangle(cornerRadius: 16)) + } + + private var composer: some View { + HStack(spacing: 8) { + TextField("Type a message...", text: $viewModel.draftText) + .textFieldStyle(.roundedBorder) + .submitLabel(.send) + .onSubmit { + viewModel.sendText() + } + + Button("Send") { + viewModel.sendText() + } + .buttonStyle(.borderedProminent) + .disabled(viewModel.connectionState == .disconnected) + } + } + + private var voiceControls: some View { + VStack(spacing: 8) { + if viewModel.isVoiceEnabled { + Button("Stop Voice") { + viewModel.stopVoice() + viewModel.connect() + } + .buttonStyle(.bordered) + .tint(.orange) + } else { + Button { + Task { + await viewModel.startVoice() + } + } label: { + Label("Start Voice", systemImage: "mic.fill") + .frame(maxWidth: .infinity) + } + .buttonStyle(.borderedProminent) + .tint(.cyan) + } + + if let errorMessage = viewModel.errorMessage { + Text(errorMessage) + .font(.footnote) + .foregroundStyle(.red) + .multilineTextAlignment(.center) + } else { + Text("Talk naturally to control lights, climate, and routines.") + .font(.footnote) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + } + } + } +} + +private struct MessageBubble: View { + let message: ChatMessage + + var body: some View { + HStack { + if message.role == .user { + Spacer(minLength: 24) + } + + Text(prefix + message.text) + .padding(12) + .background(backgroundColor) + .foregroundStyle(foregroundColor) + .clipShape(RoundedRectangle(cornerRadius: 14)) + + if message.role == .agent || message.role == .system { + Spacer(minLength: 24) + } + } + } + + private var prefix: String { + switch message.role { + case .user: + return "" + case .agent: + return "" + case .system: + return "ℹ︎ " + } + } + + private var backgroundColor: Color { + switch message.role { + case .user: + return Color.cyan.opacity(0.25) + case .agent: + return Color.white.opacity(0.08) + case .system: + return Color.yellow.opacity(0.12) + } + } + + private var foregroundColor: Color { + switch message.role { + case .user: + return .white + case .agent: + return .white + case .system: + return .yellow + } + } +} + +#Preview { + ContentView() + .environmentObject(ChatViewModel()) +} diff --git a/ios/HomeVoiceAgent/HomeVoiceAgentApp.swift b/ios/HomeVoiceAgent/HomeVoiceAgentApp.swift new file mode 100644 index 0000000..be77f55 --- /dev/null +++ b/ios/HomeVoiceAgent/HomeVoiceAgentApp.swift @@ -0,0 +1,13 @@ +import SwiftUI + +@main +struct HomeVoiceAgentApp: App { + @StateObject private var viewModel = ChatViewModel() + + var body: some Scene { + WindowGroup { + ContentView() + .environmentObject(viewModel) + } + } +} diff --git a/ios/HomeVoiceAgent/Info.plist b/ios/HomeVoiceAgent/Info.plist new file mode 100644 index 0000000..19bf16f --- /dev/null +++ b/ios/HomeVoiceAgent/Info.plist @@ -0,0 +1,46 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + Home Agent + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + NSMicrophoneUsageDescription + Home Voice Agent needs microphone access so you can talk to your home assistant. + NSLocalNetworkUsageDescription + Home Voice Agent connects to your home server on the local network. + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + + UILaunchScreen + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + + NSAppTransportSecurity + + NSAllowsLocalNetworking + + + + diff --git a/ios/HomeVoiceAgent/Models/ChatMessage.swift b/ios/HomeVoiceAgent/Models/ChatMessage.swift new file mode 100644 index 0000000..7f6f69d --- /dev/null +++ b/ios/HomeVoiceAgent/Models/ChatMessage.swift @@ -0,0 +1,45 @@ +import Foundation + +enum ChatRole { + case user + case agent + case system +} + +struct ChatMessage: Identifiable, Equatable { + let id: String + let role: ChatRole + var text: String + var isStreaming: Bool + + init(id: String = UUID().uuidString, role: ChatRole, text: String, isStreaming: Bool = false) { + self.id = id + self.role = role + self.text = text + self.isStreaming = isStreaming + } +} + +struct ServerMessage: Decodable { + let turnComplete: Bool? + let interrupted: Bool? + let mimeType: String? + let data: String? + + enum CodingKeys: String, CodingKey { + case turnComplete = "turn_complete" + case interrupted + case mimeType = "mime_type" + case data + } +} + +struct OutboundMessage: Encodable { + let mimeType: String + let data: String + + enum CodingKeys: String, CodingKey { + case mimeType = "mime_type" + case data + } +} diff --git a/ios/HomeVoiceAgent/Models/ServerSettings.swift b/ios/HomeVoiceAgent/Models/ServerSettings.swift new file mode 100644 index 0000000..39cc828 --- /dev/null +++ b/ios/HomeVoiceAgent/Models/ServerSettings.swift @@ -0,0 +1,34 @@ +import Foundation + +struct ServerSettings { + private enum Keys { + static let host = "home_agent_server_host" + static let useTLS = "home_agent_server_use_tls" + static let userID = "home_agent_user_id" + } + + static var host: String { + get { UserDefaults.standard.string(forKey: Keys.host) ?? "192.168.1.100:8000" } + set { UserDefaults.standard.set(newValue, forKey: Keys.host) } + } + + static var useTLS: Bool { + get { UserDefaults.standard.bool(forKey: Keys.useTLS) } + set { UserDefaults.standard.set(newValue, forKey: Keys.useTLS) } + } + + static var userID: String { + if let existing = UserDefaults.standard.string(forKey: Keys.userID) { + return existing + } + let generated = UUID().uuidString.prefix(10).description + UserDefaults.standard.set(generated, forKey: Keys.userID) + return generated + } + + static func websocketURL(audioMode: Bool) -> URL? { + let scheme = useTLS ? "wss" : "ws" + let audioFlag = audioMode ? "true" : "false" + return URL(string: "\(scheme)://\(host)/ws/\(userID)?is_audio=\(audioFlag)") + } +} diff --git a/ios/HomeVoiceAgent/Services/AgentWebSocketClient.swift b/ios/HomeVoiceAgent/Services/AgentWebSocketClient.swift new file mode 100644 index 0000000..ca3b58a --- /dev/null +++ b/ios/HomeVoiceAgent/Services/AgentWebSocketClient.swift @@ -0,0 +1,106 @@ +import Foundation + +enum AgentWebSocketError: LocalizedError { + case invalidURL + case notConnected + + var errorDescription: String? { + switch self { + case .invalidURL: + return "Server URL is invalid." + case .notConnected: + return "Not connected to the voice server." + } + } +} + +final class AgentWebSocketClient: @unchecked Sendable { + private let session = URLSession(configuration: .default) + private var task: URLSessionWebSocketTask? + private let encoder = JSONEncoder() + private let decoder = JSONDecoder() + + var isConnected: Bool { + task?.state == .running + } + + func connect(url: URL) async throws { + disconnect() + let socketTask = session.webSocketTask(with: url) + task = socketTask + socketTask.resume() + try await waitForConnection(socketTask) + } + + func disconnect() { + task?.cancel(with: .goingAway, reason: nil) + task = nil + } + + func send(text: String) async throws { + try await send(message: OutboundMessage(mimeType: "text/plain", data: text)) + } + + func send(pcmData: Data) async throws { + let encoded = pcmData.base64EncodedString() + try await send(message: OutboundMessage(mimeType: "audio/pcm", data: encoded)) + } + + func messages() -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let receiveTask = Task { + do { + while !Task.isCancelled { + guard let socketTask = self.task else { + throw AgentWebSocketError.notConnected + } + let message = try await socketTask.receive() + guard let serverMessage = try self.decode(message) else { + continue + } + continuation.yield(serverMessage) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + + continuation.onTermination = { _ in + receiveTask.cancel() + } + } + } + + private func send(message: OutboundMessage) async throws { + guard let socketTask = task else { + throw AgentWebSocketError.notConnected + } + let payload = try encoder.encode(message) + let text = String(decoding: payload, as: UTF8.self) + try await socketTask.send(.string(text)) + } + + private func decode(_ message: URLSessionWebSocketTask.Message) throws -> ServerMessage? { + switch message { + case .string(let text): + return try decoder.decode(ServerMessage.self, from: Data(text.utf8)) + case .data(let data): + return try decoder.decode(ServerMessage.self, from: data) + @unknown default: + return nil + } + } + + private func waitForConnection(_ socketTask: URLSessionWebSocketTask) async throws { + try await withCheckedThrowingContinuation { continuation in + socketTask.sendPing { error in + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume() + } + } + } + } +} diff --git a/ios/HomeVoiceAgent/Services/AudioCaptureManager.swift b/ios/HomeVoiceAgent/Services/AudioCaptureManager.swift new file mode 100644 index 0000000..1a1f2fb --- /dev/null +++ b/ios/HomeVoiceAgent/Services/AudioCaptureManager.swift @@ -0,0 +1,74 @@ +import AVFoundation + +final class AudioCaptureManager { + private let engine = AVAudioEngine() + private var converter: AVAudioConverter? + private let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: 16_000, + channels: 1, + interleaved: true + ) + private var onPCMData: ((Data) -> Void)? + + func start(onPCMData: @escaping (Data) -> Void) throws { + guard let targetFormat else { + throw NSError(domain: "AudioCaptureManager", code: 1) + } + + self.onPCMData = onPCMData + + let session = AVAudioSession.sharedInstance() + try session.setCategory( + .playAndRecord, + mode: .voiceChat, + options: [.defaultToSpeaker, .allowBluetooth] + ) + try session.setActive(true) + + let inputNode = engine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + converter = AVAudioConverter(from: inputFormat, to: targetFormat) + + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [weak self] buffer, _ in + self?.process(buffer: buffer, targetFormat: targetFormat) + } + + engine.prepare() + try engine.start() + } + + func stop() { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + onPCMData = nil + converter = nil + } + + private func process(buffer: AVAudioPCMBuffer, targetFormat: AVAudioFormat) { + guard let converter else { return } + + let ratio = targetFormat.sampleRate / buffer.format.sampleRate + let frameCapacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 1 + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: frameCapacity) else { + return + } + + var error: NSError? + let inputBlock: AVAudioConverterInputBlock = { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + + converter.convert(to: outputBuffer, error: &error, withInputFrom: inputBlock) + guard error == nil, let channelData = outputBuffer.int16ChannelData else { return } + + let frames = Int(outputBuffer.frameLength) + guard frames > 0 else { return } + + let byteCount = frames * MemoryLayout.size + let data = Data(bytes: channelData[0], count: byteCount) + onPCMData?(data) + } +} diff --git a/ios/HomeVoiceAgent/Services/AudioPlaybackManager.swift b/ios/HomeVoiceAgent/Services/AudioPlaybackManager.swift new file mode 100644 index 0000000..031c2da --- /dev/null +++ b/ios/HomeVoiceAgent/Services/AudioPlaybackManager.swift @@ -0,0 +1,52 @@ +import AVFoundation + +final class AudioPlaybackManager { + private let engine = AVAudioEngine() + private let playerNode = AVAudioPlayerNode() + private let format = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: 24_000, + channels: 1, + interleaved: true + ) + + func start() throws { + guard let format else { + throw NSError(domain: "AudioPlaybackManager", code: 1) + } + + engine.attach(playerNode) + engine.connect(playerNode, to: engine.mainMixerNode, format: format) + engine.prepare() + try engine.start() + playerNode.play() + } + + func enqueue(pcmData: Data) { + guard let format else { return } + + let frameCount = pcmData.count / MemoryLayout.size + guard frameCount > 0, + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)), + let channelData = buffer.int16ChannelData else { + return + } + + buffer.frameLength = AVAudioFrameCount(frameCount) + pcmData.withUnsafeBytes { rawBuffer in + guard let source = rawBuffer.bindMemory(to: Int16.self).baseAddress else { return } + channelData[0].update(from: source, count: frameCount) + } + playerNode.scheduleBuffer(buffer) + } + + func interrupt() { + playerNode.stop() + playerNode.play() + } + + func stop() { + playerNode.stop() + engine.stop() + } +} diff --git a/ios/HomeVoiceAgent/SettingsView.swift b/ios/HomeVoiceAgent/SettingsView.swift new file mode 100644 index 0000000..dbea9be --- /dev/null +++ b/ios/HomeVoiceAgent/SettingsView.swift @@ -0,0 +1,61 @@ +import SwiftUI + +struct SettingsView: View { + @EnvironmentObject private var viewModel: ChatViewModel + @Environment(\.dismiss) private var dismiss + + @State private var host = ServerSettings.host + @State private var useTLS = ServerSettings.useTLS + + var body: some View { + NavigationStack { + Form { + Section("Server") { + TextField("Host", text: $host, prompt: Text("192.168.1.100:8000")) + .textInputAutocapitalization(.never) + .autocorrectionDisabled() + + Toggle("Use TLS (wss://)", isOn: $useTLS) + } + + Section("Connection") { + LabeledContent("User ID", value: ServerSettings.userID) + + Button("Reconnect") { + saveSettings() + viewModel.disconnect() + viewModel.connect() + } + + Button("Disconnect", role: .destructive) { + viewModel.disconnect() + } + } + + Section("Help") { + Text("Run `make voice` on your home server, then enter your computer's LAN IP and port here.") + Text("For remote access, use Tailscale or HTTPS with wss:// enabled.") + } + } + .navigationTitle("Settings") + .toolbar { + ToolbarItem(placement: .confirmationAction) { + Button("Done") { + saveSettings() + dismiss() + } + } + } + } + } + + private func saveSettings() { + ServerSettings.host = host.trimmingCharacters(in: .whitespacesAndNewlines) + ServerSettings.useTLS = useTLS + } +} + +#Preview { + SettingsView() + .environmentObject(ChatViewModel()) +} diff --git a/ios/HomeVoiceAgent/ViewModels/ChatViewModel.swift b/ios/HomeVoiceAgent/ViewModels/ChatViewModel.swift new file mode 100644 index 0000000..fbac5a8 --- /dev/null +++ b/ios/HomeVoiceAgent/ViewModels/ChatViewModel.swift @@ -0,0 +1,182 @@ +import AVFoundation +import Foundation + +@MainActor +final class ChatViewModel: ObservableObject { + enum ConnectionState: String { + case disconnected = "Disconnected" + case connecting = "Connecting" + case connected = "Connected" + case voiceActive = "Voice Active" + } + + @Published var messages: [ChatMessage] = [] + @Published var connectionState: ConnectionState = .disconnected + @Published var draftText = "" + @Published var isVoiceEnabled = false + @Published var errorMessage: String? + + private let webSocketClient = AgentWebSocketClient() + private let audioCapture = AudioCaptureManager() + private let audioPlayback = AudioPlaybackManager() + + private var receiveTask: Task? + private var streamingMessageID: String? + + func connect() { + guard let url = ServerSettings.websocketURL(audioMode: isVoiceEnabled) else { + errorMessage = AgentWebSocketError.invalidURL.localizedDescription + return + } + + receiveTask?.cancel() + connectionState = .connecting + errorMessage = nil + + receiveTask = Task { + do { + try await webSocketClient.connect(url: url) + connectionState = isVoiceEnabled ? .voiceActive : .connected + appendSystemMessage("Connected to your home agent.") + + for try await message in webSocketClient.messages() { + handle(serverMessage: message) + } + + connectionState = .disconnected + } catch is CancellationError { + return + } catch { + connectionState = .disconnected + errorMessage = error.localizedDescription + appendSystemMessage("Connection lost. Tap reconnect in Settings.") + } + } + } + + func disconnect() { + receiveTask?.cancel() + receiveTask = nil + stopVoice() + webSocketClient.disconnect() + connectionState = .disconnected + } + + func sendText() { + let text = draftText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + + messages.append(ChatMessage(role: .user, text: text)) + draftText = "" + + Task { + do { + try await webSocketClient.send(text: text) + } catch { + errorMessage = error.localizedDescription + } + } + } + + func startVoice() async { + let granted = await requestMicrophonePermission() + guard granted else { + errorMessage = "Microphone access is required for voice mode." + return + } + + isVoiceEnabled = true + + do { + try audioPlayback.start() + try audioCapture.start { [weak self] pcmData in + Task { @MainActor in + await self?.sendAudioChunk(pcmData) + } + } + connect() + appendSystemMessage("Voice mode enabled. Start speaking.") + } catch { + errorMessage = error.localizedDescription + stopVoice() + } + } + + func stopVoice() { + isVoiceEnabled = false + audioCapture.stop() + audioPlayback.stop() + if connectionState == .voiceActive { + connectionState = .disconnected + } + } + + private func sendAudioChunk(_ pcmData: Data) async { + guard isVoiceEnabled, webSocketClient.isConnected else { return } + do { + try await webSocketClient.send(pcmData: pcmData) + } catch { + errorMessage = error.localizedDescription + } + } + + private func handle(serverMessage: ServerMessage) { + if serverMessage.turnComplete == true { + finalizeStreamingMessage() + return + } + + if serverMessage.interrupted == true { + audioPlayback.interrupt() + finalizeStreamingMessage() + return + } + + guard let mimeType = serverMessage.mimeType, let data = serverMessage.data else { + return + } + + if mimeType == "audio/pcm", let pcmData = Data(base64Encoded: data) { + audioPlayback.enqueue(pcmData: pcmData) + return + } + + if mimeType == "text/plain" { + appendStreamingAgentText(data) + } + } + + private func appendStreamingAgentText(_ chunk: String) { + if let streamingMessageID, + let index = messages.firstIndex(where: { $0.id == streamingMessageID }) { + messages[index].text += chunk + return + } + + let message = ChatMessage(role: .agent, text: chunk, isStreaming: true) + streamingMessageID = message.id + messages.append(message) + } + + private func finalizeStreamingMessage() { + guard let streamingMessageID, + let index = messages.firstIndex(where: { $0.id == streamingMessageID }) else { + streamingMessageID = nil + return + } + messages[index].isStreaming = false + self.streamingMessageID = nil + } + + private func appendSystemMessage(_ text: String) { + messages.append(ChatMessage(role: .system, text: text)) + } + + private func requestMicrophonePermission() async -> Bool { + await withCheckedContinuation { continuation in + AVAudioSession.sharedInstance().requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + } +} diff --git a/ios/README.md b/ios/README.md new file mode 100644 index 0000000..25a2e24 --- /dev/null +++ b/ios/README.md @@ -0,0 +1,69 @@ +# Home Voice Agent — Native iOS App + +Native SwiftUI iPhone app for talking to your home agent over the voice WebSocket server. + +## Requirements + +- macOS with Xcode 15+ +- iPhone running iOS 17+ (or adjust deployment target in Xcode) +- Home voice server running (`make voice` from repo root) + +## Open in Xcode + +1. Open `ios/HomeVoiceAgent.xcodeproj` in Xcode +2. Select your development team under **Signing & Capabilities** +3. Build and run on your iPhone (same Wi‑Fi as the server) + +## Configure the server + +In the app **Settings** screen: + +| Field | Example | +|-------|---------| +| Host | `192.168.1.42:8000` | +| Use TLS | Off for local dev, On for remote `wss://` | + +The app connects to: + +``` +ws(s):///ws/?is_audio=true +``` + +## Features + +- Real-time voice via microphone (16 kHz PCM uplink) +- Agent voice playback (24 kHz PCM downlink) +- Text chat fallback +- Persistent server settings +- Reconnect from Settings + +## Project structure + +``` +ios/HomeVoiceAgent/ +├── HomeVoiceAgentApp.swift # App entry +├── ContentView.swift # Main chat UI +├── SettingsView.swift # Server configuration +├── ViewModels/ChatViewModel.swift +├── Services/ +│ ├── AgentWebSocketClient.swift +│ ├── AudioCaptureManager.swift +│ └── AudioPlaybackManager.swift +└── Models/ + ├── ChatMessage.swift + └── ServerSettings.swift +``` + +## Remote access + +For use outside your home network: + +1. Put the Python voice server behind HTTPS (Tailscale, Cloudflare Tunnel, or reverse proxy) +2. Enable **Use TLS** in app settings +3. Enter your public hostname, e.g. `home-agent.example.com` + +## Troubleshooting + +- **Cannot connect**: Confirm `make voice` is running and the host IP is reachable from your phone +- **No audio**: Check microphone permission in iOS Settings → Home Agent +- **Voice cuts out**: Use headphones to reduce echo feedback diff --git a/pyproject.toml b/pyproject.toml index 4a955ba..9971b4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ "google-api-python-client>=2.100.0", "prefect>=3.0.0", "markdown-it-py>=3.0.0", + "httpx>=0.27.0", + "fastapi>=0.115.0", + "python-dotenv>=1.0.0", ] [build-system] @@ -30,6 +33,10 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +where = ["src"] +include = ["agents*", "workflows*", "server*"] + [dependency-groups] dev = [ "ipykernel>=6.29.5", diff --git a/src/agents/home_agent/__init__.py b/src/agents/home_agent/__init__.py new file mode 100644 index 0000000..9659561 --- /dev/null +++ b/src/agents/home_agent/__init__.py @@ -0,0 +1,3 @@ +from . import agent + +__all__ = ["agent"] diff --git a/src/agents/home_agent/agent.py b/src/agents/home_agent/agent.py new file mode 100644 index 0000000..e35e89c --- /dev/null +++ b/src/agents/home_agent/agent.py @@ -0,0 +1,39 @@ +"""Home voice agent with configurable skills and guardrails.""" + +from __future__ import annotations + +import os + +from google.adk.agents import Agent + +from .guardrails.loader import create_guardrail_callbacks, load_guardrail_config +from .skills.loader import load_skill_tools + +DEFAULT_MODEL = os.getenv("HOME_AGENT_MODEL", "gemini-2.0-flash-live-001") + +BASE_INSTRUCTION = """ +You are a helpful home assistant that controls smart home devices by voice. + +Guidelines: +- Keep responses concise and conversational for voice. +- Confirm room and device names before making changes. +- If a request is ambiguous, ask a short clarifying question. +- Never attempt actions blocked by safety rules. +- When a routine or service call needs confirmation, ask the user to confirm. +""" + +skill_tools, skill_instructions = load_skill_tools() +guardrail_callbacks = create_guardrail_callbacks(load_guardrail_config()) + +instruction_parts = [BASE_INSTRUCTION.strip()] +if skill_instructions: + instruction_parts.append("Enabled skills:\n" + "\n".join(skill_instructions)) + +root_agent = Agent( + model=DEFAULT_MODEL, + name="home_voice_agent", + description="Voice-controlled home assistant with skills and guardrails.", + instruction="\n\n".join(instruction_parts), + tools=skill_tools, + **guardrail_callbacks, +) diff --git a/src/agents/home_agent/config/guardrails.yaml b/src/agents/home_agent/config/guardrails.yaml new file mode 100644 index 0000000..8e32794 --- /dev/null +++ b/src/agents/home_agent/config/guardrails.yaml @@ -0,0 +1,38 @@ +# Guardrails enforce safety policies before the model or tools run. +guardrails: + blocked_topics: + - unlock all doors + - disable security + - delete all + - factory reset + + blocked_tools: + - shell_execute + - run_command + + # Tools that always require explicit confirmation in the response. + confirm_tools: + - call_service + - run_routine + + # Home Assistant domains the agent may control. + allowed_domains: + - light + - switch + - climate + - scene + - script + - automation + + # Entity IDs or patterns that must never be changed. + blocked_entities: + - alarm_control_panel.* + - lock.front_door + + temperature: + min_celsius: 16 + max_celsius: 30 + + response_on_block: >- + I can't help with that request because it conflicts with your home safety + rules. I can help with lights, climate, scenes, and approved routines. diff --git a/src/agents/home_agent/config/skills.yaml b/src/agents/home_agent/config/skills.yaml new file mode 100644 index 0000000..42eca81 --- /dev/null +++ b/src/agents/home_agent/config/skills.yaml @@ -0,0 +1,12 @@ +# Skills define which tool modules the home agent can use. +# Add new skills by creating a module under skills/ and listing it here. +skills: + - name: home_assistant + enabled: true + description: Control lights, switches, climate, and scenes via Home Assistant + module: home_assistant + + - name: routines + enabled: true + description: Run predefined home routines (goodnight, away, movie mode) + module: routines diff --git a/src/agents/home_agent/guardrails/__init__.py b/src/agents/home_agent/guardrails/__init__.py new file mode 100644 index 0000000..9645043 --- /dev/null +++ b/src/agents/home_agent/guardrails/__init__.py @@ -0,0 +1,13 @@ +"""Guardrail helpers for the home voice agent.""" + +from .loader import ( + GuardrailConfig, + create_guardrail_callbacks, + load_guardrail_config, +) + +__all__ = [ + "GuardrailConfig", + "create_guardrail_callbacks", + "load_guardrail_config", +] diff --git a/src/agents/home_agent/guardrails/loader.py b/src/agents/home_agent/guardrails/loader.py new file mode 100644 index 0000000..7ea12e4 --- /dev/null +++ b/src/agents/home_agent/guardrails/loader.py @@ -0,0 +1,147 @@ +"""Load guardrail policies and wire them into ADK callbacks.""" + +from __future__ import annotations + +import fnmatch +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml +from google.adk.agents.callback_context import CallbackContext +from google.adk.models import LlmRequest, LlmResponse +from google.adk.tools.tool_context import ToolContext +from google.genai import types + +DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "guardrails.yaml" + + +@dataclass +class GuardrailConfig: + blocked_topics: list[str] = field(default_factory=list) + blocked_tools: list[str] = field(default_factory=list) + confirm_tools: list[str] = field(default_factory=list) + allowed_domains: list[str] = field(default_factory=list) + blocked_entities: list[str] = field(default_factory=list) + temperature_min_celsius: int = 16 + temperature_max_celsius: int = 30 + response_on_block: str = "I can't help with that request." + + +def load_guardrail_config(path: Path | None = None) -> GuardrailConfig: + config_path = path or DEFAULT_CONFIG_PATH + with config_path.open(encoding="utf-8") as config_file: + raw = yaml.safe_load(config_file) or {} + + guardrails = raw.get("guardrails", {}) + temperature = guardrails.get("temperature", {}) + return GuardrailConfig( + blocked_topics=[topic.lower() for topic in guardrails.get("blocked_topics", [])], + blocked_tools=guardrails.get("blocked_tools", []), + confirm_tools=guardrails.get("confirm_tools", []), + allowed_domains=guardrails.get("allowed_domains", []), + blocked_entities=guardrails.get("blocked_entities", []), + temperature_min_celsius=temperature.get("min_celsius", 16), + temperature_max_celsius=temperature.get("max_celsius", 30), + response_on_block=guardrails.get("response_on_block", "I can't help with that request."), + ) + + +def _latest_user_text(llm_request: LlmRequest) -> str: + for content in reversed(llm_request.contents or []): + if content.role != "user": + continue + for part in content.parts or []: + if part.text: + return part.text.lower() + return "" + + +def _blocked_response(config: GuardrailConfig) -> LlmResponse: + return LlmResponse( + content=types.Content( + role="model", + parts=[types.Part(text=config.response_on_block)], + ) + ) + + +def _entity_is_blocked(entity_id: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(entity_id, pattern) for pattern in patterns) + + +def create_guardrail_callbacks( + config: GuardrailConfig | None = None, +) -> dict[str, Any]: + """Return ADK callback callables configured from guardrail policy.""" + + policy = config or load_guardrail_config() + + def before_model_callback( + callback_context: CallbackContext, + llm_request: LlmRequest, + ) -> LlmResponse | None: + del callback_context + user_text = _latest_user_text(llm_request) + if not user_text: + return None + + for topic in policy.blocked_topics: + if topic in user_text: + return _blocked_response(policy) + return None + + def before_tool_callback( + tool: Any, + args: dict[str, Any], + tool_context: ToolContext, + ) -> dict[str, Any] | None: + del tool_context + tool_name = getattr(tool, "name", getattr(tool, "__name__", "")) + + if tool_name in policy.blocked_tools: + return {"error": policy.response_on_block} + + if tool_name == "call_service": + domain = str(args.get("domain", "")) + entity_id = str(args.get("entity_id", "")) + + if policy.allowed_domains and domain not in policy.allowed_domains: + return { + "error": ( + f"Domain '{domain}' is not allowed. " + f"Allowed domains: {', '.join(policy.allowed_domains)}" + ) + } + + if entity_id and _entity_is_blocked(entity_id, policy.blocked_entities): + return {"error": f"Entity '{entity_id}' is protected by guardrails."} + + if domain == "climate" and "temperature" in args: + temperature = float(args["temperature"]) + if not policy.temperature_min_celsius <= temperature <= policy.temperature_max_celsius: + return { + "error": ( + "Temperature must be between " + f"{policy.temperature_min_celsius}°C and " + f"{policy.temperature_max_celsius}°C." + ) + } + + if tool_name in policy.confirm_tools: + confirmation = str(args.get("confirm", "")).lower() + if confirmation not in {"yes", "true", "confirmed"}: + return { + "status": "confirmation_required", + "message": ( + "Please confirm this action by repeating your request and " + "including the word 'confirm'." + ), + } + + return None + + return { + "before_model_callback": before_model_callback, + "before_tool_callback": before_tool_callback, + } diff --git a/src/agents/home_agent/skills/__init__.py b/src/agents/home_agent/skills/__init__.py new file mode 100644 index 0000000..e01c9c7 --- /dev/null +++ b/src/agents/home_agent/skills/__init__.py @@ -0,0 +1,5 @@ +"""Skill modules and loader for the home voice agent.""" + +from .loader import SkillConfig, load_skill_tools, load_skills_config + +__all__ = ["SkillConfig", "load_skill_tools", "load_skills_config"] diff --git a/src/agents/home_agent/skills/home_assistant.py b/src/agents/home_agent/skills/home_assistant.py new file mode 100644 index 0000000..aa0c8c7 --- /dev/null +++ b/src/agents/home_agent/skills/home_assistant.py @@ -0,0 +1,27 @@ +"""Home Assistant skill module.""" + +from ..tools.homeassistant import ( + call_service, + get_entity_state, + list_entities, + set_temperature, + turn_off, + turn_on, +) + +INSTRUCTION = """ +You can control the home through Home Assistant. +- Use list_entities to discover devices before acting. +- Prefer turn_on/turn_off for lights and switches. +- Use set_temperature for climate devices. +- Always confirm the room and device before changing state. +""" + +TOOLS = [ + list_entities, + get_entity_state, + turn_on, + turn_off, + set_temperature, + call_service, +] diff --git a/src/agents/home_agent/skills/loader.py b/src/agents/home_agent/skills/loader.py new file mode 100644 index 0000000..4273b85 --- /dev/null +++ b/src/agents/home_agent/skills/loader.py @@ -0,0 +1,68 @@ +"""Load enabled skills and expose their tools to the agent.""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +import yaml + +DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "skills.yaml" + + +@dataclass +class SkillConfig: + name: str + enabled: bool + description: str + module: str + + +def load_skills_config(path: Path | None = None) -> list[SkillConfig]: + config_path = path or DEFAULT_CONFIG_PATH + with config_path.open(encoding="utf-8") as config_file: + raw = yaml.safe_load(config_file) or {} + + skills: list[SkillConfig] = [] + for entry in raw.get("skills", []): + skills.append( + SkillConfig( + name=entry["name"], + enabled=entry.get("enabled", True), + description=entry.get("description", ""), + module=entry["module"], + ) + ) + return skills + + +def _import_skill_module(module_name: str) -> Any: + return importlib.import_module(f".{module_name}", package=__package__) + + +def load_skill_tools( + skills: list[SkillConfig] | None = None, +) -> tuple[list[Callable[..., Any]], list[str]]: + """Return tool callables and instruction snippets for enabled skills.""" + + configured_skills = skills or load_skills_config() + tools: list[Callable[..., Any]] = [] + instructions: list[str] = [] + + for skill in configured_skills: + if not skill.enabled: + continue + + module = _import_skill_module(skill.module) + skill_tools = getattr(module, "TOOLS", []) + tools.extend(skill_tools) + + skill_instruction = getattr(module, "INSTRUCTION", "") + if skill_instruction: + instructions.append(skill_instruction.strip()) + elif skill.description: + instructions.append(skill.description.strip()) + + return tools, instructions diff --git a/src/agents/home_agent/skills/routines.py b/src/agents/home_agent/skills/routines.py new file mode 100644 index 0000000..3d3ffab --- /dev/null +++ b/src/agents/home_agent/skills/routines.py @@ -0,0 +1,81 @@ +"""Predefined home routines skill module.""" + +from __future__ import annotations + +from typing import Any + +from ..tools.homeassistant import call_service + +ROUTINES: dict[str, dict[str, Any]] = { + "goodnight": { + "description": "Turn off living room lights and lower bedroom temperature", + "actions": [ + {"domain": "light", "service": "turn_off", "entity_id": "light.living_room"}, + {"domain": "climate", "service": "set_temperature", "entity_id": "climate.bedroom", "data": {"temperature": 20}}, + ], + }, + "away": { + "description": "Turn off all lights", + "actions": [ + {"domain": "light", "service": "turn_off", "entity_id": "light.all_lights"}, + ], + }, + "movie_mode": { + "description": "Dim living room lights for movie watching", + "actions": [ + {"domain": "light", "service": "turn_on", "entity_id": "light.living_room", "data": {"brightness": 40}}, + ], + }, +} + + +def list_routines() -> list[dict[str, str]]: + """List available home routines and what they do. + + Returns: + Routine names and descriptions. + """ + return [ + {"name": name, "description": routine["description"]} + for name, routine in ROUTINES.items() + ] + + +def run_routine(routine_name: str, confirm: str = "") -> dict[str, Any]: + """Run a predefined home routine. + + Args: + routine_name: One of goodnight, away, movie_mode. + confirm: Set to 'yes' or 'confirmed' before running the routine. + + Returns: + Results of each action in the routine. + """ + routine = ROUTINES.get(routine_name.lower()) + if not routine: + return { + "error": f"Unknown routine '{routine_name}'.", + "available": list(ROUTINES.keys()), + } + + results: list[dict[str, Any]] = [] + for action in routine["actions"]: + result = call_service( + domain=action["domain"], + service=action["service"], + entity_id=action.get("entity_id", ""), + data=action.get("data"), + confirm=confirm, + ) + results.append(result) + + return {"routine": routine_name, "status": "ok", "actions": results} + + +INSTRUCTION = """ +You can run predefined routines: goodnight, away, and movie_mode. +Use list_routines to show options. Routines that change multiple devices +require explicit user confirmation. +""" + +TOOLS = [list_routines, run_routine] diff --git a/src/agents/home_agent/tools/__init__.py b/src/agents/home_agent/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agents/home_agent/tools/homeassistant.py b/src/agents/home_agent/tools/homeassistant.py new file mode 100644 index 0000000..ab11ccb --- /dev/null +++ b/src/agents/home_agent/tools/homeassistant.py @@ -0,0 +1,170 @@ +"""Home Assistant integration tools.""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx + +HA_URL = os.getenv("HOME_ASSISTANT_URL", "http://homeassistant.local:8123") +HA_TOKEN = os.getenv("HOME_ASSISTANT_TOKEN", "") + + +def _headers() -> dict[str, str]: + if not HA_TOKEN: + raise RuntimeError( + "HOME_ASSISTANT_TOKEN is not set. Add it to your .env file." + ) + return { + "Authorization": f"Bearer {HA_TOKEN}", + "Content-Type": "application/json", + } + + +def _request(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: + with httpx.Client(base_url=HA_URL, timeout=10.0) as client: + response = client.request(method, path, headers=_headers(), json=payload) + response.raise_for_status() + if response.content: + return response.json() + return {"status": "ok"} + + +def list_entities(domain: str = "", area: str = "") -> list[dict[str, Any]]: + """List Home Assistant entities, optionally filtered by domain or area name. + + Args: + domain: Optional entity domain prefix (e.g. light, switch, climate). + area: Optional area/room name to filter by. + + Returns: + A list of entities with entity_id, friendly_name, state, and domain. + """ + states = _request("GET", "/api/states") + entities: list[dict[str, Any]] = [] + + for state in states: + entity_id = state.get("entity_id", "") + entity_domain = entity_id.split(".", maxsplit=1)[0] + attributes = state.get("attributes", {}) + friendly_name = attributes.get("friendly_name", entity_id) + entity_area = attributes.get("area", "") + + if domain and entity_domain != domain: + continue + if area and area.lower() not in friendly_name.lower() and area.lower() not in entity_area.lower(): + continue + + entities.append( + { + "entity_id": entity_id, + "friendly_name": friendly_name, + "state": state.get("state"), + "domain": entity_domain, + } + ) + + return entities + + +def get_entity_state(entity_id: str) -> dict[str, Any]: + """Get the current state of a Home Assistant entity. + + Args: + entity_id: The entity ID (e.g. light.living_room). + + Returns: + Entity state details including state and attributes. + """ + state = _request("GET", f"/api/states/{entity_id}") + return { + "entity_id": state.get("entity_id", entity_id), + "state": state.get("state"), + "attributes": state.get("attributes", {}), + } + + +def call_service( + domain: str, + service: str, + entity_id: str = "", + data: dict[str, Any] | None = None, + confirm: str = "", +) -> dict[str, Any]: + """Call a Home Assistant service to control a device. + + Args: + domain: Service domain (light, switch, climate, scene, script). + service: Service name (turn_on, turn_off, toggle, set_temperature). + entity_id: Target entity ID. + data: Optional extra service data (brightness, temperature, etc.). + confirm: Set to 'yes' or 'confirmed' for actions that require confirmation. + + Returns: + Result of the service call. + """ + service_data: dict[str, Any] = dict(data or {}) + if entity_id: + service_data["entity_id"] = entity_id + + result = _request( + "POST", + f"/api/services/{domain}/{service}", + payload=service_data, + ) + return { + "status": "ok", + "domain": domain, + "service": service, + "entity_id": entity_id, + "result": result, + } + + +def turn_on(entity_id: str, brightness: int | None = None) -> dict[str, Any]: + """Turn on a light or switch. + + Args: + entity_id: Target entity ID. + brightness: Optional brightness level from 0-255 for lights. + + Returns: + Service call result. + """ + domain = entity_id.split(".", maxsplit=1)[0] + data: dict[str, Any] = {} + if brightness is not None: + data["brightness"] = brightness + return call_service(domain=domain, service="turn_on", entity_id=entity_id, data=data) + + +def turn_off(entity_id: str) -> dict[str, Any]: + """Turn off a light or switch. + + Args: + entity_id: Target entity ID. + + Returns: + Service call result. + """ + domain = entity_id.split(".", maxsplit=1)[0] + return call_service(domain=domain, service="turn_off", entity_id=entity_id) + + +def set_temperature(entity_id: str, temperature: float) -> dict[str, Any]: + """Set a climate entity target temperature in Celsius. + + Args: + entity_id: Climate entity ID. + temperature: Target temperature in Celsius. + + Returns: + Service call result. + """ + return call_service( + domain="climate", + service="set_temperature", + entity_id=entity_id, + data={"temperature": temperature}, + ) diff --git a/src/server/__init__.py b/src/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server/static/css/app.css b/src/server/static/css/app.css new file mode 100644 index 0000000..37b8e2c --- /dev/null +++ b/src/server/static/css/app.css @@ -0,0 +1,88 @@ +:root { + color-scheme: dark; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: #0f172a; + color: #e2e8f0; +} + +.app { + min-height: 100dvh; + display: flex; + flex-direction: column; + padding: 1rem; + padding-bottom: calc(1rem + env(safe-area-inset-bottom)); +} + +.header h1 { + margin: 0; + font-size: 1.5rem; +} + +.subtitle { + margin: 0.25rem 0 1rem; + color: #94a3b8; +} + +.messages { + flex: 1; + overflow-y: auto; + border: 1px solid #334155; + border-radius: 1rem; + padding: 1rem; + background: #111827; + margin-bottom: 1rem; +} + +.messages p { + margin: 0.5rem 0; + line-height: 1.4; +} + +.composer { + display: flex; + gap: 0.5rem; + margin-bottom: 0.75rem; +} + +.composer input { + flex: 1; + border-radius: 999px; + border: 1px solid #334155; + background: #1e293b; + color: inherit; + padding: 0.75rem 1rem; + font-size: 1rem; +} + +.composer button, +.voice-button { + border: none; + border-radius: 999px; + background: #38bdf8; + color: #0f172a; + font-weight: 600; + padding: 0.75rem 1rem; +} + +.voice-button { + width: 100%; + font-size: 1rem; +} + +.voice-button:disabled { + opacity: 0.6; +} + +.hint { + color: #94a3b8; + font-size: 0.85rem; + text-align: center; +} diff --git a/src/server/static/index.html b/src/server/static/index.html new file mode 100644 index 0000000..f54a3a4 --- /dev/null +++ b/src/server/static/index.html @@ -0,0 +1,47 @@ + + + + + + + + + + Home Voice Agent + + + +
+
+

Home Voice Agent

+

Talk to your home

+
+ +
+ +
+ + +
+ + +

+ Tap Start Voice, allow microphone access, then speak naturally. + Add this page to your iPhone home screen for quick access. +

+
+ + + + diff --git a/src/server/static/js/app.js b/src/server/static/js/app.js new file mode 100644 index 0000000..968f016 --- /dev/null +++ b/src/server/static/js/app.js @@ -0,0 +1,143 @@ +/** + * Home Voice Agent client for iPhone Safari and desktop browsers. + */ + +const sessionId = Math.random().toString().substring(2, 12); +const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; +const wsUrl = `${wsProtocol}//${window.location.host}/ws/${sessionId}`; + +let websocket = null; +let isAudio = false; +let currentMessageId = null; + +let audioPlayerNode; +let audioPlayerContext; +let audioRecorderNode; +let audioRecorderContext; +let micStream; + +const messageForm = document.getElementById("messageForm"); +const messageInput = document.getElementById("message"); +const messagesDiv = document.getElementById("messages"); +const startAudioButton = document.getElementById("startAudioButton"); + +import { startAudioPlayerWorklet } from "./audio-player.js"; +import { startAudioRecorderWorklet } from "./audio-recorder.js"; + +function connectWebsocket() { + websocket = new WebSocket(`${wsUrl}?is_audio=${isAudio}`); + + websocket.onopen = function () { + document.getElementById("sendButton").disabled = false; + messagesDiv.textContent = "Connected. Ask me to control your home."; + addSubmitHandler(); + }; + + websocket.onmessage = function (event) { + const messageFromServer = JSON.parse(event.data); + + if (messageFromServer.turn_complete === true) { + currentMessageId = null; + return; + } + + if (messageFromServer.interrupted === true) { + if (audioPlayerNode) { + audioPlayerNode.port.postMessage({ command: "endOfAudio" }); + } + return; + } + + if (messageFromServer.mime_type === "audio/pcm" && audioPlayerNode) { + audioPlayerNode.port.postMessage(base64ToArray(messageFromServer.data)); + } + + if (messageFromServer.mime_type === "text/plain") { + if (currentMessageId == null) { + currentMessageId = Math.random().toString(36).substring(7); + const message = document.createElement("p"); + message.id = currentMessageId; + messagesDiv.appendChild(message); + } + const message = document.getElementById(currentMessageId); + message.textContent += messageFromServer.data; + messagesDiv.scrollTop = messagesDiv.scrollHeight; + } + }; + + websocket.onclose = function () { + document.getElementById("sendButton").disabled = true; + messagesDiv.textContent = "Connection closed. Reconnecting..."; + setTimeout(connectWebsocket, 3000); + }; +} + +connectWebsocket(); + +function addSubmitHandler() { + messageForm.onsubmit = function (event) { + event.preventDefault(); + const message = messageInput.value.trim(); + if (!message) { + return false; + } + + const paragraph = document.createElement("p"); + paragraph.textContent = `> ${message}`; + messagesDiv.appendChild(paragraph); + messageInput.value = ""; + sendMessage({ mime_type: "text/plain", data: message }); + return false; + }; +} + +function sendMessage(message) { + if (websocket && websocket.readyState === WebSocket.OPEN) { + websocket.send(JSON.stringify(message)); + } +} + +function base64ToArray(base64) { + const binaryString = window.atob(base64); + const bytes = new Uint8Array(binaryString.length); + for (let index = 0; index < binaryString.length; index += 1) { + bytes[index] = binaryString.charCodeAt(index); + } + return bytes.buffer; +} + +function arrayBufferToBase64(buffer) { + const bytes = new Uint8Array(buffer); + let binary = ""; + for (let index = 0; index < bytes.byteLength; index += 1) { + binary += String.fromCharCode(bytes[index]); + } + return window.btoa(binary); +} + +function audioRecorderHandler(pcmData) { + sendMessage({ + mime_type: "audio/pcm", + data: arrayBufferToBase64(pcmData), + }); +} + +function startAudio() { + startAudioPlayerWorklet().then(([node, context]) => { + audioPlayerNode = node; + audioPlayerContext = context; + }); + startAudioRecorderWorklet(audioRecorderHandler).then(([node, context, stream]) => { + audioRecorderNode = node; + audioRecorderContext = context; + micStream = stream; + }); +} + +startAudioButton.addEventListener("click", () => { + startAudioButton.disabled = true; + startAudioButton.textContent = "Voice Active"; + startAudio(); + isAudio = true; + connectWebsocket(); +}); diff --git a/src/server/static/js/audio-player.js b/src/server/static/js/audio-player.js new file mode 100644 index 0000000..2d7a63e --- /dev/null +++ b/src/server/static/js/audio-player.js @@ -0,0 +1,8 @@ +export async function startAudioPlayerWorklet() { + const audioContext = new AudioContext({ sampleRate: 24000 }); + const workletURL = new URL("./pcm-player-processor.js", import.meta.url); + await audioContext.audioWorklet.addModule(workletURL); + const audioPlayerNode = new AudioWorkletNode(audioContext, "pcm-player-processor"); + audioPlayerNode.connect(audioContext.destination); + return [audioPlayerNode, audioContext]; +} diff --git a/src/server/static/js/audio-recorder.js b/src/server/static/js/audio-recorder.js new file mode 100644 index 0000000..bb331e3 --- /dev/null +++ b/src/server/static/js/audio-recorder.js @@ -0,0 +1,29 @@ +export async function startAudioRecorderWorklet(audioRecorderHandler) { + const audioRecorderContext = new AudioContext({ sampleRate: 16000 }); + const workletURL = new URL("./pcm-recorder-processor.js", import.meta.url); + await audioRecorderContext.audioWorklet.addModule(workletURL); + + const micStream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1 }, + }); + const source = audioRecorderContext.createMediaStreamSource(micStream); + const audioRecorderNode = new AudioWorkletNode( + audioRecorderContext, + "pcm-recorder-processor", + ); + + source.connect(audioRecorderNode); + audioRecorderNode.port.onmessage = (event) => { + const pcmData = convertFloat32ToPCM(event.data); + audioRecorderHandler(pcmData); + }; + return [audioRecorderNode, audioRecorderContext, micStream]; +} + +function convertFloat32ToPCM(inputData) { + const pcm16 = new Int16Array(inputData.length); + for (let index = 0; index < inputData.length; index += 1) { + pcm16[index] = inputData[index] * 0x7fff; + } + return pcm16.buffer; +} diff --git a/src/server/static/js/pcm-player-processor.js b/src/server/static/js/pcm-player-processor.js new file mode 100644 index 0000000..74f836d --- /dev/null +++ b/src/server/static/js/pcm-player-processor.js @@ -0,0 +1,46 @@ +class PCMPlayerProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.bufferSize = 24000 * 180; + this.buffer = new Float32Array(this.bufferSize); + this.writeIndex = 0; + this.readIndex = 0; + + this.port.onmessage = (event) => { + if (event.data.command === "endOfAudio") { + this.readIndex = this.writeIndex; + return; + } + const int16Samples = new Int16Array(event.data); + this._enqueue(int16Samples); + }; + } + + _enqueue(int16Samples) { + for (let index = 0; index < int16Samples.length; index += 1) { + const floatVal = int16Samples[index] / 32768; + this.buffer[this.writeIndex] = floatVal; + this.writeIndex = (this.writeIndex + 1) % this.bufferSize; + if (this.writeIndex === this.readIndex) { + this.readIndex = (this.readIndex + 1) % this.bufferSize; + } + } + } + + process(inputs, outputs) { + const output = outputs[0]; + const framesPerBlock = output[0].length; + for (let frame = 0; frame < framesPerBlock; frame += 1) { + output[0][frame] = this.buffer[this.readIndex]; + if (output.length > 1) { + output[1][frame] = this.buffer[this.readIndex]; + } + if (this.readIndex !== this.writeIndex) { + this.readIndex = (this.readIndex + 1) % this.bufferSize; + } + } + return true; + } +} + +registerProcessor("pcm-player-processor", PCMPlayerProcessor); diff --git a/src/server/static/js/pcm-recorder-processor.js b/src/server/static/js/pcm-recorder-processor.js new file mode 100644 index 0000000..b671f15 --- /dev/null +++ b/src/server/static/js/pcm-recorder-processor.js @@ -0,0 +1,12 @@ +class PCMProcessor extends AudioWorkletProcessor { + process(inputs) { + if (inputs.length > 0 && inputs[0].length > 0) { + const inputChannel = inputs[0][0]; + const inputCopy = new Float32Array(inputChannel); + this.port.postMessage(inputCopy); + } + return true; + } +} + +registerProcessor("pcm-recorder-processor", PCMProcessor); diff --git a/src/server/static/manifest.json b/src/server/static/manifest.json new file mode 100644 index 0000000..16ba766 --- /dev/null +++ b/src/server/static/manifest.json @@ -0,0 +1,9 @@ +{ + "name": "Home Voice Agent", + "short_name": "Home Agent", + "description": "Voice-controlled home assistant", + "start_url": "/", + "display": "standalone", + "background_color": "#0f172a", + "theme_color": "#0f172a" +} diff --git a/src/server/voice_server.py b/src/server/voice_server.py new file mode 100644 index 0000000..f95c9ec --- /dev/null +++ b/src/server/voice_server.py @@ -0,0 +1,137 @@ +"""FastAPI voice server for iPhone and browser clients.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv +from fastapi import FastAPI, WebSocket +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from google.adk.agents import LiveRequestQueue +from google.adk.agents.run_config import RunConfig +from google.adk.runners import InMemoryRunner +from google.genai.types import Blob, Content, Part + +AGENTS_DIR = Path(__file__).resolve().parents[1] / "agents" +if str(AGENTS_DIR) not in sys.path: + sys.path.insert(0, str(AGENTS_DIR)) + +from home_agent.agent import root_agent # noqa: E402 + +load_dotenv() + +APP_NAME = "home_voice_agent" +STATIC_DIR = Path(__file__).resolve().parent / "static" + +app = FastAPI(title="Home Voice Agent") +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") + +runner = InMemoryRunner(app_name=APP_NAME, agent=root_agent) + + +async def start_agent_session(user_id: str, is_audio: bool = False) -> tuple[object, LiveRequestQueue]: + session = await runner.session_service.create_session( + app_name=APP_NAME, + user_id=user_id, + ) + modality = "AUDIO" if is_audio else "TEXT" + run_config = RunConfig(response_modalities=[modality]) + live_request_queue = LiveRequestQueue() + live_events = runner.run_live( + session=session, + live_request_queue=live_request_queue, + run_config=run_config, + ) + return live_events, live_request_queue + + +async def agent_to_client_messaging(websocket: WebSocket, live_events: object) -> None: + async for event in live_events: + if event.turn_complete or event.interrupted: + message = { + "turn_complete": event.turn_complete, + "interrupted": event.interrupted, + } + await websocket.send_text(json.dumps(message)) + continue + + part = event.content and event.content.parts and event.content.parts[0] + if not part: + continue + + is_audio = part.inline_data and part.inline_data.mime_type.startswith("audio/pcm") + if is_audio: + audio_data = part.inline_data and part.inline_data.data + if audio_data: + message = { + "mime_type": "audio/pcm", + "data": base64.b64encode(audio_data).decode("ascii"), + } + await websocket.send_text(json.dumps(message)) + continue + + if part.text and event.partial: + message = {"mime_type": "text/plain", "data": part.text} + await websocket.send_text(json.dumps(message)) + + +async def client_to_agent_messaging( + websocket: WebSocket, + live_request_queue: LiveRequestQueue, +) -> None: + while True: + message_json = await websocket.receive_text() + message = json.loads(message_json) + mime_type = message["mime_type"] + data = message["data"] + + if mime_type == "text/plain": + content = Content(role="user", parts=[Part.from_text(text=data)]) + live_request_queue.send_content(content=content) + elif mime_type == "audio/pcm": + decoded_data = base64.b64decode(data) + live_request_queue.send_realtime(Blob(data=decoded_data, mime_type=mime_type)) + else: + raise ValueError(f"Mime type not supported: {mime_type}") + + +@app.get("/") +async def root() -> FileResponse: + return FileResponse(STATIC_DIR / "index.html") + + +@app.get("/manifest.json") +async def manifest() -> FileResponse: + return FileResponse(STATIC_DIR / "manifest.json") + + +@app.websocket("/ws/{user_id}") +async def websocket_endpoint(websocket: WebSocket, user_id: str, is_audio: str = "false") -> None: + await websocket.accept() + live_events, live_request_queue = await start_agent_session( + user_id, + is_audio == "true", + ) + + agent_task = asyncio.create_task(agent_to_client_messaging(websocket, live_events)) + client_task = asyncio.create_task(client_to_agent_messaging(websocket, live_request_queue)) + await asyncio.wait([agent_task, client_task], return_when=asyncio.FIRST_EXCEPTION) + live_request_queue.close() + + +def main() -> None: + import uvicorn + + host = os.getenv("VOICE_SERVER_HOST", "0.0.0.0") + port = int(os.getenv("VOICE_SERVER_PORT", "8000")) + uvicorn.run("server.voice_server:app", host=host, port=port, reload=False) + + +if __name__ == "__main__": + main() diff --git a/tests/test_home_agent.py b/tests/test_home_agent.py new file mode 100644 index 0000000..93514cb --- /dev/null +++ b/tests/test_home_agent.py @@ -0,0 +1,96 @@ +"""Tests for home agent guardrails and skills.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from google.adk.models import LlmRequest +from google.genai import types + +AGENTS_DIR = Path(__file__).resolve().parents[1] / "src" / "agents" +if str(AGENTS_DIR) not in sys.path: + sys.path.insert(0, str(AGENTS_DIR)) + +from home_agent.guardrails.loader import ( # noqa: E402 + GuardrailConfig, + create_guardrail_callbacks, +) +from home_agent.skills.loader import load_skill_tools, load_skills_config # noqa: E402 + + +def test_load_skills_config_includes_home_assistant() -> None: + skills = load_skills_config() + names = {skill.name for skill in skills} + assert "home_assistant" in names + assert "routines" in names + + +def test_load_skill_tools_returns_callables() -> None: + tools, instructions = load_skill_tools() + assert len(tools) >= 5 + assert any("Home Assistant" in instruction for instruction in instructions) + + +def test_guardrail_blocks_dangerous_topic() -> None: + config = GuardrailConfig(blocked_topics=["unlock all doors"]) + callbacks = create_guardrail_callbacks(config) + + request = LlmRequest( + contents=[ + types.Content( + role="user", + parts=[types.Part(text="Please unlock all doors now")], + ) + ] + ) + + class DummyContext: + agent_name = "test" + + response = callbacks["before_model_callback"](DummyContext(), request) + assert response is not None + assert response.content is not None + assert response.content.parts[0].text + + +def test_guardrail_blocks_disallowed_domain() -> None: + config = GuardrailConfig( + allowed_domains=["light", "switch"], + response_on_block="blocked", + ) + callbacks = create_guardrail_callbacks(config) + + class DummyTool: + name = "call_service" + + class DummyContext: + state: dict[str, str] = {} + + result = callbacks["before_tool_callback"]( + DummyTool(), + {"domain": "lock", "entity_id": "lock.garage"}, + DummyContext(), + ) + assert result is not None + assert "error" in result + + +def test_guardrail_requires_confirmation_for_routines() -> None: + config = GuardrailConfig(confirm_tools=["run_routine"]) + callbacks = create_guardrail_callbacks(config) + + class DummyTool: + name = "run_routine" + + class DummyContext: + state: dict[str, str] = {} + + result = callbacks["before_tool_callback"]( + DummyTool(), + {"routine_name": "away"}, + DummyContext(), + ) + assert result is not None + assert result["status"] == "confirmation_required"