From 5201489fc0bbf60a76b2b62ee89e2b1c2eb46f54 Mon Sep 17 00:00:00 2001 From: Hushlor <25059159+Hushlor@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:18:01 +0100 Subject: [PATCH] feat: add Windows native dialog automation --- CHANGELOG.md | 3 + README.md | 4 + docs/api/index.md | 5 +- docs/api/ui-automation.md | 75 +- docs/guides/getting-started.md | 2 +- docs/index.md | 4 +- packages/mcp-server/CHANGELOG.md | 3 + packages/mcp-server/README.md | 4 +- .../mcp-server/src/driver/native-dialog.ts | 122 ++ packages/mcp-server/src/driver/protocol.ts | 4 +- packages/mcp-server/src/tools-registry.ts | 51 + .../tests/e2e/native-dialog.test.ts | 321 ++++++ .../tests/unit/native-dialog.test.ts | 150 +++ packages/tauri-plugin-mcp-bridge/CHANGELOG.md | 3 + packages/tauri-plugin-mcp-bridge/Cargo.toml | 2 +- packages/tauri-plugin-mcp-bridge/README.md | 22 +- packages/tauri-plugin-mcp-bridge/src/lib.rs | 5 + .../src/native_dialog/mod.rs | 119 ++ .../src/native_dialog/unsupported.rs | 65 ++ .../src/native_dialog/windows.rs | 1023 +++++++++++++++++ .../tauri-plugin-mcp-bridge/src/websocket.rs | 171 ++- packages/test-app/index.html | 2 + packages/test-app/src/main.ts | 16 + 23 files changed, 2156 insertions(+), 20 deletions(-) create mode 100644 packages/mcp-server/src/driver/native-dialog.ts create mode 100644 packages/mcp-server/tests/e2e/native-dialog.test.ts create mode 100644 packages/mcp-server/tests/unit/native-dialog.test.ts create mode 100644 packages/tauri-plugin-mcp-bridge/src/native_dialog/mod.rs create mode 100644 packages/tauri-plugin-mcp-bridge/src/native_dialog/unsupported.rs create mode 100644 packages/tauri-plugin-mcp-bridge/src/native_dialog/windows.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b646b6..858758a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Windows-only `native_dialog_snapshot` and `native_dialog_interact` tools use Microsoft UI Automation to inspect and control native dialogs owned by the connected Tauri application, including multi-file Open, folder selection, navigation controls, nested ownership chains, and Save overwrite confirmations. + ## [0.12.0] - 2026-07-05 ### Added diff --git a/README.md b/README.md index 8339d85..6058b5c 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ Just type the command in your AI assistant to start a guided workflow. | `read_logs` | Read console, Android, iOS, or system logs | | `webview_interact` | Click, scroll, swipe, focus, long-press | | `webview_screenshot` | Capture webview screenshots | +| `native_dialog_snapshot` | Inspect app-owned native Windows dialog chains and navigation controls | +| `native_dialog_interact` | Invoke, set single/multiple paths, or select native Windows dialog controls | | `webview_keyboard` | Type text or send key events | | `webview_wait_for` | Wait for elements, text, or events | | `webview_get_styles` | Get computed CSS styles | @@ -135,6 +137,8 @@ Just type the command in your AI assistant to start a guided workflow. > **Multi-Window Support**: All webview tools accept an optional `windowId` parameter to target specific windows. Use `manage_window` with `action: "list"` to discover available windows. +> **Native Windows dialogs**: `native_dialog_*` tools require an interactive desktop and expose only same-process dialogs whose owner chain leads to the targeted Tauri window. They support single/multi-file Open, Save, folder selection, nested confirmations, and advertised navigation controls, but not toast notifications or other applications. +
diff --git a/docs/api/index.md b/docs/api/index.md index 1066d96..2ae833b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,6 +1,6 @@ --- title: API Reference - What Your AI Can Do -description: Explore 20 powerful MCP tools for Tauri development including setup, mobile development, UI automation, and IPC debugging. +description: Explore 22 powerful MCP tools for Tauri development including setup, mobile development, UI automation, and IPC debugging. head: - - meta - name: keywords @@ -13,7 +13,7 @@ import { Wrench, Smartphone, Target, Zap, Rocket, Bug, FlaskConical, Activity, M # What Your AI Can Do -Once configured, your AI assistant has **20 powerful tools** plus **slash commands** to help you build, test, and debug your Tauri application. Just ask in natural language! +Once configured, your AI assistant has **22 powerful tools** plus **slash commands** to help you build, test, and debug your Tauri application. Just ask in natural language! ## Setup & Configuration @@ -61,6 +61,7 @@ Your AI can interact with your app's interface just like a user would. - Verify UI elements exist and work correctly - Check visual appearance with screenshots - Visually select elements with the element picker +- Inspect and interact with app-owned native dialogs on Windows **Debugging** - See console logs and errors diff --git a/docs/api/ui-automation.md b/docs/api/ui-automation.md index 86013b1..49df4c8 100644 --- a/docs/api/ui-automation.md +++ b/docs/api/ui-automation.md @@ -9,7 +9,7 @@ head: # UI Automation -Control and automate your Tauri application's UI. These tools provide comprehensive automation capabilities for testing and interaction, working seamlessly across all platforms (Linux, Windows, macOS). +Control and automate your Tauri application's UI. Webview tools work across Linux, Windows, and macOS. Native dialog automation is Windows-only. ## Multi-Window Support @@ -222,6 +222,79 @@ For real iOS/Android devices on the network: **Note**: No external driver process required. +## Native Windows Dialogs + +`native_dialog_snapshot` and `native_dialog_interact` automate native message, confirmation, single- or multi-file Open, folder selection, and Save dialogs. These windows are outside the webview DOM, so `webview_interact`, `webview_keyboard`, and `webview_screenshot` cannot inspect or control them. + +The tools require an active `driver_session` and an interactive Windows desktop. Discovery is restricted to visible dialogs in the connected Tauri process whose bounded owner chain leads back to the targeted Tauri window. This includes nested prompts such as a Save overwrite confirmation without exposing desktop-wide automation or dialogs from other applications. Windows toast notifications are not dialogs and are not supported. + +### native_dialog_snapshot + +Returns a bounded semantic snapshot containing control types, automation IDs, names, semantic roles, supported UI Automation patterns, owner depth, parent-dialog references, and opaque `elementRef` values. File-dialog snapshots include actionable navigation controls, the current-location control, navigation-tree items, and selectable file-system entries. Use `windowId` and `appIdentifier` with the same targeting behavior as webview tools. `timeoutMs` accepts 100–10000 milliseconds. + +```javascript +{ + "tool": "native_dialog_snapshot", + "windowId": "main", + "minOwnerDepth": 1, + "timeoutMs": 2000 +} +``` + +`minOwnerDepth` defaults to `1`, where the Tauri window directly owns the dialog. Set it to `2` to wait for a prompt owned by another app-owned dialog, such as an overwrite confirmation. Nested dialogs are returned before their parents. + +Controls advertise one or more supported actions: + +- `invoke` — `InvokePattern`, typically buttons such as accept, cancel, yes, or no +- `setValue` — `ValuePattern`, used for a complete absolute filename/path +- `setPaths` — `ValuePattern`, used for up to 100 complete absolute existing file paths in a multi-select Open dialog +- `select` — `SelectionItemPattern`, where the dialog exposes a selectable control + +Semantic roles are derived from automation IDs and control patterns rather than localized button text. Names and dialog text are returned to the caller for inspection but are not written to bridge logs. + +### native_dialog_interact + +Apply one advertised action to an `elementRef` from the latest snapshot: + +```javascript +{ + "tool": "native_dialog_interact", + "action": "invoke", + "elementRef": "native_..." +} +``` + +For Open or Save dialogs, set the filename field to a complete absolute path, then invoke the accept button: + +```javascript +{ + "tool": "native_dialog_interact", + "action": "setValue", + "elementRef": "native_...", + "value": "C:\\tmp\\fixture.txt" +} +``` + +For a multi-file Open dialog, use the filename field's advertised `setPaths` action and then invoke accept: + +```javascript +{ + "tool": "native_dialog_interact", + "action": "setPaths", + "elementRef": "native_...", + "paths": [ + "C:\\tmp\\first.txt", + "C:\\tmp\\second.txt" + ] +} +``` + +For a folder picker, use `setValue` with a complete absolute directory path and invoke accept. To respond to a Save overwrite prompt, invoke Save, take a new snapshot with `minOwnerDepth: 2`, and invoke the confirmation's advertised affirmative action. Back, forward, up, address/current-location, navigation-tree, and file-system entry controls are included when Windows exposes their corresponding UI Automation patterns. + +Element references are session-bound, expire after 30 seconds, and are otherwise ephemeral. A new snapshot replaces the previous reference set; invoking or selecting a control invalidates references for that dialog, while `setValue` and `setPaths` preserve them so the previously discovered accept control can be invoked. If the dialog changes or closes, the tool returns a stale-reference error and the caller must take another snapshot. + +Supported scope includes single- and multi-file Open, Save, folder selection, overwrite confirmation, message acknowledgement, ask/confirm responses, absolute-path entry, cancellation, nested app-owned dialogs, and navigation controls that expose the patterns above. Arbitrary custom controls without UI Automation patterns, permission prompts outside the app's ownership chain, notification/toast automation, image recognition, and coordinate-based native control remain out of scope. + ## webview_find_element Find UI elements using CSS, XPath, or text selectors. diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 01dfc97..2d16993 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -286,7 +286,7 @@ Some MCP clients don't support slash commands yet. See the [manual prompt instru ## Next Steps -- **[API Reference](/api/)** — Learn about all 20 available tools +- **[API Reference](/api/)** — Learn about all 22 available tools - **[IPC & Plugin Tools](/api/ipc-plugin)** — Debug your app's IPC layer - **[UI Automation](/api/ui-automation)** — Automate webview interactions diff --git a/docs/index.md b/docs/index.md index b0ba692..e2a29b5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -179,7 +179,7 @@ The server provides **slash commands** ([prompts](https://modelcontextprotocol.i [Learn more about prompts →](/api/prompts) -## 21 Powerful Tools +## 22 Powerful Tools The server exposes tools across 4 categories: @@ -190,7 +190,7 @@ The server exposes tools across 4 categories:
- UI Automation & WebView (14 tools) - Gestures, screenshots, DOM snapshots, JS execution, element picking, logs + UI Automation & WebView (15 tools) - Webview and native Windows dialog automation, screenshots, DOM snapshots, element picking, logs
diff --git a/packages/mcp-server/CHANGELOG.md b/packages/mcp-server/CHANGELOG.md index 3a8fefd..112bb12 100644 --- a/packages/mcp-server/CHANGELOG.md +++ b/packages/mcp-server/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Add `native_dialog_snapshot` and `native_dialog_interact` MCP tools for semantic Windows native-dialog automation, including multiple paths, folder pickers, navigation controls, and nested confirmations. + ## [0.12.0] - 2026-07-05 ### Added diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index cbf33e4..72387d2 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -98,7 +98,7 @@ await driver_session({ action: "stop", appIdentifier: 9223 }) await driver_session({ action: "stop" }) ``` -## Available Tools (20 total) +## Available Tools (22 total) ### Setup & Configuration @@ -115,6 +115,8 @@ await driver_session({ action: "stop" }) | `read_logs` | Read console, Android, iOS, or system logs | | `webview_interact` | Click, scroll, swipe, focus, long-press | | `webview_screenshot` | Capture webview screenshots (JPEG default) | +| `native_dialog_snapshot` | Inspect app-owned native Windows dialog chains and navigation controls | +| `native_dialog_interact` | Invoke, set paths, or select native Windows dialog controls | | `webview_keyboard` | Type text or send key events | | `webview_wait_for` | Wait for elements, text, or events | | `webview_get_styles` | Get computed CSS styles | diff --git a/packages/mcp-server/src/driver/native-dialog.ts b/packages/mcp-server/src/driver/native-dialog.ts new file mode 100644 index 0000000..23dc8b5 --- /dev/null +++ b/packages/mcp-server/src/driver/native-dialog.ts @@ -0,0 +1,122 @@ +import { randomUUID } from 'node:crypto'; +import { z } from 'zod'; + +import { ensureSessionAndConnect, type PluginClient } from './plugin-client.js'; + +const TargetSchema = { + windowId: z.string().optional().describe('Tauri window label whose owned native dialog chain should be targeted'), + appIdentifier: z.union([ z.string(), z.number() ]).optional().describe( + 'App identifier (port or bundle ID) when multiple Tauri apps are connected' + ), + timeoutMs: z.number().int().min(100).max(10000).default(2000).describe( + 'Bounded time to wait for the native operation, from 100 to 10000 milliseconds' + ), +} as const; + +/** Schema for discovering native dialogs owned by the targeted Tauri window. */ +export const NativeDialogSnapshotSchema = z.object({ + ...TargetSchema, + minOwnerDepth: z.number().int().min(1).max(8).default(1).describe( + 'Minimum owner-chain depth to return: 1 for direct dialogs, 2 for confirmations owned by another dialog' + ), +}); + +const InteractionTargetSchema = { + elementRef: z.string().min(1).describe('Opaque elementRef from the latest native_dialog_snapshot'), + ...TargetSchema, +} as const; + +/** + * Schema for invoking UI Automation patterns on a discovered native dialog element. + */ +export const NativeDialogInteractSchema = z.discriminatedUnion('action', [ + z.object({ + action: z.literal('invoke').describe('Invoke a button through UI Automation InvokePattern'), + ...InteractionTargetSchema, + }), + z.object({ + action: z.literal('setValue').describe('Set an editable filename/path through UI Automation ValuePattern'), + value: z.string().min(1).max(32767).describe('Complete absolute filename/path; the value is never logged'), + ...InteractionTargetSchema, + }), + z.object({ + action: z.literal('setPaths').describe('Set one or more existing files in a multi-select Open dialog'), + paths: z.array(z.string().min(1).max(32767)).min(1).max(100).describe( + 'Complete absolute file paths for a multi-file Open dialog; path values are never logged' + ), + ...InteractionTargetSchema, + }), + z.object({ + action: z.literal('select').describe('Select a control through UI Automation SelectionItemPattern'), + ...InteractionTargetSchema, + }), +]); + +export type NativeDialogSnapshotOptions = z.input; +export type NativeDialogInteractOptions = z.input; + +const sessionScopes = new WeakMap(); + +function getSessionScope(client: PluginClient): string { + const existing = sessionScopes.get(client); + + if (existing) { + return existing; + } + + const scope = randomUUID(); + + sessionScopes.set(client, scope); + return scope; +} + +function resultAsJSON(data: unknown): string { + return JSON.stringify(data, null, 2); +} + +/** Discover bounded semantic UI Automation controls in native Windows dialogs. */ +export async function snapshotNativeDialog(options: NativeDialogSnapshotOptions): Promise { + const parsed = NativeDialogSnapshotSchema.parse(options), + client = await ensureSessionAndConnect(parsed.appIdentifier); + + const response = await client.sendCommand({ + command: 'native_dialog_snapshot', + args: { + windowLabel: parsed.windowId, + timeoutMs: parsed.timeoutMs, + scopeId: getSessionScope(client), + minOwnerDepth: parsed.minOwnerDepth, + }, + }, parsed.timeoutMs + 1500); + + if (!response.success) { + throw new Error(response.error || 'Native dialog snapshot failed'); + } + + return resultAsJSON(response.data); +} + +/** Interact with a native dialog element using its advertised UI Automation pattern. */ +export async function interactWithNativeDialog(options: NativeDialogInteractOptions): Promise { + const parsed = NativeDialogInteractSchema.parse(options), + client = await ensureSessionAndConnect(parsed.appIdentifier); + + const response = await client.sendCommand({ + command: 'native_dialog_interact', + args: { + windowLabel: parsed.windowId, + timeoutMs: parsed.timeoutMs, + scopeId: getSessionScope(client), + elementRef: parsed.elementRef, + action: parsed.action, + value: parsed.action === 'setValue' ? parsed.value : undefined, + paths: parsed.action === 'setPaths' ? parsed.paths : undefined, + }, + }, parsed.timeoutMs + 1500); + + if (!response.success) { + throw new Error(response.error || 'Native dialog interaction failed'); + } + + return resultAsJSON(response.data); +} diff --git a/packages/mcp-server/src/driver/protocol.ts b/packages/mcp-server/src/driver/protocol.ts index f286425..0eee8f4 100644 --- a/packages/mcp-server/src/driver/protocol.ts +++ b/packages/mcp-server/src/driver/protocol.ts @@ -15,7 +15,9 @@ export type PluginCommandType = | 'stop_ipc_monitor' | 'get_ipc_events' | 'execute_js' - | 'capture_native_screenshot'; + | 'capture_native_screenshot' + | 'native_dialog_snapshot' + | 'native_dialog_interact'; /** Request message sent from MCP server to Tauri plugin */ export interface PluginRequest { diff --git a/packages/mcp-server/src/tools-registry.ts b/packages/mcp-server/src/tools-registry.ts index 54f75c6..b76fbbe 100644 --- a/packages/mcp-server/src/tools-registry.ts +++ b/packages/mcp-server/src/tools-registry.ts @@ -29,6 +29,10 @@ import { selectElement, getPointedElement, SelectElementSchema, GetPointedElementSchema, } from './driver/element-picker.js'; +import { + interactWithNativeDialog, snapshotNativeDialog, + NativeDialogInteractSchema, NativeDialogSnapshotSchema, +} from './driver/native-dialog.js'; import { SETUP_INSTRUCTIONS } from './constants.js'; /** @@ -287,6 +291,53 @@ export const TOOLS: ToolDefinition[] = [ return result.content; }, }, + { + name: 'native_dialog_snapshot', + description: + '[Windows Tauri Apps Only] Discover native message, confirmation, Open, or Save dialogs ' + + 'in the ownership chain of a targeted Tauri window. Returns a bounded semantic UI Automation snapshot ' + + 'with ownership depth, navigation and file-system controls, automation IDs, semantic roles, supported patterns, ' + + 'and ephemeral elementRef values. Set minOwnerDepth to 2 to wait for a child confirmation such as Save overwrite. ' + + 'Requires an active driver_session and an interactive Windows desktop. ' + + MULTI_APP_DESC + ' Native dialogs are outside the webview DOM; Windows toast notifications are not supported.', + category: TOOL_CATEGORIES.UI_AUTOMATION, + schema: NativeDialogSnapshotSchema, + annotations: { + title: 'Inspect Native Windows Dialog', + readOnlyHint: true, + destructiveHint: false, + openWorldHint: false, + }, + handler: async (args) => { + const parsed = NativeDialogSnapshotSchema.parse(args); + + return await snapshotNativeDialog(parsed); + }, + }, + { + name: 'native_dialog_interact', + description: + '[Windows Tauri Apps Only] Interact with an elementRef from the latest native_dialog_snapshot ' + + 'using UI Automation InvokePattern, ValuePattern, or SelectionItemPattern. Use invoke for dialog buttons, ' + + 'setValue with a complete absolute file or folder path, setPaths with complete absolute existing file paths ' + + 'for multi-select Open dialogs, and select for advertised navigation or file-system controls. ' + + 'References are session-bound, expire after 30 seconds, and are otherwise ephemeral; ' + + 'take a new snapshot after a stale-reference error. ' + + 'Requires an active driver_session and an interactive Windows desktop. ' + MULTI_APP_DESC, + category: TOOL_CATEGORIES.UI_AUTOMATION, + schema: NativeDialogInteractSchema, + annotations: { + title: 'Interact with Native Windows Dialog', + readOnlyHint: false, + destructiveHint: false, + openWorldHint: false, + }, + handler: async (args) => { + const parsed = NativeDialogInteractSchema.parse(args); + + return await interactWithNativeDialog(parsed); + }, + }, { name: 'webview_keyboard', description: diff --git a/packages/mcp-server/tests/e2e/native-dialog.test.ts b/packages/mcp-server/tests/e2e/native-dialog.test.ts new file mode 100644 index 0000000..f98d32a --- /dev/null +++ b/packages/mcp-server/tests/e2e/native-dialog.test.ts @@ -0,0 +1,321 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { + interactWithNativeDialog, + snapshotNativeDialog, +} from '../../src/driver/native-dialog'; +import { manageDriverSession } from '../../src/driver/session-manager'; +import { executeJavaScript, interact, waitFor } from '../../src/driver/webview-interactions'; +import { getTestAppPort } from '../test-utils'; + +interface SnapshotControl { + elementRef?: string; + name?: string; + semanticRole?: string; + supportedActions: string[]; +} + +interface Snapshot { + dialogs: Array<{ + dialogRef: string; + parentDialogRef?: string; + ownerDepth: number; + kind: string; + controls: SnapshotControl[]; + }>; +} + +const windowsDescribe = process.platform === 'win32' ? describe : describe.skip; + +windowsDescribe('Native Windows dialog automation E2E', () => { + const TIMEOUT = 15000; + + beforeAll(async () => { + await manageDriverSession('start', undefined, getTestAppPort()); + }); + + afterAll(async () => { + await manageDriverSession('stop'); + }); + + afterEach(async () => { + try { + const snapshot: Snapshot = JSON.parse(await snapshotNativeDialog({ timeoutMs: 500 })), + control = findControl(snapshot, 'invoke', [ 'cancel', 'close', 'no', 'accept' ]); + + await invoke(control); + } catch{ + // No dialog is the expected cleanup state. + } + }); + + async function openDialog(selector: string): Promise { + await interact({ action: 'click', selector }); + return JSON.parse(await snapshotNativeDialog({ timeoutMs: 8000 })); + } + + function findControl( + snapshot: Snapshot, + action: string, + semanticRoles: string[] + ): SnapshotControl & { elementRef: string } { + const controls = snapshot.dialogs.flatMap((dialog) => { return dialog.controls; }); + + function supportsAction(control: SnapshotControl): boolean { + return control.supportedActions.includes(action); + } + + function findSemanticControl(role: string): SnapshotControl | undefined { + return controls.find((control) => { + return control.semanticRole === role && control.supportedActions.includes(action); + }); + } + + const semanticMatch = semanticRoles.map(findSemanticControl).find((control) => { return Boolean(control); }), + patternMatch = controls.find(supportsAction); + + const control = semanticMatch ?? patternMatch; + + if (!control?.elementRef) { + throw new Error(`No ${action} control found in native dialog snapshot`); + } + return { ...control, elementRef: control.elementRef }; + } + + async function invoke(control: SnapshotControl): Promise { + await interactWithNativeDialog({ + action: 'invoke', + elementRef: control.elementRef, + timeoutMs: 3000, + }); + } + + it('discovers and acknowledges a message dialog without breaking webview automation', async () => { + const snapshot = await openDialog('#dialog-message'); + + expect(snapshot.dialogs[0]?.kind).toBe('message'); + await invoke(findControl(snapshot, 'invoke', [ 'accept', 'close' ])); + await waitFor({ type: 'text', value: 'Message dialog closed.', timeout: 3000 }); + + const webviewResult = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(webviewResult).toContain('Message dialog closed.'); + }, TIMEOUT); + + it('answers ask and confirm dialogs using non-localized semantic roles', async () => { + const askSnapshot = await openDialog('#dialog-ask'); + + await invoke(findControl(askSnapshot, 'invoke', [ 'yes', 'accept', 'primary' ])); + await waitFor({ type: 'text', value: 'Ask result: Yes', timeout: 3000 }); + + const confirmSnapshot = await openDialog('#dialog-confirm'); + + await invoke(findControl(confirmSnapshot, 'invoke', [ 'cancel', 'secondary' ])); + await waitFor({ type: 'text', value: 'Confirm result: Cancel', timeout: 3000 }); + }, TIMEOUT); + + it('sets absolute Open and Save paths through ValuePattern and accepts them', async () => { + const fixtureDirectory = await mkdtemp(path.join(tmpdir(), 'tauri-mcp-dialog-')), + fixturePath = path.join(fixtureDirectory, 'fixture.txt'), + savePath = path.join(fixtureDirectory, 'saved-output.txt'); + + try { + await writeFile(fixturePath, 'temporary fixture', 'utf8'); + const snapshot = await openDialog('#dialog-open'), + valueControl = findControl(snapshot, 'setValue', [ 'fileName', 'editable' ]), + acceptControl = findControl(snapshot, 'invoke', [ 'accept' ]); + + await interactWithNativeDialog({ + action: 'setValue', + elementRef: valueControl.elementRef, + value: fixturePath, + timeoutMs: 3000, + }); + await invoke(acceptControl); + await waitFor({ type: 'text', value: 'Open result:', timeout: 3000 }); + + const result = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(result).toContain('fixture.txt'); + + const saveSnapshot = await openDialog('#dialog-save'), + saveValueControl = findControl(saveSnapshot, 'setValue', [ 'fileName', 'editable' ]), + saveAcceptControl = findControl(saveSnapshot, 'invoke', [ 'accept' ]); + + await interactWithNativeDialog({ + action: 'setValue', + elementRef: saveValueControl.elementRef, + value: savePath, + timeoutMs: 3000, + }); + await invoke(saveAcceptControl); + await waitFor({ type: 'text', value: 'Save result:', timeout: 3000 }); + + const saveResult = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(saveResult).toContain('saved-output.txt'); + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } + }, TIMEOUT); + + it('selects multiple absolute files in one Open dialog', async () => { + const fixtureDirectory = await mkdtemp(path.join(tmpdir(), 'tauri-mcp-multi-dialog-')), + firstPath = path.join(fixtureDirectory, 'first.txt'), + secondPath = path.join(fixtureDirectory, 'second.txt'); + + try { + await Promise.all([ + writeFile(firstPath, 'first fixture', 'utf8'), + writeFile(secondPath, 'second fixture', 'utf8'), + ]); + const snapshot = await openDialog('#dialog-open-multiple'), + valueControl = findControl(snapshot, 'setPaths', [ 'fileName' ]), + acceptControl = findControl(snapshot, 'invoke', [ 'accept' ]); + + await interactWithNativeDialog({ + action: 'setPaths', + elementRef: valueControl.elementRef, + paths: [ firstPath, secondPath ], + timeoutMs: 3000, + }); + await invoke(acceptControl); + await waitFor({ type: 'text', value: 'Multiple open result:', timeout: 3000 }); + + const result = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(result).toContain('first.txt'); + expect(result).toContain('second.txt'); + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } + }, TIMEOUT); + + it('selects an absolute directory in a folder picker', async () => { + const fixtureDirectory = await mkdtemp(path.join(tmpdir(), 'tauri-mcp-folder-dialog-')); + + try { + const snapshot = await openDialog('#dialog-folder'), + valueControl = findControl(snapshot, 'setValue', [ 'fileName', 'editable' ]), + acceptControl = findControl(snapshot, 'invoke', [ 'accept' ]); + + await interactWithNativeDialog({ + action: 'setValue', + elementRef: valueControl.elementRef, + value: fixtureDirectory, + timeoutMs: 3000, + }); + await invoke(acceptControl); + await waitFor({ type: 'text', value: 'Folder result:', timeout: 3000 }); + + const result = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(result).toContain(path.basename(fixtureDirectory)); + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } + }, TIMEOUT); + + it('discovers navigation controls and selectable file-system entries', async () => { + function isNavigationRole(role: string | undefined): boolean { + return [ 'back', 'forward', 'up', 'addressBar', 'currentLocation', 'navigationTreeItem' ] + .includes(role ?? ''); + } + + function isFileSystemEntry(control: SnapshotControl): boolean { + return control.semanticRole === 'fileSystemEntry'; + } + + const fixtureDirectory = await mkdtemp(path.join(tmpdir(), 'tauri-mcp-navigation-dialog-')), + fixturePath = path.join(fixtureDirectory, 'navigation-fixture.txt'); + + try { + await writeFile(fixturePath, 'navigation fixture', 'utf8'); + const snapshot = await openDialog('#dialog-open'), + controls = snapshot.dialogs.flatMap((dialog) => { return dialog.controls; }), + roles = controls.map((control) => { return control.semanticRole; }), + hasNavigationControl = roles.some(isNavigationRole), + valueControl = findControl(snapshot, 'setValue', [ 'fileName' ]), + acceptControl = findControl(snapshot, 'invoke', [ 'accept' ]); + + expect(hasNavigationControl).toBe(true); + await interactWithNativeDialog({ + action: 'setValue', + elementRef: valueControl.elementRef, + value: fixtureDirectory, + timeoutMs: 3000, + }); + await invoke(acceptControl); + + const navigatedSnapshot: Snapshot = JSON.parse(await snapshotNativeDialog({ timeoutMs: 4000 })), + navigatedControls = navigatedSnapshot.dialogs.flatMap((dialog) => { return dialog.controls; }), + entries = navigatedControls.filter(isFileSystemEntry); + + expect(entries.some((entry) => { return entry.name === 'navigation-fixture.txt'; })).toBe(true); + await invoke(findControl(navigatedSnapshot, 'invoke', [ 'cancel' ])); + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } + }, TIMEOUT); + + it('follows nested ownership to accept a Save overwrite confirmation', async () => { + const fixtureDirectory = await mkdtemp(path.join(tmpdir(), 'tauri-mcp-overwrite-dialog-')), + existingPath = path.join(fixtureDirectory, 'existing.txt'); + + try { + await writeFile(existingPath, 'existing fixture', 'utf8'); + const saveSnapshot = await openDialog('#dialog-save'), + valueControl = findControl(saveSnapshot, 'setValue', [ 'fileName' ]), + acceptControl = findControl(saveSnapshot, 'invoke', [ 'accept' ]); + + await interactWithNativeDialog({ + action: 'setValue', + elementRef: valueControl.elementRef, + value: existingPath, + timeoutMs: 3000, + }); + await invoke(acceptControl); + + const confirmation: Snapshot = JSON.parse(await snapshotNativeDialog({ + minOwnerDepth: 2, + timeoutMs: 4000, + })); + + expect(confirmation.dialogs[0]?.ownerDepth).toBeGreaterThanOrEqual(2); + await invoke(findControl(confirmation, 'invoke', [ 'yes', 'accept', 'primary' ])); + await waitFor({ type: 'text', value: 'Save result:', timeout: 3000 }); + + const result = await executeJavaScript({ script: 'return document.querySelector("#dialog-result")?.textContent' }); + + expect(result).toContain('existing.txt'); + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } + }, TIMEOUT); + + it('cancels Open and Save dialogs', async () => { + const openSnapshot = await openDialog('#dialog-open'); + + await invoke(findControl(openSnapshot, 'invoke', [ 'cancel' ])); + await waitFor({ type: 'text', value: 'Open result: Cancelled', timeout: 3000 }); + + const saveSnapshot = await openDialog('#dialog-save'); + + await invoke(findControl(saveSnapshot, 'invoke', [ 'cancel' ])); + await waitFor({ type: 'text', value: 'Save result: Cancelled', timeout: 3000 }); + }, TIMEOUT); + + it('fails safely for stale references and no-dialog timeouts', async () => { + const snapshot = await openDialog('#dialog-message'), + control = findControl(snapshot, 'invoke', [ 'accept', 'close' ]); + + await invoke(control); + await expect(invoke(control)).rejects.toThrow('Stale native dialog element reference'); + await expect(snapshotNativeDialog({ timeoutMs: 500 })).rejects.toThrow( + 'No native dialog owned by the targeted Tauri window' + ); + }, TIMEOUT); +}); diff --git a/packages/mcp-server/tests/unit/native-dialog.test.ts b/packages/mcp-server/tests/unit/native-dialog.test.ts new file mode 100644 index 0000000..ff7bae8 --- /dev/null +++ b/packages/mcp-server/tests/unit/native-dialog.test.ts @@ -0,0 +1,150 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { sendCommand, ensureSessionAndConnect } = vi.hoisted(() => { + return { + sendCommand: vi.fn(), + ensureSessionAndConnect: vi.fn(), + }; +}); + +vi.mock('../../src/driver/plugin-client', () => { + return { ensureSessionAndConnect }; +}); + +import { + interactWithNativeDialog, + NativeDialogInteractSchema, + NativeDialogSnapshotSchema, + snapshotNativeDialog, +} from '../../src/driver/native-dialog'; + +describe('Native dialog tools', () => { + beforeEach(() => { + vi.clearAllMocks(); + ensureSessionAndConnect.mockResolvedValue({ sendCommand }); + }); + + it('sends a bounded semantic snapshot request through the active session', async () => { + sendCommand.mockResolvedValue({ + success: true, + data: { platform: 'windows', dialogs: [], dialogCount: 0 }, + }); + + const result = JSON.parse(await snapshotNativeDialog({ + windowId: 'settings', + appIdentifier: 9300, + timeoutMs: 500, + })); + + expect(result.platform).toBe('windows'); + expect(ensureSessionAndConnect).toHaveBeenCalledWith(9300); + expect(sendCommand).toHaveBeenCalledWith({ + command: 'native_dialog_snapshot', + args: { + windowLabel: 'settings', + timeoutMs: 500, + scopeId: expect.any(String), + minOwnerDepth: 1, + }, + }, 2000); + }); + + it('does not log or echo a path in an interaction result', async () => { + const sensitivePath = 'C:\\temporary\\private-fixture.txt', + logSpy = vi.spyOn(console, 'log').mockImplementation(() => { return undefined; }), + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => { return undefined; }); + + sendCommand.mockResolvedValue({ + success: true, + data: { action: 'setValue', elementRef: 'native_ref', referencesInvalidated: false }, + }); + + const result = await interactWithNativeDialog({ + action: 'setValue', + elementRef: 'native_ref', + value: sensitivePath, + timeoutMs: 500, + }); + + expect(result).not.toContain(sensitivePath); + expect(logSpy).not.toHaveBeenCalled(); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it('sends multiple absolute file paths without exposing them in the result', async () => { + const paths = [ 'C:\\temporary\\first.txt', 'C:\\temporary\\second.txt' ]; + + sendCommand.mockResolvedValue({ + success: true, + data: { action: 'setPaths', elementRef: 'native_ref', referencesInvalidated: false }, + }); + + const result = await interactWithNativeDialog({ + action: 'setPaths', + elementRef: 'native_ref', + paths, + timeoutMs: 500, + }); + + expect(result).not.toContain(paths[0]); + expect(result).not.toContain(paths[1]); + expect(sendCommand).toHaveBeenCalledWith({ + command: 'native_dialog_interact', + args: { + action: 'setPaths', + elementRef: 'native_ref', + paths, + scopeId: expect.any(String), + timeoutMs: 500, + value: undefined, + windowLabel: undefined, + }, + }, 2000); + }); + + it('surfaces stale and unsupported-platform bridge errors clearly', async () => { + sendCommand.mockResolvedValueOnce({ + success: false, + error: 'Stale native dialog element reference; take a new native_dialog_snapshot', + }); + + const staleInteraction = interactWithNativeDialog({ + action: 'invoke', + elementRef: 'expired', + timeoutMs: 500, + }); + + await expect(staleInteraction) + .rejects + .toThrow('Stale native dialog element reference'); + + sendCommand.mockResolvedValueOnce({ + success: false, + error: 'Native dialog automation is only supported on Windows with an interactive desktop', + }); + + await expect(snapshotNativeDialog({ timeoutMs: 500 })) + .rejects + .toThrow('only supported on Windows'); + }); + + it('rejects unbounded timeouts and missing setValue data', () => { + function parseMissingValue(): unknown { + return NativeDialogInteractSchema.parse({ action: 'setValue', elementRef: 'native_ref' }); + } + + function parseMissingPaths(): unknown { + return NativeDialogInteractSchema.parse({ + action: 'setPaths', + elementRef: 'native_ref', + paths: [], + }); + } + + expect(() => { return NativeDialogSnapshotSchema.parse({ timeoutMs: 10001 }); }) + .toThrow(); + + expect(parseMissingValue).toThrow(); + expect(parseMissingPaths).toThrow(); + }); +}); diff --git a/packages/tauri-plugin-mcp-bridge/CHANGELOG.md b/packages/tauri-plugin-mcp-bridge/CHANGELOG.md index 9fce9fe..fbcf597 100644 --- a/packages/tauri-plugin-mcp-bridge/CHANGELOG.md +++ b/packages/tauri-plugin-mcp-bridge/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Add Windows UI Automation dialog discovery and interaction on a dedicated COM MTA thread, restricted to same-process dialog ownership chains rooted at the connected Tauri window and supporting multi-file, folder, navigation, and nested confirmation workflows. + ## [0.12.0] - 2026-07-05 ### Added diff --git a/packages/tauri-plugin-mcp-bridge/Cargo.toml b/packages/tauri-plugin-mcp-bridge/Cargo.toml index 6d4eecc..21b2b49 100644 --- a/packages/tauri-plugin-mcp-bridge/Cargo.toml +++ b/packages/tauri-plugin-mcp-bridge/Cargo.toml @@ -62,7 +62,7 @@ objc2-web-kit = { version = "0.2", features = ["WKSnapshotConfiguration", "block [target.'cfg(target_os = "windows")'.dependencies] webview2-com = "0.38" -windows = { version = "0.61", features = ["Win32_System_Com", "Win32_System_Com_StructuredStorage", "Win32_Foundation"] } +windows = { version = "0.61", features = ["Win32_System_Com", "Win32_System_Com_StructuredStorage", "Win32_Foundation", "Win32_UI_Accessibility", "Win32_UI_WindowsAndMessaging"] } windows-core = "0.61" # Linux: Native screenshot not yet implemented due to webkit2gtk/glib version conflicts diff --git a/packages/tauri-plugin-mcp-bridge/README.md b/packages/tauri-plugin-mcp-bridge/README.md index 60c5ad0..60cf795 100644 --- a/packages/tauri-plugin-mcp-bridge/README.md +++ b/packages/tauri-plugin-mcp-bridge/README.md @@ -117,13 +117,13 @@ await invoke('plugin:mcp-bridge|emit_event', { ## MCP Server Integration -This plugin is part of the larger MCP Server for Tauri, which provides **20 total MCP tools** for comprehensive Tauri development and testing. The plugin specifically enables the following tools: +This plugin is part of the larger MCP Server for Tauri, which provides **22 total MCP tools** for comprehensive Tauri development and testing. The plugin specifically enables the following tools: ### Mobile Development Tools (1) 1. **list_devices** - List connected Android devices and iOS simulators -### UI Automation & WebView Tools (14) +### UI Automation & WebView Tools (16) Tools for UI automation and webview interaction via the plugin's WebSocket connection: @@ -133,14 +133,16 @@ Tools for UI automation and webview interaction via the plugin's WebSocket conne 4. **read_logs** - Read logs (console, Android logcat, iOS, system) 5. **webview_interact** - Perform gestures (click, double-click, long-press, swipe, scroll, focus) 6. **webview_screenshot** - Take screenshots (JPEG default, with optional resizing) -7. **webview_keyboard** - Type text or simulate keyboard events with optional modifiers -8. **webview_wait_for** - Wait for element selectors, text content, or IPC events -9. **webview_get_styles** - Get computed CSS styles for element(s) -10. **webview_execute_js** - Execute arbitrary JavaScript code in the webview context -11. **webview_dom_snapshot** - Get structured DOM snapshot (accessibility or structure type) -12. **webview_select_element** - Visual element picker — user clicks an element, returns metadata + screenshot -13. **webview_get_pointed_element** - Retrieve metadata for element user Alt+Shift+Clicked -14. **get_setup_instructions** - Get setup/update instructions for the MCP Bridge plugin +7. **native_dialog_snapshot** - Inspect same-process native Windows dialog ownership chains and navigation controls through UI Automation +8. **native_dialog_interact** - Invoke, set single/multiple paths, or select native Windows dialog controls +9. **webview_keyboard** - Type text or simulate keyboard events with optional modifiers +10. **webview_wait_for** - Wait for element selectors, text content, or IPC events +11. **webview_get_styles** - Get computed CSS styles for element(s) +12. **webview_execute_js** - Execute arbitrary JavaScript code in the webview context +13. **webview_dom_snapshot** - Get structured DOM snapshot (accessibility or structure type) +14. **webview_select_element** - Visual element picker — user clicks an element, returns metadata + screenshot +15. **webview_get_pointed_element** - Retrieve metadata for element user Alt+Shift+Clicked +16. **get_setup_instructions** - Get setup/update instructions for the MCP Bridge plugin ### IPC Tools (5) diff --git a/packages/tauri-plugin-mcp-bridge/src/lib.rs b/packages/tauri-plugin-mcp-bridge/src/lib.rs index b20e3a6..9dba521 100644 --- a/packages/tauri-plugin-mcp-bridge/src/lib.rs +++ b/packages/tauri-plugin-mcp-bridge/src/lib.rs @@ -83,6 +83,7 @@ pub mod config; pub mod discovery; mod logging; pub mod monitor; +pub mod native_dialog; pub mod screenshot; pub mod script_registry; pub mod utils; @@ -187,6 +188,10 @@ pub fn init_with_config(config: Config) -> TauriPlugin { let script_registry = create_shared_registry(); app.manage(script_registry); + // UI Automation stays on its own COM MTA thread. The managed value + // contains only the request channel used by WebSocket handlers. + app.manage(native_dialog::NativeDialogAutomation::new()); + // Find an available port for WebSocket server let port = find_available_port(&bind_address, base_port); diff --git a/packages/tauri-plugin-mcp-bridge/src/native_dialog/mod.rs b/packages/tauri-plugin-mcp-bridge/src/native_dialog/mod.rs new file mode 100644 index 0000000..7201b08 --- /dev/null +++ b/packages/tauri-plugin-mcp-bridge/src/native_dialog/mod.rs @@ -0,0 +1,119 @@ +//! Native Windows dialog discovery and interaction. +//! +//! UI Automation objects are apartment-bound. The platform implementation keeps +//! every COM object on a dedicated MTA thread and exposes only serializable data +//! and opaque element references to the WebSocket layer. + +use serde::Serialize; +use std::time::Duration; + +#[cfg(not(target_os = "windows"))] +mod unsupported; +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(not(target_os = "windows"))] +pub use unsupported::NativeDialogAutomation; +#[cfg(target_os = "windows")] +pub use windows::NativeDialogAutomation; + +/// Maximum time a caller may ask the UI Automation worker to wait. +pub const MAX_TIMEOUT: Duration = Duration::from_secs(10); + +/// A request to discover dialogs in the ownership chain of a Tauri window. +#[derive(Debug, Clone)] +pub struct SnapshotRequest { + pub process_id: u32, + pub owner_window: usize, + pub scope_id: String, + pub min_owner_depth: usize, + pub timeout: Duration, +} + +/// A semantic action supported by a UI Automation control pattern. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum NativeDialogAction { + Invoke, + SetValue, + SetPaths, + Select, +} + +impl NativeDialogAction { + /// Parses the public WebSocket action name without including command data in errors. + pub fn parse(value: &str) -> Result { + match value { + "invoke" => Ok(Self::Invoke), + "setValue" => Ok(Self::SetValue), + "setPaths" => Ok(Self::SetPaths), + "select" => Ok(Self::Select), + _ => Err("Unsupported native dialog action".to_string()), + } + } +} + +/// A request to interact with an element from the latest snapshot in a session. +#[derive(Debug, Clone)] +pub struct InteractRequest { + pub process_id: u32, + pub owner_window: usize, + pub scope_id: String, + pub element_ref: String, + pub action: NativeDialogAction, + pub value: Option, + pub paths: Option>, + pub timeout: Duration, +} + +/// Semantic metadata for a native dialog control. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeDialogControl { + pub element_ref: Option, + pub control_type: String, + pub name: String, + pub automation_id: String, + pub semantic_role: Option, + pub enabled: bool, + pub offscreen: bool, + pub depth: usize, + pub supported_actions: Vec, +} + +/// A bounded semantic snapshot of one native dialog. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeDialog { + pub dialog_ref: String, + pub parent_dialog_ref: Option, + pub owner_depth: usize, + pub kind: String, + pub title: String, + pub automation_id: String, + pub controls: Vec, + pub truncated: bool, +} + +/// Snapshot returned to the MCP server. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeDialogSnapshot { + pub platform: &'static str, + pub interactive_desktop_required: bool, + pub dialogs: Vec, + pub dialog_count: usize, +} + +/// Result of applying a UI Automation control pattern. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeDialogInteractionResult { + pub action: NativeDialogAction, + pub element_ref: String, + pub references_invalidated: bool, +} + +pub(crate) fn bounded_timeout(timeout: Duration) -> Duration { + timeout.min(MAX_TIMEOUT) +} diff --git a/packages/tauri-plugin-mcp-bridge/src/native_dialog/unsupported.rs b/packages/tauri-plugin-mcp-bridge/src/native_dialog/unsupported.rs new file mode 100644 index 0000000..59dceca --- /dev/null +++ b/packages/tauri-plugin-mcp-bridge/src/native_dialog/unsupported.rs @@ -0,0 +1,65 @@ +use super::{ + InteractRequest, NativeDialogInteractionResult, NativeDialogSnapshot, SnapshotRequest, +}; + +const UNSUPPORTED: &str = + "Native dialog automation is only supported on Windows with an interactive desktop"; + +/// Non-Windows implementation that preserves cross-platform compilation and errors clearly. +#[derive(Clone, Default)] +pub struct NativeDialogAutomation; + +impl NativeDialogAutomation { + pub fn new() -> Self { + Self + } + + pub fn snapshot(&self, _request: SnapshotRequest) -> Result { + Err(UNSUPPORTED.to_string()) + } + + pub fn interact( + &self, + _request: InteractRequest, + ) -> Result { + Err(UNSUPPORTED.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::native_dialog::{NativeDialogAction, SnapshotRequest}; + use std::time::Duration; + + #[test] + fn unsupported_platform_fails_clearly() { + let automation = NativeDialogAutomation::new(); + let error = automation + .snapshot(SnapshotRequest { + process_id: 1, + owner_window: 0, + scope_id: "test".to_string(), + min_owner_depth: 1, + timeout: Duration::from_millis(100), + }) + .unwrap_err(); + + assert!(error.contains("only supported on Windows")); + + let error = automation + .interact(InteractRequest { + process_id: 1, + owner_window: 0, + scope_id: "test".to_string(), + element_ref: "missing".to_string(), + action: NativeDialogAction::Invoke, + value: None, + paths: None, + timeout: Duration::from_millis(100), + }) + .unwrap_err(); + + assert!(error.contains("interactive desktop")); + } +} diff --git a/packages/tauri-plugin-mcp-bridge/src/native_dialog/windows.rs b/packages/tauri-plugin-mcp-bridge/src/native_dialog/windows.rs new file mode 100644 index 0000000..5ae4a21 --- /dev/null +++ b/packages/tauri-plugin-mcp-bridge/src/native_dialog/windows.rs @@ -0,0 +1,1023 @@ +use super::{ + bounded_timeout, InteractRequest, NativeDialog, NativeDialogAction, NativeDialogControl, + NativeDialogInteractionResult, NativeDialogSnapshot, SnapshotRequest, +}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::Path; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::thread; +use std::time::{Duration, Instant}; +use uuid::Uuid; +use windows::core::{BOOL, BSTR}; +use windows::Win32::Foundation::{HWND, LPARAM}; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, +}; +use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern, + IUIAutomationSelectionItemPattern, IUIAutomationValuePattern, UIA_ButtonControlTypeId, + UIA_CheckBoxControlTypeId, UIA_ComboBoxControlTypeId, UIA_DataItemControlTypeId, + UIA_EditControlTypeId, UIA_InvokePatternId, UIA_ListItemControlTypeId, UIA_PaneControlTypeId, + UIA_RadioButtonControlTypeId, UIA_SelectionItemPatternId, UIA_TextControlTypeId, + UIA_TreeItemControlTypeId, UIA_ValuePatternId, UIA_WindowControlTypeId, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, GetWindow, GetWindowThreadProcessId, IsWindow, IsWindowVisible, GW_OWNER, +}; + +const MAX_DIALOGS: usize = 4; +const MAX_ENUMERATED_DIALOGS: usize = 32; +const MAX_CONTROLS_PER_DIALOG: usize = 128; +const MAX_DEPTH: usize = 12; +const MAX_OWNER_CHAIN_DEPTH: usize = 8; +const MAX_MULTI_SELECT_PATHS: usize = 100; +const MAX_TEXT_CHARS: usize = 256; +const MAX_PATH_CHARS: usize = 32_767; +const REQUEST_QUEUE_CAPACITY: usize = 16; +const POLL_INTERVAL: Duration = Duration::from_millis(50); +const ELEMENT_REFERENCE_TTL: Duration = Duration::from_secs(30); +const STALE_REFERENCE_ERROR: &str = + "Stale native dialog element reference; take a new native_dialog_snapshot"; + +enum WorkerRequest { + Snapshot { + request: SnapshotRequest, + response: SyncSender>, + }, + Interact { + request: InteractRequest, + response: SyncSender>, + }, +} + +/// Request-channel facade for a dedicated UI Automation MTA thread. +#[derive(Clone)] +pub struct NativeDialogAutomation { + sender: Option>, + startup_error: Option, +} + +impl Default for NativeDialogAutomation { + fn default() -> Self { + Self::new() + } +} + +impl NativeDialogAutomation { + pub fn new() -> Self { + let (sender, receiver) = mpsc::sync_channel(REQUEST_QUEUE_CAPACITY); + let spawn_result = thread::Builder::new() + .name("tauri-mcp-uia-mta".to_string()) + .spawn(move || run_worker(receiver)); + + match spawn_result { + Ok(_) => Self { + sender: Some(sender), + startup_error: None, + }, + Err(_) => Self { + sender: None, + startup_error: Some( + "Failed to start the native dialog automation thread".to_string(), + ), + }, + } + } + + pub fn snapshot(&self, request: SnapshotRequest) -> Result { + let timeout = bounded_timeout(request.timeout); + let (response_tx, response_rx) = mpsc::sync_channel(1); + let Some(sender) = &self.sender else { + return Err(self + .startup_error + .clone() + .unwrap_or_else(|| "Native dialog automation is unavailable".to_string())); + }; + + sender + .try_send(WorkerRequest::Snapshot { + request, + response: response_tx, + }) + .map_err(request_queue_error)?; + + response_rx + .recv_timeout(timeout + Duration::from_secs(1)) + .map_err(|_| "Native dialog snapshot timed out".to_string())? + } + + pub fn interact( + &self, + request: InteractRequest, + ) -> Result { + let timeout = bounded_timeout(request.timeout); + let (response_tx, response_rx) = mpsc::sync_channel(1); + let Some(sender) = &self.sender else { + return Err(self + .startup_error + .clone() + .unwrap_or_else(|| "Native dialog automation is unavailable".to_string())); + }; + + sender + .try_send(WorkerRequest::Interact { + request, + response: response_tx, + }) + .map_err(request_queue_error)?; + + response_rx + .recv_timeout(timeout + Duration::from_secs(1)) + .map_err(|_| "Native dialog interaction timed out".to_string())? + } +} + +fn request_queue_error(error: TrySendError) -> String { + match error { + TrySendError::Full(_) => "Native dialog automation is busy; retry the request".to_string(), + TrySendError::Disconnected(_) => { + "Native dialog automation thread stopped unexpectedly".to_string() + } + } +} + +fn run_worker(receiver: Receiver) { + let initialization = unsafe { initialize_worker() }; + + match initialization { + Ok(mut worker) => { + loop { + match receiver.recv_timeout(Duration::from_secs(1)) { + Ok(WorkerRequest::Snapshot { request, response }) => { + let _ = response.send(worker.snapshot(request)); + } + Ok(WorkerRequest::Interact { request, response }) => { + let _ = response.send(worker.interact(request)); + } + Err(RecvTimeoutError::Timeout) => worker.purge_expired(), + Err(RecvTimeoutError::Disconnected) => break, + } + } + drop(worker); + unsafe { CoUninitialize() }; + } + Err(error) => { + while let Ok(request) = receiver.recv() { + match request { + WorkerRequest::Snapshot { response, .. } => { + let _ = response.send(Err(error.clone())); + } + WorkerRequest::Interact { response, .. } => { + let _ = response.send(Err(error.clone())); + } + } + } + } + } +} + +unsafe fn initialize_worker() -> Result { + CoInitializeEx(None, COINIT_MULTITHREADED) + .ok() + .map_err(|error| hresult_error("COM MTA initialization", &error))?; + + let automation: IUIAutomation = CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|error| { + CoUninitialize(); + hresult_error("UI Automation initialization", &error) + })?; + + Ok(NativeDialogWorker { + automation, + elements: HashMap::new(), + }) +} + +struct CachedElement { + element: IUIAutomationElement, + dialog_window: HWND, + process_id: u32, + owner_window: HWND, + scope_id: String, + supported_actions: Vec, + expires_at: Instant, +} + +impl Clone for CachedElement { + fn clone(&self) -> Self { + Self { + element: self.element.clone(), + dialog_window: self.dialog_window, + process_id: self.process_id, + owner_window: self.owner_window, + scope_id: self.scope_id.clone(), + supported_actions: self.supported_actions.clone(), + expires_at: self.expires_at, + } + } +} + +struct NativeDialogWorker { + automation: IUIAutomation, + elements: HashMap, +} + +impl NativeDialogWorker { + fn snapshot(&mut self, request: SnapshotRequest) -> Result { + self.elements.clear(); + let timeout = bounded_timeout(request.timeout); + let deadline = Instant::now() + timeout; + + loop { + let dialogs = self.collect_dialogs(&request, deadline)?; + if !dialogs.is_empty() { + return Ok(NativeDialogSnapshot { + platform: "windows", + interactive_desktop_required: true, + dialog_count: dialogs.len(), + dialogs, + }); + } + + if Instant::now() >= deadline { + return Err(format!( + "No native dialog owned by the targeted Tauri window appeared within {} ms", + timeout.as_millis() + )); + } + + thread::sleep(POLL_INTERVAL.min(deadline.saturating_duration_since(Instant::now()))); + } + } + + fn collect_dialogs( + &mut self, + request: &SnapshotRequest, + deadline: Instant, + ) -> Result, String> { + let owner_window = hwnd_from_usize(request.owner_window); + let dialog_windows = enumerate_owned_dialog_windows(owner_window, request.process_id)?; + let selected_windows: Vec = dialog_windows + .into_iter() + .filter(|dialog| dialog.owner_depth >= request.min_owner_depth) + .take(MAX_DIALOGS) + .collect(); + let dialog_refs: HashMap = selected_windows + .iter() + .map(|dialog| { + ( + hwnd_as_usize(dialog.window), + format!("dialog_{}", Uuid::new_v4()), + ) + }) + .collect(); + let mut dialogs = Vec::with_capacity(selected_windows.len()); + + for owned_dialog in selected_windows { + if Instant::now() >= deadline { + return Err("Native dialog snapshot timed out during bounded traversal".to_string()); + } + let dialog_window = owned_dialog.window; + + let root = unsafe { self.automation.ElementFromHandle(dialog_window) }.map_err(|error| { + hresult_error( + "UI Automation could not access the native dialog; an interactive desktop is required", + &error, + ) + })?; + + let current_process = unsafe { root.CurrentProcessId() }.unwrap_or_default(); + if current_process != request.process_id as i32 { + continue; + } + + let title = bounded_bstr(unsafe { root.CurrentName() }); + let automation_id = bounded_bstr(unsafe { root.CurrentAutomationId() }); + let (mut controls, truncated) = + self.collect_controls(&root, dialog_window, owner_window, request, deadline)?; + let kind = infer_dialog_kind(&controls).to_string(); + if kind == "file" { + controls.retain(is_supported_file_dialog_control); + let retained_refs: HashSet<&str> = controls + .iter() + .filter_map(|control| control.element_ref.as_deref()) + .collect(); + self.elements.retain(|element_ref, cached| { + cached.dialog_window != dialog_window + || retained_refs.contains(element_ref.as_str()) + }); + } + if controls.iter().all(|control| control.element_ref.is_none()) { + continue; + } + + dialogs.push(NativeDialog { + dialog_ref: dialog_refs + .get(&hwnd_as_usize(dialog_window)) + .cloned() + .unwrap_or_else(|| format!("dialog_{}", Uuid::new_v4())), + parent_dialog_ref: owned_dialog + .immediate_owner + .and_then(|owner| dialog_refs.get(&hwnd_as_usize(owner)).cloned()), + owner_depth: owned_dialog.owner_depth, + kind, + title, + automation_id, + controls, + truncated, + }); + } + + Ok(dialogs) + } + + fn collect_controls( + &mut self, + root: &IUIAutomationElement, + dialog_window: HWND, + owner_window: HWND, + request: &SnapshotRequest, + deadline: Instant, + ) -> Result<(Vec, bool), String> { + let walker = unsafe { self.automation.ControlViewWalker() } + .map_err(|error| hresult_error("UI Automation tree walker creation", &error))?; + let mut queue = VecDeque::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(root) } { + queue.push_back((child, 0)); + } + + let mut controls = Vec::new(); + let mut visited = 0; + let mut truncated = false; + + while let Some((element, depth)) = queue.pop_front() { + if Instant::now() >= deadline { + return Err("Native dialog snapshot timed out during bounded traversal".to_string()); + } + if visited >= MAX_CONTROLS_PER_DIALOG { + truncated = true; + break; + } + visited += 1; + + let process_id = unsafe { element.CurrentProcessId() }.unwrap_or_default(); + if process_id == request.process_id as i32 { + let control = + self.describe_control(&element, dialog_window, owner_window, request, depth); + controls.push(control); + } + + if depth < MAX_DEPTH { + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + queue.push_back((child, depth + 1)); + } + } else { + truncated = true; + } + + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + queue.push_back((sibling, depth)); + } + } + + Ok((controls, truncated)) + } + + fn describe_control( + &mut self, + element: &IUIAutomationElement, + dialog_window: HWND, + owner_window: HWND, + request: &SnapshotRequest, + depth: usize, + ) -> NativeDialogControl { + let control_type = unsafe { element.CurrentControlType() }.unwrap_or_default(); + let control_type_name = control_type_name(control_type.0); + let is_password = unsafe { element.CurrentIsPassword() } + .map(|value| value.as_bool()) + .unwrap_or(false); + let name = if is_password { + String::new() + } else { + bounded_bstr(unsafe { element.CurrentName() }) + }; + let automation_id = bounded_bstr(unsafe { element.CurrentAutomationId() }); + let supported_actions = supported_actions(element, control_type.0, &automation_id); + let semantic_role = infer_semantic_role(control_type.0, &automation_id, &supported_actions); + let enabled = unsafe { element.CurrentIsEnabled() } + .map(|value| value.as_bool()) + .unwrap_or(false); + let offscreen = unsafe { element.CurrentIsOffscreen() } + .map(|value| value.as_bool()) + .unwrap_or(false); + + let element_ref = if supported_actions.is_empty() || is_password { + None + } else { + let element_ref = format!("native_{}", Uuid::new_v4()); + self.elements.insert( + element_ref.clone(), + CachedElement { + element: element.clone(), + dialog_window, + process_id: request.process_id, + owner_window, + scope_id: request.scope_id.clone(), + supported_actions: supported_actions.clone(), + expires_at: Instant::now() + ELEMENT_REFERENCE_TTL, + }, + ); + Some(element_ref) + }; + + NativeDialogControl { + element_ref, + control_type: control_type_name.to_string(), + name, + automation_id, + semantic_role, + enabled, + offscreen, + depth, + supported_actions, + } + } + + fn interact( + &mut self, + request: InteractRequest, + ) -> Result { + self.purge_expired(); + let Some(cached) = self.elements.get(&request.element_ref).cloned() else { + return Err(STALE_REFERENCE_ERROR.to_string()); + }; + let owner_window = hwnd_from_usize(request.owner_window); + if cached.scope_id != request.scope_id + || cached.process_id != request.process_id + || cached.owner_window != owner_window + || !is_within_security_boundary( + cached.process_id, + request.process_id, + cached.owner_window, + owner_window, + ) + || !is_owned_dialog_window(cached.dialog_window, owner_window, request.process_id) + { + self.elements.remove(&request.element_ref); + return Err(STALE_REFERENCE_ERROR.to_string()); + } + + let element_process = unsafe { cached.element.CurrentProcessId() }.map_err(|_| { + self.elements.remove(&request.element_ref); + STALE_REFERENCE_ERROR.to_string() + })?; + if element_process != request.process_id as i32 + || !cached.supported_actions.contains(&request.action) + { + self.elements.remove(&request.element_ref); + return Err(STALE_REFERENCE_ERROR.to_string()); + } + + match request.action { + NativeDialogAction::Invoke => { + let pattern: IUIAutomationInvokePattern = + unsafe { cached.element.GetCurrentPatternAs(UIA_InvokePatternId) } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + unsafe { pattern.Invoke() } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + } + NativeDialogAction::SetValue => { + let value = request + .value + .as_deref() + .ok_or_else(|| "setValue requires a complete absolute path".to_string())?; + validate_absolute_path(value)?; + let pattern: IUIAutomationValuePattern = + unsafe { cached.element.GetCurrentPatternAs(UIA_ValuePatternId) } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + let value = BSTR::from(value); + unsafe { pattern.SetValue(&value) } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + } + NativeDialogAction::SetPaths => { + let paths = request.paths.as_deref().ok_or_else(|| { + "setPaths requires one or more absolute file paths".to_string() + })?; + let formatted_paths = format_multi_select_paths(paths)?; + let pattern: IUIAutomationValuePattern = + unsafe { cached.element.GetCurrentPatternAs(UIA_ValuePatternId) } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + let value = BSTR::from(formatted_paths); + unsafe { pattern.SetValue(&value) } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + } + NativeDialogAction::Select => { + let pattern: IUIAutomationSelectionItemPattern = unsafe { + cached + .element + .GetCurrentPatternAs(UIA_SelectionItemPatternId) + } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + unsafe { pattern.Select() } + .map_err(|error| self.interaction_error(cached.dialog_window, &error))?; + } + } + + let references_invalidated = !matches!( + request.action, + NativeDialogAction::SetValue | NativeDialogAction::SetPaths + ); + if references_invalidated { + self.elements + .retain(|_, item| item.dialog_window != cached.dialog_window); + } + + Ok(NativeDialogInteractionResult { + action: request.action, + element_ref: request.element_ref, + references_invalidated, + }) + } + + fn interaction_error(&mut self, dialog_window: HWND, error: &windows::core::Error) -> String { + self.elements + .retain(|_, cached| cached.dialog_window != dialog_window); + let _ = error; + STALE_REFERENCE_ERROR.to_string() + } + + fn purge_expired(&mut self) { + let now = Instant::now(); + self.elements.retain(|_, cached| cached.expires_at > now); + } +} + +fn supported_actions( + element: &IUIAutomationElement, + control_type: i32, + automation_id: &str, +) -> Vec { + let mut actions = Vec::with_capacity(4); + if unsafe { element.GetCurrentPatternAs::(UIA_InvokePatternId) } + .is_ok() + { + actions.push(NativeDialogAction::Invoke); + } + if matches!( + control_type, + value if value == UIA_EditControlTypeId.0 || value == UIA_ComboBoxControlTypeId.0 + ) { + if let Ok(pattern) = + unsafe { element.GetCurrentPatternAs::(UIA_ValuePatternId) } + { + let read_only = unsafe { pattern.CurrentIsReadOnly() } + .map(|value| value.as_bool()) + .unwrap_or(true); + if !read_only { + actions.push(NativeDialogAction::SetValue); + if control_type == UIA_EditControlTypeId.0 + && is_file_name_automation_id(automation_id) + { + actions.push(NativeDialogAction::SetPaths); + } + } + } + } + if unsafe { + element.GetCurrentPatternAs::(UIA_SelectionItemPatternId) + } + .is_ok() + { + actions.push(NativeDialogAction::Select); + } + actions +} + +fn infer_dialog_kind(controls: &[NativeDialogControl]) -> &'static str { + if controls.iter().any(|control| { + matches!(control.control_type.as_str(), "edit" | "combobox") + && control + .supported_actions + .contains(&NativeDialogAction::SetValue) + }) { + "file" + } else if controls + .iter() + .any(|control| control.control_type == "button") + { + "message" + } else { + "unknown" + } +} + +fn infer_semantic_role( + control_type: i32, + automation_id: &str, + actions: &[NativeDialogAction], +) -> Option { + if actions == [NativeDialogAction::Invoke] + && automation_id == "1" + && !is_file_system_item_control_type(control_type) + { + return Some("accept".to_string()); + } + if actions == [NativeDialogAction::Invoke] + && automation_id == "2" + && !is_file_system_item_control_type(control_type) + { + return Some("cancel".to_string()); + } + let navigation_role = match automation_id { + "NavigationButton" => Some("navigationMenu"), + "40960" => Some("back"), + "40961" => Some("forward"), + "40962" | "40966" => Some("up"), + "41477" | "Breadcrumb Parent" => Some("addressBar"), + "SearchBox" => Some("search"), + "NamespaceTreeControl" | "TreeView" => Some("navigationTree"), + _ => None, + }; + if let Some(role) = navigation_role { + return Some(role.to_string()); + } + + if control_type == UIA_ButtonControlTypeId.0 { + if automation_id == "CommandButton_1" { + return Some("primary".to_string()); + } + if automation_id == "CommandButton_2" { + return Some("secondary".to_string()); + } + let numeric_id = automation_id + .strip_prefix("Button") + .unwrap_or(automation_id) + .parse::() + .ok(); + let role = match numeric_id { + Some(1) => "accept", + Some(2) => "cancel", + Some(3) => "abort", + Some(4) => "retry", + Some(5) => "ignore", + Some(6) => "yes", + Some(7) => "no", + Some(8) => "close", + Some(9) => "help", + Some(10) => "tryAgain", + Some(11) => "continue", + _ => "button", + }; + return Some(role.to_string()); + } + + if actions.contains(&NativeDialogAction::SetValue) { + let is_file_name_control = is_file_name_automation_id(automation_id); + let role = if is_file_name_control && control_type == UIA_EditControlTypeId.0 { + "fileName" + } else if is_file_name_control { + "fileNameHost" + } else { + "editable" + }; + return Some(role.to_string()); + } + + if actions.contains(&NativeDialogAction::Select) { + let role = if control_type == UIA_TreeItemControlTypeId.0 { + "navigationTreeItem" + } else if is_file_system_item_control_type(control_type) { + "fileSystemEntry" + } else { + "selectable" + }; + return Some(role.to_string()); + } + + if automation_id == "1001" { + return Some("currentLocation".to_string()); + } + + None +} + +fn is_file_name_automation_id(automation_id: &str) -> bool { + matches!( + automation_id, + "1001" | "1148" | "1152" | "FileNameControlHost" + ) +} + +fn is_file_system_item_control_type(control_type: i32) -> bool { + matches!( + control_type, + value if value == UIA_ListItemControlTypeId.0 || value == UIA_DataItemControlTypeId.0 + ) +} + +fn is_supported_file_dialog_control(control: &NativeDialogControl) -> bool { + control.element_ref.is_some() + || matches!( + control.semantic_role.as_deref(), + Some( + "currentLocation" + | "addressBar" + | "navigationTree" + | "fileSystemEntry" + | "navigationTreeItem" + ) + ) +} + +fn control_type_name(control_type: i32) -> &'static str { + match control_type { + value if value == UIA_ButtonControlTypeId.0 => "button", + value if value == UIA_CheckBoxControlTypeId.0 => "checkbox", + value if value == UIA_ComboBoxControlTypeId.0 => "combobox", + value if value == UIA_DataItemControlTypeId.0 => "dataItem", + value if value == UIA_EditControlTypeId.0 => "edit", + value if value == UIA_ListItemControlTypeId.0 => "listItem", + value if value == UIA_PaneControlTypeId.0 => "pane", + value if value == UIA_RadioButtonControlTypeId.0 => "radioButton", + value if value == UIA_TextControlTypeId.0 => "text", + value if value == UIA_TreeItemControlTypeId.0 => "treeItem", + value if value == UIA_WindowControlTypeId.0 => "window", + _ => "custom", + } +} + +fn bounded_bstr(result: windows::core::Result) -> String { + let value = result.map(|value| value.to_string()).unwrap_or_default(); + value.chars().take(MAX_TEXT_CHARS).collect() +} + +fn validate_absolute_path(value: &str) -> Result<(), String> { + if value.contains(['\0', '"']) + || value.chars().count() > MAX_PATH_CHARS + || !Path::new(value).is_absolute() + { + return Err("setValue requires a complete absolute path".to_string()); + } + Ok(()) +} + +fn format_multi_select_paths(paths: &[String]) -> Result { + if paths.is_empty() || paths.len() > MAX_MULTI_SELECT_PATHS { + return Err("setPaths requires between 1 and 100 absolute file paths".to_string()); + } + + for path in paths { + if validate_absolute_path(path).is_err() || !Path::new(path).is_file() { + return Err("setPaths requires existing absolute file paths".to_string()); + } + } + + let formatted = paths + .iter() + .map(|path| format!("\"{path}\"")) + .collect::>() + .join(" "); + if formatted.chars().count() > MAX_PATH_CHARS { + return Err("setPaths combined path data is too long".to_string()); + } + Ok(formatted) +} + +fn hresult_error(context: &str, error: &windows::core::Error) -> String { + format!("{context} failed (HRESULT 0x{:08X})", error.code().0 as u32) +} + +fn hwnd_from_usize(value: usize) -> HWND { + HWND(value as *mut core::ffi::c_void) +} + +fn hwnd_as_usize(value: HWND) -> usize { + value.0 as usize +} + +struct EnumContext { + process_id: u32, + owner_window: HWND, + windows: Vec, +} + +#[derive(Clone, Copy)] +struct OwnedDialogWindow { + window: HWND, + immediate_owner: Option, + owner_depth: usize, +} + +unsafe extern "system" fn enum_dialog_window(hwnd: HWND, context: LPARAM) -> BOOL { + let context = &mut *(context.0 as *mut EnumContext); + if context.windows.len() >= MAX_ENUMERATED_DIALOGS + || !IsWindowVisible(hwnd).as_bool() + || hwnd == context.owner_window + { + return BOOL(1); + } + + let mut process_id = 0; + GetWindowThreadProcessId(hwnd, Some(&mut process_id)); + if process_id == context.process_id { + if let Some((owner_depth, immediate_owner)) = + owner_chain_depth(hwnd, context.owner_window, context.process_id) + { + context.windows.push(OwnedDialogWindow { + window: hwnd, + immediate_owner: Some(immediate_owner), + owner_depth, + }); + } + } + BOOL(1) +} + +fn enumerate_owned_dialog_windows( + owner_window: HWND, + process_id: u32, +) -> Result, String> { + let mut context = EnumContext { + process_id, + owner_window, + windows: Vec::new(), + }; + unsafe { + EnumWindows( + Some(enum_dialog_window), + LPARAM((&mut context as *mut EnumContext) as isize), + ) + } + .map_err(|error| hresult_error("Native dialog window enumeration", &error))?; + context.windows.sort_by(|left, right| { + right + .owner_depth + .cmp(&left.owner_depth) + .then_with(|| hwnd_as_usize(left.window).cmp(&hwnd_as_usize(right.window))) + }); + Ok(context.windows) +} + +fn is_owned_dialog_window(dialog_window: HWND, owner_window: HWND, process_id: u32) -> bool { + if !unsafe { IsWindow(Some(dialog_window)) }.as_bool() + || !unsafe { IsWindowVisible(dialog_window) }.as_bool() + { + return false; + } + + let mut actual_process_id = 0; + unsafe { GetWindowThreadProcessId(dialog_window, Some(&mut actual_process_id)) }; + if actual_process_id != process_id { + return false; + } + + owner_chain_depth(dialog_window, owner_window, process_id).is_some() +} + +fn owner_chain_depth( + dialog_window: HWND, + root_owner: HWND, + expected_process_id: u32, +) -> Option<(usize, HWND)> { + let mut current = dialog_window; + let mut immediate_owner = None; + let mut visited = HashSet::with_capacity(MAX_OWNER_CHAIN_DEPTH + 1); + visited.insert(hwnd_as_usize(dialog_window)); + + for depth in 1..=MAX_OWNER_CHAIN_DEPTH { + let owner = unsafe { GetWindow(current, GW_OWNER) }.ok()?; + if owner.0.is_null() || !visited.insert(hwnd_as_usize(owner)) { + return None; + } + immediate_owner.get_or_insert(owner); + + let mut owner_process_id = 0; + unsafe { GetWindowThreadProcessId(owner, Some(&mut owner_process_id)) }; + if owner_process_id != expected_process_id { + return None; + } + if owner == root_owner { + return immediate_owner.map(|immediate| (depth, immediate)); + } + current = owner; + } + None +} + +fn is_within_security_boundary( + actual_process_id: u32, + expected_process_id: u32, + actual_owner: HWND, + expected_owner: HWND, +) -> bool { + actual_process_id == expected_process_id && actual_owner == expected_owner +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn security_boundary_rejects_other_processes_and_owners() { + let expected_owner = hwnd_from_usize(100); + assert!(is_within_security_boundary( + 42, + 42, + expected_owner, + expected_owner + )); + assert!(!is_within_security_boundary( + 7, + 42, + expected_owner, + expected_owner + )); + assert!(!is_within_security_boundary( + 42, + 42, + hwnd_from_usize(101), + expected_owner + )); + } + + #[test] + fn semantic_roles_do_not_depend_on_localized_button_names() { + assert_eq!( + infer_semantic_role( + UIA_ButtonControlTypeId.0, + "6", + &[NativeDialogAction::Invoke] + ), + Some("yes".to_string()) + ); + assert_eq!( + infer_semantic_role( + UIA_ButtonControlTypeId.0, + "CommandButton_2", + &[NativeDialogAction::Invoke] + ), + Some("secondary".to_string()) + ); + assert_eq!( + infer_semantic_role(12345, "1", &[NativeDialogAction::Invoke]), + Some("accept".to_string()) + ); + } + + #[test] + fn relative_paths_fail_without_echoing_sensitive_values() { + let sensitive_value = "private-file.txt"; + let error = validate_absolute_path(sensitive_value).unwrap_err(); + assert!(!error.contains(sensitive_value)); + assert!(error.contains("absolute path")); + } + + #[test] + fn multi_select_paths_are_quoted_without_echoing_invalid_values() { + let fixture_directory = + std::env::temp_dir().join(format!("tauri-mcp-path-test-{}", Uuid::new_v4())); + std::fs::create_dir_all(&fixture_directory).unwrap(); + let first = fixture_directory.join("first.txt"); + let second = fixture_directory.join("second file.txt"); + std::fs::write(&first, b"first").unwrap(); + std::fs::write(&second, b"second").unwrap(); + + let paths = vec![ + first.to_string_lossy().into_owned(), + second.to_string_lossy().into_owned(), + ]; + let formatted = format_multi_select_paths(&paths).unwrap(); + assert_eq!(formatted, format!("\"{}\" \"{}\"", paths[0], paths[1])); + + let sensitive_value = fixture_directory.join("missing.txt"); + let sensitive_text = sensitive_value.to_string_lossy().into_owned(); + let error = format_multi_select_paths(std::slice::from_ref(&sensitive_text)).unwrap_err(); + assert!(!error.contains(sensitive_text.as_str())); + + std::fs::remove_dir_all(fixture_directory).unwrap(); + } + + #[test] + fn navigation_roles_use_stable_automation_ids() { + assert_eq!( + infer_semantic_role( + UIA_ButtonControlTypeId.0, + "40960", + &[NativeDialogAction::Invoke] + ), + Some("back".to_string()) + ); + assert_eq!( + infer_semantic_role(UIA_PaneControlTypeId.0, "41477", &[]), + Some("addressBar".to_string()) + ); + assert_eq!( + infer_semantic_role( + UIA_ListItemControlTypeId.0, + "unlocalized-item-id", + &[NativeDialogAction::Select] + ), + Some("fileSystemEntry".to_string()) + ); + } +} diff --git a/packages/tauri-plugin-mcp-bridge/src/websocket.rs b/packages/tauri-plugin-mcp-bridge/src/websocket.rs index e43575f..388b846 100644 --- a/packages/tauri-plugin-mcp-bridge/src/websocket.rs +++ b/packages/tauri-plugin-mcp-bridge/src/websocket.rs @@ -6,10 +6,14 @@ use crate::commands::{self, resolve_window_with_context, ScriptExecutor, WindowContext}; use crate::logging::{mcp_log_error, mcp_log_info}; +use crate::native_dialog::{ + bounded_timeout, InteractRequest, NativeDialogAction, NativeDialogAutomation, SnapshotRequest, +}; use crate::script_registry::{ScriptEntry, ScriptType, SharedScriptRegistry}; use futures_util::{SinkExt, StreamExt}; use serde_json::{self, Value}; use std::net::SocketAddr; +use std::time::Duration; use tauri::{AppHandle, Manager, Runtime, WebviewWindow}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{broadcast, mpsc}; @@ -360,6 +364,155 @@ async fn handle_capture_screenshot( } } +#[cfg(target_os = "windows")] +fn native_owner_handle(window: &WebviewWindow) -> Result { + window + .hwnd() + .map(|hwnd| hwnd.0 as usize) + .map_err(|_| "Failed to resolve the targeted Tauri window handle".to_string()) +} + +#[cfg(not(target_os = "windows"))] +fn native_owner_handle(_window: &WebviewWindow) -> Result { + Ok(0) +} + +fn native_dialog_timeout(args: &Value) -> Duration { + let requested = args + .get("timeoutMs") + .and_then(|value| value.as_u64()) + .unwrap_or(2_000) + .max(100); + bounded_timeout(Duration::from_millis(requested)) +} + +/// Handles bounded native dialog discovery on the dedicated UI Automation thread. +async fn handle_native_dialog_snapshot( + app: &AppHandle, + id: &str, + args: &Value, +) -> Value { + let Some(scope_id) = args.get("scopeId").and_then(|value| value.as_str()) else { + return error_response(id, "Missing native dialog session scope"); + }; + let window_label = args + .get("windowLabel") + .and_then(|value| value.as_str()) + .map(str::to_string); + let resolved = match resolve_window_with_context(app, window_label) { + Ok(resolved) => resolved, + Err(error) => return error_response(id, error), + }; + let owner_window = match native_owner_handle(&resolved.window) { + Ok(owner_window) => owner_window, + Err(error) => return error_response(id, error), + }; + let context = resolved.context; + let automation = app.state::().inner().clone(); + let request = SnapshotRequest { + process_id: std::process::id(), + owner_window, + scope_id: scope_id.to_string(), + min_owner_depth: args + .get("minOwnerDepth") + .and_then(|value| value.as_u64()) + .unwrap_or(1) + .clamp(1, 8) as usize, + timeout: native_dialog_timeout(args), + }; + + match tokio::task::spawn_blocking(move || automation.snapshot(request)).await { + Ok(Ok(snapshot)) => serde_json::json!({ + "id": id, + "success": true, + "data": snapshot, + "windowContext": context + }), + Ok(Err(error)) => serde_json::json!({ + "id": id, + "success": false, + "error": error, + "windowContext": context + }), + Err(_) => error_response(id, "Native dialog automation worker failed"), + } +} + +/// Handles semantic UI Automation patterns for a previously discovered element. +async fn handle_native_dialog_interact( + app: &AppHandle, + id: &str, + args: &Value, +) -> Value { + let Some(scope_id) = args.get("scopeId").and_then(|value| value.as_str()) else { + return error_response(id, "Missing native dialog session scope"); + }; + let Some(element_ref) = args.get("elementRef").and_then(|value| value.as_str()) else { + return error_response(id, "Missing native dialog elementRef"); + }; + let Some(action_name) = args.get("action").and_then(|value| value.as_str()) else { + return error_response(id, "Missing native dialog action"); + }; + let action = match NativeDialogAction::parse(action_name) { + Ok(action) => action, + Err(error) => return error_response(id, error), + }; + let paths = match args.get("paths") { + None => None, + Some(Value::Array(values)) if values.iter().all(Value::is_string) => Some( + values + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(), + ), + Some(_) => return error_response(id, "Native dialog paths must be an array of strings"), + }; + let window_label = args + .get("windowLabel") + .and_then(|value| value.as_str()) + .map(str::to_string); + let resolved = match resolve_window_with_context(app, window_label) { + Ok(resolved) => resolved, + Err(error) => return error_response(id, error), + }; + let owner_window = match native_owner_handle(&resolved.window) { + Ok(owner_window) => owner_window, + Err(error) => return error_response(id, error), + }; + let context = resolved.context; + let automation = app.state::().inner().clone(); + let request = InteractRequest { + process_id: std::process::id(), + owner_window, + scope_id: scope_id.to_string(), + element_ref: element_ref.to_string(), + action, + value: args + .get("value") + .and_then(|value| value.as_str()) + .map(str::to_string), + paths, + timeout: native_dialog_timeout(args), + }; + + match tokio::task::spawn_blocking(move || automation.interact(request)).await { + Ok(Ok(result)) => serde_json::json!({ + "id": id, + "success": true, + "data": result, + "windowContext": context + }), + Ok(Err(error)) => serde_json::json!({ + "id": id, + "success": false, + "error": error, + "windowContext": context + }), + Err(_) => error_response(id, "Native dialog automation worker failed"), + } +} + /// Handles the resize_window command. async fn handle_resize_window(app: &AppHandle, id: &str, args: &Value) -> Value { let width = args.get("width").and_then(|v| v.as_u64()).map(|w| w as u32); @@ -579,6 +732,20 @@ async fn dispatch_command(app: &AppHandle, command: &Value) -> Va } } "capture_native_screenshot" => handle_capture_screenshot(app, id, args).await, + "native_dialog_snapshot" => { + if let Some(args) = args { + handle_native_dialog_snapshot(app, id, args).await + } else { + error_response(id, "Missing args for native_dialog_snapshot") + } + } + "native_dialog_interact" => { + if let Some(args) = args { + handle_native_dialog_interact(app, id, args).await + } else { + error_response(id, "Missing args for native_dialog_interact") + } + } "resize_window" => { if let Some(args) = args { handle_resize_window(app, id, args).await @@ -687,7 +854,9 @@ async fn handle_connection( let response = dispatch_command(&app, &command).await; let _ = response_tx.send(response.to_string()); } else { - eprintln!("Failed to parse command: {text}"); + // Command payloads can contain paths and other private data. + // Never include them in plugin logs. + eprintln!("Failed to parse WebSocket command"); } } Ok(Message::Close(_)) => { diff --git a/packages/test-app/index.html b/packages/test-app/index.html index f6caa20..f192dd6 100644 --- a/packages/test-app/index.html +++ b/packages/test-app/index.html @@ -36,6 +36,8 @@

Dialogs

+ +

diff --git a/packages/test-app/src/main.ts b/packages/test-app/src/main.ts index baa0fab..2d99c39 100644 --- a/packages/test-app/src/main.ts +++ b/packages/test-app/src/main.ts @@ -56,6 +56,22 @@ window.addEventListener('DOMContentLoaded', () => { }); }); + document.querySelector('#dialog-open-multiple')?.addEventListener('click', () => { + open({ multiple: true, directory: false }) + .then((files) => { + const result = Array.isArray(files) ? files.join('|') : files; + + showDialogResult(dialogResult, `Multiple open result: ${result ?? 'Cancelled'}`); + }); + }); + + document.querySelector('#dialog-folder')?.addEventListener('click', () => { + open({ multiple: false, directory: true }) + .then((folder) => { + showDialogResult(dialogResult, `Folder result: ${folder ?? 'Cancelled'}`); + }); + }); + document.querySelector('#dialog-save')?.addEventListener('click', () => { save({ defaultPath: 'untitled.txt' }) .then((path) => {