From 1c39f700ce45bec4dfd1a416959e15e25cab0181 Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:45:54 -0400 Subject: [PATCH] feat(server): tell agents about the Threadlines browser panel Claude never learned the browser panel existed. The SDK defers MCP tool descriptions behind tool search, so Claude saw eighteen tool names and none of their text, and nothing in its system prompt mentioned Threadlines or the panel. Codex had a note, but it only said what "our browser" meant; it never said to prefer the panel. Agents defaulted to Chrome DevTools or Playwright unless told otherwise. Load the browser MCP server's tools into Claude's prompt (alwaysLoad), append a browser note to Claude's system prompt, and rewrite the note both providers share so it says when to use the panel and why, naming each provider's rival tooling. Tool descriptions and the no-host error now use the same words: "browser panel", and "no desktop window is showing this thread" instead of asking the user to open a panel that opens itself. --- apps/server/src/mcp/browserTools.ts | 8 +-- .../provider/CodexDeveloperInstructions.ts | 32 +--------- .../src/provider/Layers/ClaudeAdapter.ts | 13 +++- .../src/provider/previewPanelInstructions.ts | 63 +++++++++++++++++++ packages/contracts/src/previewAutomation.ts | 2 +- 5 files changed, 83 insertions(+), 35 deletions(-) create mode 100644 apps/server/src/provider/previewPanelInstructions.ts diff --git a/apps/server/src/mcp/browserTools.ts b/apps/server/src/mcp/browserTools.ts index 1750edf41..796a74137 100644 --- a/apps/server/src/mcp/browserTools.ts +++ b/apps/server/src/mcp/browserTools.ts @@ -64,7 +64,7 @@ export const BrowserSnapshotTool = readsOnly( export const BrowserScreenshotTool = readsOnly( Tool.make("browser_screenshot", { description: - "See what the page in the Threadlines preview panel currently looks like. Use this for anything visual -- layout, spacing, colour, whether something is actually on screen -- and use browser_snapshot instead when you need to act on an element or read an error.", + "See what the page in the Threadlines browser panel currently looks like. Use this for anything visual -- layout, spacing, colour, whether something is actually on screen -- and use browser_snapshot instead when you need to act on an element or read an error.", parameters: PreviewAutomationEmptyInputSchema, success: PreviewAutomationScreenshotSchema, failure: PreviewAutomationErrorSchema, @@ -75,7 +75,7 @@ export const BrowserScreenshotTool = readsOnly( export const BrowserStatusTool = readsOnly( Tool.make("browser_status", { description: - "Where the Threadlines preview panel is right now: its URL, title, size, and whether it is still loading. Cheap; use it to confirm a navigation landed rather than taking a whole snapshot.", + "Where the Threadlines browser panel is right now: its URL, title, size, and whether it is still loading. Cheap; use it to confirm a navigation landed rather than taking a whole snapshot.", parameters: PreviewAutomationEmptyInputSchema, success: PreviewAutomationStatusSchema, failure: PreviewAutomationErrorSchema, @@ -86,7 +86,7 @@ export const BrowserStatusTool = readsOnly( export const BrowserTabsTool = readsOnly( Tool.make("browser_tabs", { description: - "Every page open in the Threadlines preview panel, including each stable tab id, with the user's visible tab and your pinned tab marked. Use a tab id on any browser tool to act there without changing what the user sees.", + "Every page open in the Threadlines browser panel, including each stable tab id, with the user's visible tab and your pinned tab marked. Use a tab id on any browser tool to act there without changing what the user sees.", success: PreviewAutomationTabsSchema, failure: PreviewAutomationErrorSchema, dependencies, @@ -129,7 +129,7 @@ export const BrowserSelectTabTool = changesThePage( export const BrowserNavigateTool = changesThePage( Tool.make("browser_navigate", { description: - "Go to a URL in the Threadlines preview panel the user is watching. This replaces what they are looking at, so navigate when you need to be somewhere else, not to reset state you can reach by clicking. Local and private addresses always work, but a site outside the project's approved list needs the user's approval: if this is refused for that reason, wait for them to allow it in the browser panel and then navigate again.", + "Go to a URL in the Threadlines browser panel the user is watching. This replaces what they are looking at, so navigate when you need to be somewhere else, not to reset state you can reach by clicking. Local and private addresses always work, but a site outside the project's approved list needs the user's approval: if this is refused for that reason, wait for them to allow it in the browser panel and then navigate again.", parameters: PreviewAutomationNavigateInputSchema, success: PreviewAutomationStatusSchema, failure: PreviewAutomationErrorSchema, diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index dc7e36aa0..731586de0 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -133,32 +133,6 @@ Use the \`request_user_input\` tool only when it is listed in the available tool In Default mode, strongly prefer making reasonable assumptions and executing the user's request rather than stopping to ask questions. If you absolutely must ask a question because the answer cannot be discovered from local context and a reasonable assumption would be risky, ask the user directly with a concise plain-text question. Never write a multiple choice question as a textual assistant message. `; -/** - * Which browser the user means. - * - * Codex ships a bundled browser plugin whose skill claims the phrase "in-app - * browser" as one of its own aliases, and instructs the model outright: do not - * use external MCP browser-control tools for this surface. Asked to click - * something "in our browser", the model followed that skill, found no ChatGPT - * in-app browser available, silently fell back to the user's real Chrome, and - * reported on a signed-in personal session -- while telling the user it was - * using the in-app browser. It was not being careless; it was obeying a skill - * that had claimed the words. - * - * A tool description cannot answer that: the model commits to the skill before - * it reads our tool list. Developer instructions can, because developer-role - * text outranks a skill file, which reaches the model as tool output. - * - * Deliberately narrow. It says which surface the user is looking at and how to - * reach it, and leaves Codex's plugins alone for what they are for -- asked for - * Chrome, the model should still drive Chrome. - */ -export const CODEX_PREVIEW_PANEL_DEVELOPER_INSTRUCTIONS = ` -This session runs inside Threadlines. The user is looking at a live page in Threadlines' embedded preview panel. - -When the user says "our browser", "the browser", "the in-app browser", or "the page", they mean that panel. They do not mean Chrome, and they do not mean Codex's own in-app browser; those are different surfaces and neither can see what the user is looking at. - -The panel is reachable only through the MCP tools \`mcp__threadlines_browser__*\` (browser_snapshot, browser_click, browser_navigate, browser_type, and the rest). Start with \`mcp__threadlines_browser__browser_snapshot\`. - -Do not use the Browser plugin, the Chrome plugin, or any other browser automation for the user's panel. Use those only when the user explicitly asks for Chrome or for a browser outside Threadlines. -`; +/** Which browser the user means, and why to reach for it. Shared with Claude; + * see previewPanelInstructions.ts for the reasoning. */ +export { CODEX_PREVIEW_PANEL_DEVELOPER_INSTRUCTIONS } from "./previewPanelInstructions.ts"; diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index be5a4b73e..22a417581 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -73,6 +73,7 @@ import { isLinkedWorktreeCheckout, missingWorkingDirectoryDetail, } from "../../vcs/CheckoutPresence.ts"; +import { CLAUDE_PREVIEW_PANEL_INSTRUCTIONS } from "../previewPanelInstructions.ts"; import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -6482,6 +6483,13 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( type: "http", url: mcpEndpointUrl(serverConfig.port), headers: { Authorization: `Bearer ${browserCredential}` }, + // In the prompt rather than deferred behind tool search. Deferred, + // the model sees eighteen tool names and none of the sentences in + // browserTools.ts, and reaches for whichever browser has a blurb in + // context instead. About 3k tokens, all in the cached prefix. The + // SDK waits for the server before the first turn; it is this + // process, so the wait is nothing. + alwaysLoad: true, }, }, ...(apiModelId ? { model: apiModelId } : {}), @@ -6489,7 +6497,10 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( systemPrompt: { type: "preset", preset: "claude_code", - ...(runsInManagedWorktree ? { append: MANAGED_WORKTREE_INSTRUCTION } : {}), + append: [ + CLAUDE_PREVIEW_PANEL_INSTRUCTIONS, + ...(runsInManagedWorktree ? [MANAGED_WORKTREE_INSTRUCTION] : []), + ].join("\n\n"), }, settingSources: [...CLAUDE_SETTING_SOURCES], // SDK 0.3.233 dropped the todo/task tools from the default tool diff --git a/apps/server/src/provider/previewPanelInstructions.ts b/apps/server/src/provider/previewPanelInstructions.ts new file mode 100644 index 000000000..03158f898 --- /dev/null +++ b/apps/server/src/provider/previewPanelInstructions.ts @@ -0,0 +1,63 @@ +/** + * The browser panel, introduced to a model. + * + * Two jobs, in this order. First, sell it: an agent that needs to look at a web + * page reaches for whatever browser it already knows -- Chrome DevTools, + * Playwright, a bundled browser plugin -- unless its instructions say the panel + * beside the chat is the one to use, and why. Second, disambiguate: Codex ships + * a browser plugin whose skill claims the phrase "in-app browser" as one of its + * own aliases and tells the model outright not to use external MCP browser + * tools for it. Asked to click something "in our browser", the model followed + * that skill, found no ChatGPT in-app browser, silently fell back to the user's + * real Chrome, and reported on a signed-in personal session -- while telling + * the user it was using the in-app browser. It was not being careless; it was + * obeying a skill that had claimed the words. + * + * A tool description can do neither job. The model commits to a skill before + * it reads the tool list, and Claude does not see MCP tool descriptions at all + * until it searches for them. System-level text can, because developer-role + * instructions outrank a skill file, which reaches the model as tool output. + * + * One body with the rivals named per provider, so both providers hear the same + * story about the same panel. Kept short on purpose: every line here competes + * with the user's actual task on every turn. + */ + +interface PreviewPanelRivals { + /** What the user does not mean by "the in-app browser", for this provider. */ + readonly notThePanel: string; + /** The browser tooling this provider would otherwise reach for. */ + readonly doNotUse: string; +} + +const RIVALS: Record<"claude" | "codex", PreviewPanelRivals> = { + claude: { + notThePanel: "Claude in Chrome", + doNotUse: + "Do not use Claude in Chrome, the Chrome DevTools MCP tools, Playwright, or any other browser automation for the user's panel.", + }, + codex: { + notThePanel: "Codex's own in-app browser", + doNotUse: + "Do not use the Browser plugin, the Chrome plugin, or any other browser automation for the user's panel.", + }, +}; + +export function buildPreviewPanelInstructions(provider: "claude" | "codex"): string { + const rivals = RIVALS[provider]; + return ` +This session runs inside Threadlines, a desktop workspace for coding agents. A browser panel sits beside this chat. Use it for any work that involves looking at a web page: opening a local dev server, checking a layout or a style change, reproducing a UI bug, reading console errors, or filling in a form. Prefer it over Chrome, headless browsers, and other browser automation, because the user watches every step in the panel, sees where you click, and can mark up the page and send the marks back to you. A browser you open anywhere else shows them nothing. + +The panel is reachable only through the MCP tools \`mcp__threadlines_browser__*\` (browser_snapshot, browser_screenshot, browser_navigate, browser_click, browser_type, and the rest). Start with \`mcp__threadlines_browser__browser_snapshot\`. You do not need to ask the user to open the panel: your first browser tool call opens it. If a call reports that no browser is connected, the user is not viewing this thread in the Threadlines desktop app; say so before reaching for any other browser. + +When the user says "our browser", "the browser", "the in-app browser", "the preview", or "the page", they mean that panel. They do not mean Chrome, and they do not mean ${rivals.notThePanel}; those are different surfaces and none of them can see what the user is looking at. + +${rivals.doNotUse} Use those only when the user explicitly asks for Chrome or for a browser outside Threadlines. +`; +} + +/** Appended to Claude's system prompt on every session. */ +export const CLAUDE_PREVIEW_PANEL_INSTRUCTIONS = buildPreviewPanelInstructions("claude"); + +/** Appended to Codex's developer instructions on every turn. */ +export const CODEX_PREVIEW_PANEL_DEVELOPER_INSTRUCTIONS = buildPreviewPanelInstructions("codex"); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 577ad81bd..c71855b91 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -120,7 +120,7 @@ export class PreviewAutomationNoHostError extends Schema.TaggedError