diff --git a/.changeset/openapi-transport-unreachable.md b/.changeset/openapi-transport-unreachable.md new file mode 100644 index 0000000000..8921b9b31f --- /dev/null +++ b/.changeset/openapi-transport-unreachable.md @@ -0,0 +1,6 @@ +--- +"executor": patch +"@executor-js/plugin-openapi": patch +--- + +OpenAPI tools that cannot reach the upstream server now return an `upstream_unreachable` error with an actionable network message instead of `Internal tool error [id]`. diff --git a/e2e/scenarios/openapi-unreachable-artifact.test.ts b/e2e/scenarios/openapi-unreachable-artifact.test.ts new file mode 100644 index 0000000000..179ca5048d --- /dev/null +++ b/e2e/scenarios/openapi-unreachable-artifact.test.ts @@ -0,0 +1,226 @@ +// Cross-target: an artifact whose OpenAPI query cannot reach its upstream gets +// an actionable network error, not the opaque defect mask. This walks the real +// path from a saved artifact through the nested shell, execute-action, sandbox, +// OpenAPI transport, and back into ArtifactError. +import { randomBytes } from "node:crypto"; +import { createServer } from "node:http"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import type { Page } from "playwright"; +import { composePluginApi } from "@executor-js/api/server"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { ConnectionName, IntegrationSlug, type ArtifactId } from "@executor-js/sdk/shared"; + +import { scenario } from "../src/scenario"; +import { Api, Browser, Mcp, Target } from "../src/services"; +import { visit } from "../src/surfaces/browser"; +import type { McpSession } from "../src/surfaces/mcp"; + +const api = composePluginApi([openApiHttpPlugin()] as const); + +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; + +type DroppingUpstream = { + readonly url: string; + readonly requests: () => number; + readonly close: () => void; +}; + +// Accept the request, then drop the socket before sending response headers. +// This produces a real transport failure without relying on a hardcoded or +// temporarily-unused port. +const serveDroppingUpstream = () => + Effect.acquireRelease( + Effect.callback((resume) => { + let hits = 0; + const server = createServer((_request, response) => { + hits += 1; + response.destroy(); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + resume( + Effect.succeed({ + url: `http://127.0.0.1:${port}`, + requests: () => hits, + close: () => { + server.close(); + server.closeAllConnections(); + }, + }), + ); + }); + }), + (server) => Effect.sync(server.close), + ); + +const unreachableSpec = (baseUrl: string): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Unreachable API", version: "1.0.0" }, + servers: [{ url: baseUrl }], + paths: { + "/things": { + get: { + tags: ["things"], + operationId: "listThings", + summary: "List things", + responses: { + "200": { + description: "Things", + content: { + "application/json": { + schema: { type: "array", items: { type: "object" } }, + }, + }, + }, + }, + }, + }, + }, + }); + +const createConnectionCode = (slug: string) => ` +const created = await tools.executor.coreTools.connections.create({ + owner: "org", + name: "public", + integration: ${JSON.stringify(slug)}, + template: "none", +}); +return JSON.stringify(created.ok ? { ok: true } : { ok: false, error: created.error }); +`; + +const executeApproved = (session: McpSession, code: string) => + Effect.gen(function* () { + let result = yield* session.call("execute", { code }); + let guard = 0; + while (result.text.includes("executionId:") && guard < 10) { + result = yield* session.approvePaused(result.text); + guard += 1; + } + expect(result.ok, `execute completed (got: ${result.text.slice(0, 400)})`).toBe(true); + return result.text; + }); + +const artifactSource = (slug: string) => ` +function App() { + const query = useQuery(tools.${slug}.things.listThings.queryOptions({})); + const result = query.data; + return ( +
+

Upstream status

+
+ {query.isLoading ? ( + + ) : query.error ? ( + + ) : result?.ok === false ? ( + + ) : ( +

Unexpected upstream success

+ )} +
+
+ ); +} +`; + +const structuredOf = (result: { readonly raw: unknown }): Record => + ((result.raw as { structuredContent?: Record }).structuredContent ?? + {}) as Record; + +const artifactContent = (page: Page) => + page.frameLocator('[data-testid="artifact-shell-frame"]').frameLocator("iframe"); + +scenario( + "Artifacts ยท an unreachable OpenAPI host shows actionable retry guidance instead of an internal error", + { timeout: 180_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const browser = yield* Browser; + const mcp = yield* Mcp; + const { client: makeClient } = yield* Api; + + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const session = mcp.session(identity); + const upstream = yield* serveDroppingUpstream(); + const slug = unique("unreachable"); + const title = `Unreachable upstream ${randomBytes(4).toString("hex")}`; + let artifactId: ArtifactId | undefined; + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: unreachableSpec(upstream.url) }, + slug, + baseUrl: upstream.url, + }, + }); + + const created = yield* executeApproved(session, createConnectionCode(slug)); + expect(created, `the no-auth connection was created: ${created}`).toContain('"ok":true'); + + const rendered = yield* session.call("create-artifact", { + code: artifactSource(slug), + title, + description: "Shows whether the upstream API is reachable", + connections: { [slug]: `${slug}.org.public` }, + }); + expect(rendered.ok, `create-artifact succeeded: ${rendered.text}`).toBe(true); + + const structured = structuredOf(rendered); + artifactId = structured.artifactId as ArtifactId; + expect(artifactId, "the artifact was persisted").toBeTruthy(); + + yield* browser.session(identity, async ({ page, step }) => { + await step("Open the artifact that reads from the unreachable API", async () => { + await visit(page, String(structured.url)); + await page.getByRole("heading", { name: title }).waitFor({ timeout: 20_000 }); + }); + + await step( + "The artifact explains that the upstream host could not be reached", + async () => { + const state = artifactContent(page).getByTestId("upstream-state"); + await state.locator('[data-slot="artifact-error"]').waitFor({ timeout: 30_000 }); + const message = await state.innerText(); + + expect(message, "the user gets actionable network guidance").toContain( + "Could not reach the upstream server", + ); + expect(message, "the opaque defect mask never reaches the artifact").not.toContain( + "Internal tool error", + ); + expect(message, "the request path is not leaked").not.toContain("/things"); + }, + ); + }); + + expect(upstream.requests(), "the artifact made a real upstream request").toBeGreaterThan( + 0, + ); + }), + Effect.gen(function* () { + if (artifactId !== undefined) { + yield* client.artifacts.remove({ params: { artifactId } }).pipe(Effect.ignore); + } + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(slug), + name: ConnectionName.make("public"), + }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); + }), + ); + }), + ), +); diff --git a/packages/plugins/openapi/src/sdk/backing.ts b/packages/plugins/openapi/src/sdk/backing.ts index b7a54d4f43..251822cf68 100644 --- a/packages/plugins/openapi/src/sdk/backing.ts +++ b/packages/plugins/openapi/src/sdk/backing.ts @@ -727,7 +727,16 @@ export const invokeOpenApiBackedTool = (input: { details: error.cause ?? error, }), }) - : Effect.fail(error), + : error.cause !== undefined && Option.isNone(error.statusCode) + ? Effect.succeed({ + ok: false as const, + failure: ToolResult.fail({ + code: "upstream_unreachable", + message: + "Could not reach the upstream server. Check your network and try again.", + }), + }) + : Effect.fail(error), ), ); diff --git a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts index 2089e2fdf0..f07144d4c5 100644 --- a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts +++ b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts @@ -394,9 +394,12 @@ describe("OpenAPI upstream failure modes", () => { const { baseUrl } = yield* startDroppingServer(); const { executor, address } = yield* buildExecutor(baseUrl); - const exit = yield* executor.execute(address, {}).pipe(Effect.exit); + const result = yield* executor.execute(address, {}); - expect(Exit.isFailure(exit)).toBe(true); + expect(result).toMatchObject({ + ok: false, + error: { code: "upstream_unreachable" }, + }); }), ); @@ -446,4 +449,26 @@ describe("OpenAPI upstream failure modes", () => { expect(result.data).toEqual([]); }), ); + + // Port 1 refuses immediately. The same path used to throw `Internal tool + // error [hex]` because the raw HttpClientError carries the request URL. + it.effect("connection refused returns upstream_unreachable without leaking the path", () => + Effect.gen(function* () { + const { executor, address } = yield* buildExecutor("http://127.0.0.1:1"); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { code: "upstream_unreachable" }, + }); + const failure = result as { + readonly ok: false; + readonly error: { readonly message: string }; + }; + expect(failure.error.message).toContain("Could not reach"); + expect(failure.error.message).not.toContain("Internal tool error"); + expect(failure.error.message).not.toContain("/things"); + }), + ); });