From 47b489a56708db4240ad3ef64b66fdda5fccb895 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 20:42:31 -0700 Subject: [PATCH 01/51] feat: add Foundry hosted Responses adapter Signed-off-by: Sertac Ozercan --- .../core_v1alpha1_agentruntime_foundry.yaml | 17 +- .../README.md | 2 +- examples/fibey-custom-agent-demo/README.md | 16 +- .../agent-foundry.yaml | 6 +- .../agentruntime-foundry.yaml | 16 +- .../secret-foundry.yaml | 13 +- .../fibey-custom-agent-demo/switch-backend.sh | 2 +- .../task-foundry-responses.yaml | 20 + .../tools-foundry-responses.yaml | 85 ++ examples/harness/foundry-responses/Dockerfile | 13 + examples/harness/foundry-responses/README.md | 126 ++ examples/harness/foundry-responses/main.go | 1290 +++++++++++++++++ .../harness/foundry-responses/main_test.go | 1102 ++++++++++++++ .../golden/01_initial_hosted_request.json | 3 + .../golden/02_function_call_response.json | 12 + .../golden/03_tool_call_requested_frame.json | 13 + .../golden/04_orka_continue_request.json | 20 + .../05_hosted_continuation_request.json | 12 + .../golden/06_final_message_response.json | 13 + .../golden/07_approval_declined_output.json | 6 + .../08_tool_policy_rejection_output.json | 6 + .../09_tool_execution_failure_output.json | 6 + .../golden/10_multiple_calls_response.json | 8 + examples/harness/foundry/README.md | 34 +- .../guides/bring-your-own-agent-runtime.md | 30 + 25 files changed, 2835 insertions(+), 36 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/task-foundry-responses.yaml create mode 100644 examples/fibey-custom-agent-demo/tools-foundry-responses.yaml create mode 100644 examples/harness/foundry-responses/Dockerfile create mode 100644 examples/harness/foundry-responses/README.md create mode 100644 examples/harness/foundry-responses/main.go create mode 100644 examples/harness/foundry-responses/main_test.go create mode 100644 examples/harness/foundry-responses/testdata/golden/01_initial_hosted_request.json create mode 100644 examples/harness/foundry-responses/testdata/golden/02_function_call_response.json create mode 100644 examples/harness/foundry-responses/testdata/golden/03_tool_call_requested_frame.json create mode 100644 examples/harness/foundry-responses/testdata/golden/04_orka_continue_request.json create mode 100644 examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json create mode 100644 examples/harness/foundry-responses/testdata/golden/06_final_message_response.json create mode 100644 examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json create mode 100644 examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json create mode 100644 examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json create mode 100644 examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json diff --git a/config/samples/core_v1alpha1_agentruntime_foundry.yaml b/config/samples/core_v1alpha1_agentruntime_foundry.yaml index 07fcc0296..e4bb0c12e 100644 --- a/config/samples/core_v1alpha1_agentruntime_foundry.yaml +++ b/config/samples/core_v1alpha1_agentruntime_foundry.yaml @@ -4,26 +4,29 @@ metadata: labels: app.kubernetes.io/name: orka app.kubernetes.io/managed-by: kustomize - name: sample-foundry-runtime + name: sample-foundry-responses-runtime spec: - # Namespace-local facade for an operator-deployed Azure AI Foundry hosted-agent adapter. - # Foundry credentials belong to the adapter Secret. Orka Tool credentials stay in - # Orka-governed credential sources and are used only by brokered execution. + # Namespace-local facade for an operator-deployed Foundry hosted Responses + # adapter (examples/harness/foundry-responses). Foundry credentials belong to + # the adapter Secret. Orka Tool credentials stay in Orka-governed credential + # sources and are used only by brokered execution. contractVersion: orka.harness.v1 deployment: mode: external-endpoint - endpoint: http://sample-foundry-runtime.default.svc.cluster.local:8080 + endpoint: http://sample-foundry-responses-runtime.default.svc.cluster.local:8080 clientAuth: bearerTokenSecretRef: - name: sample-foundry-runtime-token + name: sample-foundry-responses-runtime-token key: token capabilities: + # Mirror only the classes the hosted AgentKit deployment is statically + # configured and conformance-tested to request. Add write only after the + # hosted AgentKit write schema and Orka brokered-write conformance pass. toolExecutionModes: - observed - brokered brokeredToolClasses: - read - - write supportsCancel: true supportsRuntimeSessions: true supportsContinuation: true diff --git a/examples/bring-your-own-agent-runtime-demo/README.md b/examples/bring-your-own-agent-runtime-demo/README.md index 67240b48e..acaddf86c 100644 --- a/examples/bring-your-own-agent-runtime-demo/README.md +++ b/examples/bring-your-own-agent-runtime-demo/README.md @@ -90,7 +90,7 @@ examples/fibey-custom-agent-demo/switch-backend.sh http examples/fibey-custom-agent-demo/switch-backend.sh foundry ``` -The workflow, Tool CRDs, approval UX, and task/result APIs remain Orka-owned. Remote backends receive safe tool schemas and scoped turn metadata only; they do not receive downstream Tool credentials. +The workflow, Tool CRDs, approval UX, and task/result APIs remain Orka-owned. Remote backends receive safe tool schemas and scoped turn metadata only; they do not receive downstream Tool credentials. For Foundry hosted AgentKit Responses, the adapter does not send request-level `tools`; AgentKit schemas must be configured statically and the facade capabilities must match the conformance-tested classes. ## Troubleshooting diff --git a/examples/fibey-custom-agent-demo/README.md b/examples/fibey-custom-agent-demo/README.md index 66111f349..cebbc3985 100644 --- a/examples/fibey-custom-agent-demo/README.md +++ b/examples/fibey-custom-agent-demo/README.md @@ -2,7 +2,7 @@ This demo exercises the first bring-your-own agent runtime slice: Orka registers a namespace-local `AgentRuntime` facade for a remote execution backend, then an `Agent` routes `type: agent` work to it with `spec.runtime.runtimeRef`. -The checked-in backend is a deterministic generic HTTP harness fixture. It advertises `runtimeName: fibey-http-runtime`, supports `orka.harness.v1`, and runs in `observed` tool mode by default. AgentKit Serve and Foundry should plug in by swapping only the backend Service/adapter endpoint and `AgentRuntime` facade, not the Orka workflow. +The checked-in backend is a deterministic generic HTTP harness fixture. It advertises `runtimeName: fibey-http-runtime`, supports `orka.harness.v1`, and runs in `observed` tool mode by default. AgentKit Serve, Foundry Assistants, and Foundry hosted AgentKit Responses should plug in by swapping only the backend Service/adapter endpoint and `AgentRuntime` facade, not the Orka workflow. ## Backend facades @@ -10,7 +10,7 @@ The checked-in backend is a deterministic generic HTTP harness fixture. It adver | --- | --- | --- | | `fibey-http-runtime` | Generic mock/self-hosted HTTP runtime | Harness bearer token only | | `fibey-agentkit-runtime` | AgentKit Serve adapter | Adapter/runtime config only | -| `fibey-foundry-runtime` | Foundry adapter | Adapter Secret; no Orka Tool production credentials | +| `fibey-agentkit-foundry-responses` | Foundry hosted AgentKit Responses adapter | Adapter Secret; no Orka Tool production credentials | `fibey-agentkit-runtime` is intentionally observed-only in the checked-in demo: it should show `toolExecutionModes: [observed]`, `supportsCancel: true`, and `supportsRuntimeSessions: true`, with no `brokeredToolClasses` or `supportsContinuation`. AgentKit brokered read/write/coordination exist only for deployments that explicitly enable those conformance-gated profiles. @@ -106,12 +106,16 @@ kubectl apply -f examples/fibey-custom-agent-demo/agentruntime-agentkit.yaml kubectl apply -f examples/fibey-custom-agent-demo/agent-agentkit.yaml kubectl wait --for=condition=Ready agentruntime/fibey-agentkit-runtime --timeout=60s -# Foundry adapter facade; requires a Service named fibey-foundry-runtime. -# Build/deploy examples/harness/foundry with ORKA_FOUNDRY_* credentials first. +# Foundry hosted AgentKit Responses facade; requires a Service named fibey-agentkit-foundry-responses. +# Build/deploy examples/harness/foundry-responses with ORKA_FOUNDRY_RESPONSES_* credentials first. kubectl apply -f examples/fibey-custom-agent-demo/secret-foundry.yaml kubectl apply -f examples/fibey-custom-agent-demo/agentruntime-foundry.yaml kubectl apply -f examples/fibey-custom-agent-demo/agent-foundry.yaml -kubectl wait --for=condition=Ready agentruntime/fibey-foundry-runtime --timeout=60s +kubectl wait --for=condition=Ready agentruntime/fibey-agentkit-foundry-responses --timeout=60s + +# Optional literal brokered Fibey read/write scenario once downstream services exist. +kubectl apply -f examples/fibey-custom-agent-demo/tools-foundry-responses.yaml +kubectl apply -f examples/fibey-custom-agent-demo/task-foundry-responses.yaml ``` Run the same task against another backend by changing only `spec.agentRef.name`, for example: @@ -125,4 +129,4 @@ examples/fibey-custom-agent-demo/switch-backend.sh http The script validates the selected `AgentRuntime` and `Agent`, then patches only the Task's `spec.agentRef.name`. -Brokered mode is used only when the selected runtime advertises brokered capabilities and the task/agent exposes allowed tools. Current AgentKit Serve facades do not advertise brokered mode, so AgentKit-owned tools remain internal to AgentKit and Orka observes only lifecycle/output frames. Orka-owned side-effect tools stay behind Orka brokered governance; production tool credentials are not handed to the remote backend. +Brokered mode is used only when the selected runtime advertises brokered capabilities and the task/agent exposes allowed tools. Current AgentKit Serve facades do not advertise brokered mode, so AgentKit-owned tools remain internal to AgentKit and Orka observes only lifecycle/output frames. The Foundry hosted Responses facade must advertise only the brokered classes statically configured in AgentKit and verified by conformance. Orka-owned side-effect tools stay behind Orka brokered governance; production tool credentials are not handed to the remote backend, and hosted Responses requests do not include request-level `tools`. diff --git a/examples/fibey-custom-agent-demo/agent-foundry.yaml b/examples/fibey-custom-agent-demo/agent-foundry.yaml index fcb8a28b0..ef1058cb0 100644 --- a/examples/fibey-custom-agent-demo/agent-foundry.yaml +++ b/examples/fibey-custom-agent-demo/agent-foundry.yaml @@ -1,4 +1,4 @@ -# Optional Agent that selects the Foundry hosted-agent backend facade. +# Optional Agent that selects the Foundry hosted AgentKit Responses backend facade. apiVersion: core.orka.ai/v1alpha1 kind: Agent metadata: @@ -6,7 +6,7 @@ metadata: spec: runtime: runtimeRef: - name: fibey-foundry-runtime + name: fibey-agentkit-foundry-responses systemPrompt: inline: | - You are Fibey's Foundry-backed incident scout. Produce a concise dossier and request Orka-brokered tools for evidence or consequential actions. + You are Fibey's Foundry-hosted AgentKit incident scout. Use Orka-brokered read tools for evidence and Orka-brokered write tools only when approval is required. diff --git a/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml b/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml index ded01edb3..ddb549dd4 100644 --- a/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml +++ b/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml @@ -1,20 +1,24 @@ -# Optional namespace-local facade for a Foundry hosted-agent adapter implementing orka.harness.v1. -# Apply this with secret-foundry.yaml and point an Agent runtimeRef at fibey-foundry-runtime -# after deploying the adapter Service named fibey-foundry-runtime. +# Optional namespace-local facade for a Foundry hosted AgentKit Responses adapter +# implementing orka.harness.v1. Apply this with secret-foundry.yaml and point an +# Agent runtimeRef at fibey-agentkit-foundry-responses after deploying the +# examples/harness/foundry-responses adapter Service with the same name. apiVersion: core.orka.ai/v1alpha1 kind: AgentRuntime metadata: - name: fibey-foundry-runtime + name: fibey-agentkit-foundry-responses spec: contractVersion: orka.harness.v1 deployment: mode: external-endpoint - endpoint: http://fibey-foundry-runtime.default.svc.cluster.local:8080 + endpoint: http://fibey-agentkit-foundry-responses.default.svc.cluster.local:8080 clientAuth: bearerTokenSecretRef: - name: fibey-foundry-runtime-token + name: fibey-agentkit-foundry-responses-token key: token capabilities: + # This sample assumes the hosted AgentKit deployment has static read/write + # schemas and has passed the adapter's fake-server read/write conformance. + # Narrow this list if the deployment only supports read. toolExecutionModes: - observed - brokered diff --git a/examples/fibey-custom-agent-demo/secret-foundry.yaml b/examples/fibey-custom-agent-demo/secret-foundry.yaml index 3c328d638..c2e25ba33 100644 --- a/examples/fibey-custom-agent-demo/secret-foundry.yaml +++ b/examples/fibey-custom-agent-demo/secret-foundry.yaml @@ -1,13 +1,14 @@ -# Optional harness bearer token for the Foundry adapter facade. -# This authenticates Orka to the adapter endpoint; Foundry credentials stay in the adapter Secret. +# Optional harness bearer token for the Foundry hosted Responses adapter facade. +# This authenticates Orka to the adapter endpoint; Foundry credentials stay in +# the adapter Deployment Secret/env and are never stored in this AgentRuntime. apiVersion: v1 kind: Secret metadata: - name: fibey-foundry-runtime-token + name: fibey-agentkit-foundry-responses-token annotations: - orka.ai/agent-runtime-endpoint: http://fibey-foundry-runtime.default.svc.cluster.local:8080 + orka.ai/agent-runtime-endpoint: http://fibey-agentkit-foundry-responses.default.svc.cluster.local:8080 labels: orka.ai/agent-runtime-auth: "true" - orka.ai/agent-runtime-name: fibey-foundry-runtime + orka.ai/agent-runtime-name: fibey-agentkit-foundry-responses stringData: - token: mock-token + token: REDACTED diff --git a/examples/fibey-custom-agent-demo/switch-backend.sh b/examples/fibey-custom-agent-demo/switch-backend.sh index fcfb41345..e0812e715 100755 --- a/examples/fibey-custom-agent-demo/switch-backend.sh +++ b/examples/fibey-custom-agent-demo/switch-backend.sh @@ -27,7 +27,7 @@ case "${backend}" in ;; foundry) agent="fibey-remote-foundry" - runtime="fibey-foundry-runtime" + runtime="fibey-agentkit-foundry-responses" ;; -h|--help|help|"") usage diff --git a/examples/fibey-custom-agent-demo/task-foundry-responses.yaml b/examples/fibey-custom-agent-demo/task-foundry-responses.yaml new file mode 100644 index 000000000..1bf07439d --- /dev/null +++ b/examples/fibey-custom-agent-demo/task-foundry-responses.yaml @@ -0,0 +1,20 @@ +# Optional Foundry hosted AgentKit Responses task that exposes the Fibey brokered +# read/write tools. Apply with tools-foundry-responses.yaml and the Foundry +# Responses facade manifests after deploying the downstream mock/real services. +apiVersion: core.orka.ai/v1alpha1 +kind: Task +metadata: + name: fibey-foundry-responses-quincy-north-alert +spec: + type: agent + agentRef: + name: fibey-remote-foundry + agentRuntime: + allowedTools: + - check-network-telemetry + - get-active-incidents + - dispatch-work-order + - escalate-incident + prompt: | + Quincy North alert: pump telemetry is anomalous after overnight maintenance. + Investigate likely cause with brokered read tools, summarize evidence, and request approval before dispatching any work order. diff --git a/examples/fibey-custom-agent-demo/tools-foundry-responses.yaml b/examples/fibey-custom-agent-demo/tools-foundry-responses.yaml new file mode 100644 index 000000000..ac115b231 --- /dev/null +++ b/examples/fibey-custom-agent-demo/tools-foundry-responses.yaml @@ -0,0 +1,85 @@ +# Optional Fibey brokered tools for a Foundry hosted AgentKit Responses demo. +# These are not included in the default kustomization because the downstream +# services are deployment-specific. Remote hosted AgentKit receives only safe +# schemas; Orka owns execution, credentials, approvals, idempotency, and audit. +apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: check-network-telemetry +spec: + description: Read sanitized Fibey network telemetry for a site or asset. + brokeredToolClass: read + parameters: + type: object + properties: + site: + type: string + asset: + type: string + required: + - site + http: + url: http://fibey-telemetry.default.svc.cluster.local:8080/check + method: POST +--- +apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: get-active-incidents +spec: + description: Read active Fibey incident records for a site. + brokeredToolClass: read + parameters: + type: object + properties: + site: + type: string + required: + - site + http: + url: http://fibey-incidents.default.svc.cluster.local:8080/active + method: POST +--- +apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: dispatch-work-order +spec: + description: Dispatch an approved Fibey work order. Requires Orka approval and idempotency. + brokeredToolClass: write + parameters: + type: object + properties: + site: + type: string + action: + type: string + severity: + type: string + required: + - site + - action + http: + url: http://fibey-dispatch.default.svc.cluster.local:8080/work-orders + method: POST +--- +apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: escalate-incident +spec: + description: Escalate an approved Fibey incident to an on-call owner. Requires Orka approval and idempotency. + brokeredToolClass: write + parameters: + type: object + properties: + incident: + type: string + reason: + type: string + required: + - incident + - reason + http: + url: http://fibey-dispatch.default.svc.cluster.local:8080/escalations + method: POST diff --git a/examples/harness/foundry-responses/Dockerfile b/examples/harness/foundry-responses/Dockerfile new file mode 100644 index 000000000..00920fef9 --- /dev/null +++ b/examples/harness/foundry-responses/Dockerfile @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN mkdir -p /out && CGO_ENABLED=0 GOOS=linux go build -o /out/orka-foundry-responses-harness-adapter ./examples/harness/foundry-responses + +FROM gcr.io/distroless/static:nonroot +COPY --from=build /out/orka-foundry-responses-harness-adapter /orka-foundry-responses-harness-adapter +USER 65532:65532 +EXPOSE 8090 +ENTRYPOINT ["/orka-foundry-responses-harness-adapter"] diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md new file mode 100644 index 000000000..66c4e7250 --- /dev/null +++ b/examples/harness/foundry-responses/README.md @@ -0,0 +1,126 @@ +# Foundry hosted Responses AgentRuntime adapter + +This adapter presents an Azure AI Foundry **hosted AgentKit agent** endpoint as an `orka.harness.v1` runtime. It targets endpoint-scoped hosted Responses: + +```text +POST /agents//endpoint/protocols/openai/responses?api-version=... +``` + +Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `examples/harness/foundry` only for the older Assistants/threads/run protocol. + +## Security model + +- The adapter never sends request-level `tools` to hosted `/responses`; hosted AgentKit must be statically configured with the safe function schemas it is allowed to request. +- Orka remains authoritative. Hosted AgentKit can request a function by name, but Orka still validates the request against the Task policy and Tool CRDs, performs approval checks, injects idempotency keys, executes/brokers the tool, and audits the result. +- Orka Tool URLs, auth refs, headers, and production tool credentials are not sent to Foundry. +- Adapter bearer tokens and Foundry credentials must live in Kubernetes Secrets or environment variables. Do not put them in `AgentRuntime` specs or logs. + +## Configuration + +| Env var | Purpose | +| --- | --- | +| `ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR` | HTTP listen address, default `:8090`. | +| `ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME` | Runtime name advertised in `/v1/capabilities`, default `foundry-agentkit-responses`. | +| `ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN` | Bearer token Orka uses for mutating/streaming harness endpoints. | +| `ORKA_FOUNDRY_RESPONSES_ENDPOINT` | Preferred full hosted Responses endpoint URL, including `/agents//endpoint/protocols/openai/responses`. The adapter appends `api-version` from `ORKA_FOUNDRY_RESPONSES_API_VERSION` when missing. | +| `ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT` + `ORKA_FOUNDRY_RESPONSES_AGENT_NAME` | Optional alternative to build the hosted Responses endpoint from a project endpoint and agent name. | +| `ORKA_FOUNDRY_RESPONSES_API_VERSION` | API version query value, default `v1`. | +| `ORKA_FOUNDRY_RESPONSES_API_KEY` | Static API-key auth mode. Tests/demo only unless your deployment standard permits it. | +| `ORKA_FOUNDRY_RESPONSES_AUTH_BEARER` | Static bearer auth mode. Tests/demo only unless supplied by a production token refresher sidecar. | +| `ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE` | Reserved for future workload-identity token refresh support; currently not used. | +| `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` | Comma-separated static classes the hosted AgentKit deployment has been configured and conformance-tested to request, e.g. `read` or `read,write`. Empty means observed-only. | +| `ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT` | Per-request timeout for hosted Responses calls, default `20s`. | +| `ORKA_FOUNDRY_RESPONSES_STATE_RETENTION` | How long terminal in-memory turn/session state is retained, default `10m`. | +| `ORKA_FOUNDRY_RESPONSES_MAX_APPROVAL_WAIT` | Maximum time a pending brokered call may wait before a late continuation fails safely, default `30m`. | + +Exactly one Foundry auth mode (`API_KEY` or `AUTH_BEARER`) must be set. + +## Endpoint safety rules + +The hosted Responses endpoint must: + +- use HTTPS in production; +- use HTTP only for loopback tests (`localhost`, `127.0.0.1`, or `::1`); +- end in `/responses`; +- not include username/password, fragments, or query parameters other than `api-version`. + +The adapter returns degraded health and rejects starts when the endpoint is unsafe. + +## Capability discipline + +Capabilities must reflect the **static schemas actually deployed in AgentKit**: + +- If `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` is empty, the adapter advertises only `observed` and `supportsContinuation=false`. +- If it is `read`, the adapter advertises brokered read only. +- Advertise `write` only after the hosted AgentKit deployment has a static write schema and passes write conformance. Orka will still gate the write with approval/idempotency, but the hosted model must not be told it can request writes unless that path is intentionally enabled. + +## Protocol mapping + +Initial turn: + +```json +{"input":"Investigate incident"} +``` + +No `tools` field is sent. + +Hosted AgentKit function call: + +```json +{"type":"function_call","call_id":"call_1","name":"check-network-telemetry","arguments":"{\"site\":\"quincy-north\"}"} +``` + +Adapter emits `ToolCallRequested` with the exact `call_id`, function name, and compact JSON object arguments. + +Orka continuation: + +```json +{"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}","status":"completed"} +``` + +The hosted continuation request includes `previous_response_id` and the `function_call_output` item. + +## Output and error encoding + +`function_call_output.output` is always a compact JSON string: + +- successful tool result: `{"approved":true,"output":}` +- declined approval or policy/execution error: `{"approved":false,"error":}` + +Approval decline, tool policy rejection, and tool execution failure fixtures live under `testdata/golden/`. + +## State, restart, and sessions + +This MVP stores turn state in memory. That is intentionally fail-safe: + +- duplicate identical `/continue` calls for a submitted call are accepted without a second hosted continuation; +- conflicting duplicate `/continue` calls are rejected; +- if the adapter restarts while a tool approval is pending, `/continue` returns `turn not found` and does not call Foundry, so the adapter itself does not duplicate a side effect; +- Orka's broker/idempotency ledger remains the source of truth for actual write execution. + +The adapter captures Foundry session headers such as `x-agent-session-id` and reuses them for later calls with the same Orka `runtimeSessionID`. Session/auth headers are stored only in memory and are not logged. + +## Local build + +```bash +docker build -t ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest -f examples/harness/foundry-responses/Dockerfile . +``` + +## Tests + +```bash +go test ./examples/harness/foundry-responses +``` + +The tests use a fake hosted Responses server and golden fixtures for initial requests, function calls, `ToolCallRequested`, continuations, final messages, error encoding, and buffered multiple-call behavior. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +| --- | --- | --- | +| Health is degraded | Missing adapter bearer, missing/unsafe endpoint, bad brokered class config, or missing Foundry auth | Check env vars and endpoint safety. | +| `tools` rejected by hosted endpoint | You are using the wrong adapter or adding request-level tools | This adapter intentionally never sends `tools`; configure AgentKit static schemas instead. | +| Brokered start rejected | Adapter does not advertise the requested tool class | Set `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` only after AgentKit static schema/conformance is ready. | +| Turn fails with unknown tool | Hosted AgentKit requested a function name not present in Orka's safe `StartTurnRequest.input.tools` | Fix the AgentKit static schema or Task `allowedTools`. | +| Turn fails with malformed arguments | Hosted AgentKit emitted non-object or invalid JSON arguments | Fix the hosted schema/prompting; Orka rejects before tool execution. | +| Continue returns `turn not found` after restart | In-memory state was lost while waiting for approval | Re-run/fail the Task safely; no hosted continuation was sent by the restarted adapter. | diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go new file mode 100644 index 000000000..b5ad4d300 --- /dev/null +++ b/examples/harness/foundry-responses/main.go @@ -0,0 +1,1290 @@ +package main + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "maps" + "net/http" + "net/url" + "os" + "reflect" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/orka-agents/orka/internal/harness" +) + +const ( + defaultAddr = ":8090" + defaultAPIVersion = "v1" + defaultRequestTimeout = 20 * time.Second + defaultStateRetention = 10 * time.Minute + defaultMaxApprovalWait = 30 * time.Minute + maxFoundryOutputBytes = 1 << 20 + maxFoundryBodyBytes = 4 << 20 + + envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" + envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" + envAdapterBearer = "ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_" + "TOKEN" + envEndpoint = "ORKA_FOUNDRY_RESPONSES_ENDPOINT" + envProjectEndpoint = "ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT" + envAgentName = "ORKA_FOUNDRY_RESPONSES_AGENT_NAME" + envAuthBearer = "ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" + envFoundryAuth = "ORKA_FOUNDRY_RESPONSES_API_" + "KEY" + envAPIVersion = "ORKA_FOUNDRY_RESPONSES_API_VERSION" + envRequestTimeout = "ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT" + envStateRetention = "ORKA_FOUNDRY_RESPONSES_STATE_RETENTION" + envMaxApprovalWait = "ORKA_FOUNDRY_RESPONSES_MAX_APPROVAL_WAIT" + envBrokeredToolClasses = "ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES" + envAudience = "ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE" +) + +type config struct { + addr string + runtimeName string + adapterBearer string + endpoint string + projectEndpoint string + agentName string + authBearer string + foundryAuth string + apiVersion string + requestTimeout time.Duration + stateRetention time.Duration + maxApprovalWait time.Duration + brokeredToolClasses []harness.BrokeredToolClass + configError string +} + +type server struct { + cfg config + client *http.Client + + mu sync.Mutex + turns map[harness.HarnessTurnID]*turnState + runtimeSessions map[harness.RuntimeSessionID]foundrySession +} + +type foundrySession struct { + ID string + LastSeen time.Time +} + +type turnState struct { + request harness.StartTurnRequest + responseID string + foundrySessionID string + pendingTools map[string]string + pendingSince map[string]time.Time + bufferedResults map[string]harness.ToolCallResult + bufferedPayloads map[string]string + submittedPayloads map[string]string + frames []harness.HarnessEventFrame + completed bool + continueMu sync.Mutex +} + +type responsesRequest struct { + Input any `json:"input"` + PreviousResponseID string `json:"previous_response_id,omitempty"` + AgentSessionID string `json:"agent_session_id,omitempty"` +} + +type responsesFunctionCallOutput struct { + Type string `json:"type"` + CallID string `json:"call_id"` + Output string `json:"output"` + Status string `json:"status,omitempty"` +} + +type responsesResponse struct { + ID string `json:"id"` + AgentSessionID string `json:"agent_session_id,omitempty"` + Status string `json:"status,omitempty"` + Output []responsesOutput `json:"output,omitempty"` + Error *responsesError `json:"error,omitempty"` +} + +type responsesOutput struct { + Type string `json:"type"` + Role string `json:"role,omitempty"` + Name string `json:"name,omitempty"` + CallID string `json:"call_id,omitempty"` + Arguments json.RawMessage `json:"arguments,omitempty"` + Content any `json:"content,omitempty"` + Text string `json:"text,omitempty"` + Status string `json:"status,omitempty"` +} + +type responsesError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +type pendingFunctionCall struct { + callID string + name string + args json.RawMessage +} + +const responsesEndpointRequirement = "foundry hosted Responses endpoint must use https " + + "(http allowed only for loopback), target /responses, and must not include foundryAuths, " + + "fragments, or query parameters other than api-version" + +func main() { + cfg := loadConfig() + s := newServer(cfg, &http.Client{Timeout: cfg.requestTimeout}) + log.Printf( + "Foundry hosted Responses AgentRuntime adapter listening on %s runtime=%s endpoint=%s", + cfg.addr, + cfg.runtimeName, + sanitizeEndpoint(cfg.endpoint), + ) + if err := http.ListenAndServe(cfg.addr, s.handler()); err != nil { + log.Fatal(err) + } +} + +func loadConfig() config { + classes, classErr := parseBrokeredToolClasses(os.Getenv(envBrokeredToolClasses)) + cfg := config{ + addr: firstNonBlank(os.Getenv(envAddr), defaultAddr), + runtimeName: firstNonBlank(os.Getenv(envRuntimeName), "foundry-agentkit-responses"), + adapterBearer: strings.TrimSpace(os.Getenv(envAdapterBearer)), + endpoint: strings.TrimSpace(os.Getenv(envEndpoint)), + projectEndpoint: strings.TrimRight(strings.TrimSpace(os.Getenv(envProjectEndpoint)), "/"), + agentName: strings.TrimSpace(os.Getenv(envAgentName)), + authBearer: strings.TrimSpace(os.Getenv(envAuthBearer)), + foundryAuth: strings.TrimSpace(os.Getenv(envFoundryAuth)), + apiVersion: firstNonBlank(os.Getenv(envAPIVersion), defaultAPIVersion), + requestTimeout: parseDurationEnv(envRequestTimeout, defaultRequestTimeout), + stateRetention: parseDurationEnv(envStateRetention, defaultStateRetention), + maxApprovalWait: parseDurationEnv(envMaxApprovalWait, defaultMaxApprovalWait), + brokeredToolClasses: classes, + } + _ = os.Getenv(envAudience) // Reserved for a future workload-identity token provider; never logged. + if classErr != nil { + cfg.configError = classErr.Error() + } + return cfg +} + +func newServer(cfg config, client *http.Client) *server { + if client == nil { + client = &http.Client{Timeout: cfg.requestTimeout} + } + return &server{ + cfg: cfg, + client: client, + turns: map[harness.HarnessTurnID]*turnState{}, + runtimeSessions: map[harness.RuntimeSessionID]foundrySession{}, + } +} + +func (s *server) handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc(harness.HealthPath, s.health) + mux.HandleFunc(harness.CapabilitiesPath, s.capabilities) + mux.HandleFunc(harness.TurnsPath, s.startTurn) + mux.HandleFunc(harness.TurnsPath+"/", s.turn) + return mux +} + +func (s *server) health(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + _, endpointErr := s.responsesEndpoint() + ready := s.cfg.configError == "" && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + status := harness.HealthStatusOK + msg := "ready" + if !ready { + status = harness.HealthStatusDegraded + parts := []string{ + "adapter bearer, safe Foundry hosted Responses endpoint, and exactly one Foundry auth mode are required", + } + if s.cfg.configError != "" { + parts = append(parts, s.cfg.configError) + } + if endpointErr != nil { + parts = append(parts, endpointErr.Error()) + } + msg = strings.Join(parts, "; ") + } + harness.WriteJSON(w, http.StatusOK, harness.HealthResponse{ + Version: harness.ProtocolVersion, + Status: status, + Ready: ready, + CheckedAt: time.Now().UTC(), + Message: msg, + Metadata: map[string]string{"backend": "foundry-responses"}, + }) +} + +func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + modes := []harness.ToolExecutionMode{harness.ToolExecutionModeObserved} + maxTurnSeconds := int(s.cfg.requestTimeout.Seconds()) + if len(s.cfg.brokeredToolClasses) > 0 { + modes = append(modes, harness.ToolExecutionModeBrokered) + maxTurnSeconds = int((s.cfg.requestTimeout + s.cfg.maxApprovalWait).Seconds()) + } + harness.WriteJSON(w, http.StatusOK, harness.CapabilitiesResponse{ + Version: harness.ProtocolVersion, + ProtocolVersion: harness.ProtocolVersion, + Transport: harness.HTTPTransport, + RuntimeName: s.cfg.runtimeName, + RuntimeVersion: "foundry-responses-adapter", + ProviderKind: harness.ProviderKindRemote, + ToolExecutionModes: modes, + BrokeredToolClasses: append([]harness.BrokeredToolClass(nil), s.cfg.brokeredToolClasses...), + SupportsCancel: true, + SupportsRuntimeSessions: true, + SupportsContinuation: len(s.cfg.brokeredToolClasses) > 0, + SupportsArtifacts: false, + MaxConcurrentTurns: 1, + MaxTurnSeconds: maxTurnSeconds, + MaxOutputBytes: maxFoundryOutputBytes, + Metadata: map[string]string{"backend": "foundry-responses"}, + }) +} + +func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { + if !s.authorized(w, r) { + return + } + if r.Method != http.MethodPost { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + var req harness.StartTurnRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + harness.WriteError(w, http.StatusBadRequest, "invalid JSON request") + return + } + if err := req.Validate(); err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if err := s.validateStartRequest(req); err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + eventsPath, err := harness.EventStreamPath(req.TurnID) + if err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + s.mu.Lock() + if existing := s.turns[req.TurnID]; existing != nil { + response := startTurnResponse(existing.request, eventsPath) + if !sameStartTurnRequest(existing.request, req) { + s.mu.Unlock() + harness.WriteError(w, http.StatusConflict, "turn already exists") + return + } + s.mu.Unlock() + harness.WriteJSON(w, http.StatusAccepted, response) + return + } + turn := &turnState{ + request: req, + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + s.turns[req.TurnID] = turn + s.mu.Unlock() + + ctx, cancel := context.WithTimeout(r.Context(), s.cfg.requestTimeout) + defer cancel() + var response responsesResponse + initialRequest := responsesRequest{Input: req.Input.Prompt} + if err := s.postResponses(ctx, req.RuntimeSessionID, initialRequest, &response); err != nil { + s.mu.Lock() + delete(s.turns, req.TurnID) + s.mu.Unlock() + harness.WriteError(w, http.StatusBadGateway, err.Error()) + return + } + s.mu.Lock() + s.updateTurnSessionLocked(turn) + s.mu.Unlock() + s.handleResponsesResponse(turn, response) + harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) +} + +func (s *server) turn(w http.ResponseWriter, r *http.Request) { + if !s.authorized(w, r) { + return + } + turnID, resource, err := harness.ParseTurnResourcePath(r.URL.EscapedPath()) + if err != nil { + harness.WriteError(w, http.StatusNotFound, "not found") + return + } + s.mu.Lock() + turn := s.turns[turnID] + s.mu.Unlock() + if turn == nil { + harness.WriteError(w, http.StatusNotFound, "turn not found") + return + } + switch resource { + case harness.TurnResourceEvents: + if r.Method != http.MethodGet { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + s.streamEvents(w, r, turn) + case harness.TurnResourceContinue: + if r.Method != http.MethodPost { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + s.continueTurn(w, r, turn) + case harness.TurnResourceCancel: + s.cancelTurn(w, r, turn) + default: + harness.WriteError(w, http.StatusNotFound, "not found") + } +} + +func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turnState) { + afterSeq := parseAfterSeq(r.URL.Query().Get("afterSeq")) + w.Header().Set("Content-Type", "text/event-stream") + s.mu.Lock() + frames := append([]harness.HarnessEventFrame(nil), turn.frames...) + completed := turn.completed + s.mu.Unlock() + for _, frame := range frames { + if frame.Seq > afterSeq { + _ = harness.WriteSSEFrame(w, frame) + } + } + if completed { + _ = harness.WriteSSEDone(w) + } +} + +func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turnState) { + var req harness.ContinueTurnRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + harness.WriteError(w, http.StatusBadRequest, "invalid JSON request") + return + } + if err := req.Validate(); err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if !sameContinueIdentity(turn.request, req) { + harness.WriteError(w, http.StatusBadRequest, "continue request does not match started turn") + return + } + turn.continueMu.Lock() + defer turn.continueMu.Unlock() + + if err := s.ensureTerminalContinueIsDuplicate(turn, req.ToolResults); err != nil { + harness.WriteError(w, http.StatusConflict, err.Error()) + return + } + s.mu.Lock() + completed := turn.completed + s.mu.Unlock() + if completed { + harness.WriteJSON( + w, + http.StatusAccepted, + continueResponse(req, "duplicate continue accepted for terminal turn"), + ) + return + } + + resultsToSubmit, err := s.recordContinueResults(turn, req.ToolResults) + if err != nil { + harness.WriteError(w, http.StatusConflict, err.Error()) + return + } + if len(resultsToSubmit) == 0 { + harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) + return + } + outputs, payloadByCall, err := functionCallOutputs(resultsToSubmit) + if err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + s.mu.Lock() + previousResponseID := turn.responseID + s.mu.Unlock() + if strings.TrimSpace(previousResponseID) == "" { + harness.WriteError(w, http.StatusConflict, "cannot continue before Foundry response id is known") + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.cfg.requestTimeout) + defer cancel() + var response responsesResponse + continuation := responsesRequest{ + PreviousResponseID: previousResponseID, + Input: outputs, + } + s.markSubmittedPayloads(turn, payloadByCall) + if err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response); err != nil { + s.mu.Lock() + s.appendFailedLocked( + turn, + "foundry_continuation_unknown", + "hosted continuation failed after submission was attempted; "+ + "failing closed to avoid duplicate continuation: "+err.Error(), + ) + s.mu.Unlock() + harness.WriteError(w, http.StatusBadGateway, err.Error()) + return + } + s.mu.Lock() + for _, result := range resultsToSubmit { + toolName := turn.pendingTools[result.ToolCallID] + if toolName == "" { + toolName = result.ToolCallID + } + s.appendFrameLocked( + turn, + harness.FrameToolResultReceived, + "brokered tool result received", + func(f *harness.HarnessEventFrame) { + f.ToolName = toolName + f.ToolCallID = result.ToolCallID + f.Content = result.Output + f.Error = result.Error + }, + ) + delete(turn.pendingTools, result.ToolCallID) + delete(turn.pendingSince, result.ToolCallID) + delete(turn.bufferedResults, result.ToolCallID) + delete(turn.bufferedPayloads, result.ToolCallID) + } + s.updateTurnSessionLocked(turn) + s.mu.Unlock() + s.handleResponsesResponse(turn, response) + harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) +} + +func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnState) { + if r.Method != http.MethodPost { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + turn.continueMu.Lock() + defer turn.continueMu.Unlock() + s.mu.Lock() + if !turn.completed { + s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled", nil) + turn.completed = true + s.scheduleTurnCleanupLocked(turn) + } + req := turn.request + s.mu.Unlock() + harness.WriteJSON( + w, + http.StatusAccepted, + harness.CancelTurnResponse{ + Version: harness.ProtocolVersion, + Accepted: true, + RuntimeSessionID: req.RuntimeSessionID, + TurnID: req.TurnID, + CorrelationID: req.CorrelationID, + }, + ) +} + +func (s *server) handleResponsesResponse(turn *turnState, response responsesResponse) { + s.mu.Lock() + defer s.mu.Unlock() + if turn.completed { + return + } + responseIDPresent := strings.TrimSpace(response.ID) != "" + if responseIDPresent { + turn.responseID = response.ID + } + if response.Error != nil { + s.appendFailedLocked( + turn, + "foundry_response_error", + firstNonBlank(response.Error.Message, response.Error.Code, "Foundry hosted Responses returned an error"), + ) + return + } + calls, err := s.extractFunctionCalls(turn.request, response.Output) + if err != nil { + s.appendFailedLocked(turn, "foundry_function_call_invalid", err.Error()) + return + } + if len(calls) > 0 { + if !responseIDPresent { + s.appendFailedLocked( + turn, + "foundry_response_id_missing", + "hosted response returned a function_call without an id needed for continuation", + ) + return + } + now := time.Now().UTC() + for _, call := range calls { + if _, submitted := turn.submittedPayloads[call.callID]; submitted { + continue + } + if _, pending := turn.pendingTools[call.callID]; pending { + continue + } + turn.pendingTools[call.callID] = call.name + turn.pendingSince[call.callID] = now + s.appendFrameLocked( + turn, + harness.FrameToolCallRequested, + "foundry hosted tool call requested", + func(f *harness.HarnessEventFrame) { + f.ToolName = call.name + f.ToolCallID = call.callID + f.Content = call.args + }, + ) + } + return + } + if isFailureStatus(response.Status) { + s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) + return + } + result := responsesMessageText(response.Output) + if len([]byte(result)) > maxFoundryOutputBytes { + s.appendFailedLocked(turn, "foundry_output_too_large", "foundry completion exceeded advertised output limit") + return + } + s.appendFrameLocked( + turn, + harness.FrameTurnCompleted, + "foundry hosted response completed", + func(f *harness.HarnessEventFrame) { + f.Completed = &harness.TurnCompleted{Result: result, FinalEventSeq: f.Seq} + }, + ) + turn.completed = true + s.scheduleTurnCleanupLocked(turn) +} + +func (s *server) extractFunctionCalls( + request harness.StartTurnRequest, + output []responsesOutput, +) ([]pendingFunctionCall, error) { + calls := []responsesOutput{} + for _, item := range output { + if strings.EqualFold(strings.TrimSpace(item.Type), "function_call") { + calls = append(calls, item) + } + } + if len(calls) == 0 { + return nil, nil + } + if request.ToolExecutionMode != harness.ToolExecutionModeBrokered { + return nil, fmt.Errorf( + "hosted response requested a function_call while Orka turn is not in brokered mode", + ) + } + pending := make([]pendingFunctionCall, 0, len(calls)) + seenCallIDs := map[string]struct{}{} + for _, call := range calls { + callID := strings.TrimSpace(call.CallID) + name := strings.TrimSpace(call.Name) + if callID == "" { + return nil, fmt.Errorf("hosted response function_call missing call_id") + } + if _, exists := seenCallIDs[callID]; exists { + return nil, fmt.Errorf("hosted response repeated function_call call_id %q", callID) + } + seenCallIDs[callID] = struct{}{} + if name == "" { + return nil, fmt.Errorf("hosted response function_call %q missing name", callID) + } + definition, ok := findToolDefinition(request.Input.Tools, name) + if !ok { + return nil, fmt.Errorf( + "hosted response requested tool %q that Orka did not expose for this turn", + name, + ) + } + if !s.supportsBrokeredClass(definition.BrokeredClass) { + return nil, fmt.Errorf( + "hosted response requested tool %q with unsupported brokered class %q", + name, + definition.BrokeredClass, + ) + } + args, err := normalizeResponsesToolArguments(call.Arguments) + if err != nil { + return nil, err + } + pending = append(pending, pendingFunctionCall{callID: callID, name: name, args: args}) + } + return pending, nil +} + +func (s *server) recordContinueResults( + turn *turnState, + results []harness.ToolCallResult, +) ([]harness.ToolCallResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(turn.pendingTools) == 0 { + return nil, fmt.Errorf("no tool calls are pending for this turn") + } + now := time.Now().UTC() + for _, result := range results { + payload, err := canonicalToolResultOutput(result) + if err != nil { + return nil, err + } + if submitted, done := turn.submittedPayloads[result.ToolCallID]; done { + if submitted == payload { + continue + } + return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) + } + if _, pending := turn.pendingTools[result.ToolCallID]; !pending { + return nil, fmt.Errorf("tool result %q is not pending for this turn", result.ToolCallID) + } + if pendingAt := turn.pendingSince[result.ToolCallID]; !pendingAt.IsZero() && s.cfg.maxApprovalWait > 0 && + now.Sub(pendingAt) > s.cfg.maxApprovalWait { + turn.completed = true + s.appendFrameLocked( + turn, + harness.FrameTurnFailed, + "approval wait exceeded", + func(f *harness.HarnessEventFrame) { + f.Failed = &harness.TurnFailed{ + Reason: "approval_wait_exceeded", + Message: "maximum brokered tool wait exceeded", + } + f.Error = &harness.ErrorInfo{ + Code: "approval_wait_exceeded", + Message: "maximum brokered tool wait exceeded", + } + }, + ) + s.scheduleTurnCleanupLocked(turn) + return nil, fmt.Errorf("maximum brokered tool wait exceeded") + } + if buffered, exists := turn.bufferedPayloads[result.ToolCallID]; exists { + if buffered == payload { + continue + } + return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) + } + turn.bufferedResults[result.ToolCallID] = result + turn.bufferedPayloads[result.ToolCallID] = payload + } + if len(turn.bufferedResults) < len(turn.pendingTools) { + return nil, nil + } + ids := make([]string, 0, len(turn.pendingTools)) + for id := range turn.pendingTools { + ids = append(ids, id) + } + sort.Strings(ids) + toSubmit := make([]harness.ToolCallResult, 0, len(ids)) + for _, id := range ids { + toSubmit = append(toSubmit, turn.bufferedResults[id]) + } + return toSubmit, nil +} + +func (s *server) markSubmittedPayloads(turn *turnState, payloadByCall map[string]string) { + s.mu.Lock() + defer s.mu.Unlock() + maps.Copy(turn.submittedPayloads, payloadByCall) +} + +func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []harness.ToolCallResult) error { + s.mu.Lock() + defer s.mu.Unlock() + if !turn.completed { + return nil + } + for _, result := range results { + payload, err := canonicalToolResultOutput(result) + if err != nil { + return err + } + submitted, done := turn.submittedPayloads[result.ToolCallID] + if !done { + return fmt.Errorf("terminal turn cannot accept new tool result %q", result.ToolCallID) + } + if submitted != payload { + return fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) + } + } + return nil +} + +func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, map[string]string, error) { + outputs := make([]responsesFunctionCallOutput, 0, len(results)) + payloadByCall := map[string]string{} + for _, result := range results { + payload, err := canonicalToolResultOutput(result) + if err != nil { + return nil, nil, err + } + outputs = append( + outputs, + responsesFunctionCallOutput{ + Type: "function_call_output", + CallID: result.ToolCallID, + Output: payload, + Status: "completed", + }, + ) + payloadByCall[result.ToolCallID] = payload + } + return outputs, payloadByCall, nil +} + +func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { + if result.Error != nil { + return compactJSON(struct { + Approved bool `json:"approved"` + Error *harness.ErrorInfo `json:"error"` + }{Approved: false, Error: result.Error}) + } + if !result.Approved { + return compactJSON(struct { + Approved bool `json:"approved"` + Error *harness.ErrorInfo `json:"error"` + }{Approved: false, Error: &harness.ErrorInfo{Code: "approval_declined", Message: "tool call was not approved"}}) + } + output := json.RawMessage(`{}`) + if len(result.Output) > 0 { + compacted, err := compactRawJSON(result.Output) + if err != nil { + return "", fmt.Errorf("tool result %q output must be valid JSON: %w", result.ToolCallID, err) + } + output = compacted + } + return compactJSON(struct { + Approved bool `json:"approved"` + Output json.RawMessage `json:"output"` + }{Approved: true, Output: output}) +} + +func compactRawJSON(raw json.RawMessage) (json.RawMessage, error) { + var compacted bytes.Buffer + if err := json.Compact(&compacted, raw); err != nil { + return nil, err + } + return json.RawMessage(compacted.Bytes()), nil +} + +func compactJSON(value any) (string, error) { + encoded, err := json.Marshal(value) + if err != nil { + return "", err + } + return string(encoded), nil +} + +func (s *server) postResponses( + ctx context.Context, + runtimeSessionID harness.RuntimeSessionID, + body responsesRequest, + out *responsesResponse, +) error { + if !exactlyOneFoundryAuth(s.cfg) { + return fmt.Errorf("exactly one Foundry auth mode is required") + } + endpoint, err := s.responsesEndpoint() + if err != nil { + return err + } + s.mu.Lock() + session := s.runtimeSessions[runtimeSessionID] + s.mu.Unlock() + if body.AgentSessionID == "" && session.ID != "" { + body.AgentSessionID = session.ID + } + payload, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("Foundry-Features", "HostedAgents=V1Preview") + if s.cfg.foundryAuth != "" { + req.Header.Set("api-key", s.cfg.foundryAuth) + } + if s.cfg.authBearer != "" { + req.Header.Set("Authorization", "Bearer "+s.cfg.authBearer) + } + if session.ID != "" { + req.Header.Set("x-agent-session-id", session.ID) + } + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck + sessionID := firstNonBlank( + resp.Header.Get("x-agent-session-id"), + resp.Header.Get("x-ms-agent-session-id"), + ) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf( + "foundry hosted Responses request failed: HTTP %d: %s", + resp.StatusCode, + strings.TrimSpace(string(data)), + ) + } + if out != nil { + decoder := json.NewDecoder(io.LimitReader(resp.Body, maxFoundryBodyBytes)) + if err := decoder.Decode(out); err != nil { + return fmt.Errorf("decode Foundry hosted Responses response: %w", err) + } + sessionID = firstNonBlank(out.AgentSessionID, sessionID) + } + if sessionID != "" { + s.mu.Lock() + s.runtimeSessions[runtimeSessionID] = foundrySession{ID: sessionID, LastSeen: time.Now().UTC()} + s.mu.Unlock() + } + return nil +} + +func (s *server) responsesEndpoint() (string, error) { + if strings.TrimSpace(s.cfg.endpoint) != "" { + return responsesEndpointWithVersion(s.cfg.endpoint, s.cfg.apiVersion) + } + if strings.TrimSpace(s.cfg.projectEndpoint) == "" || strings.TrimSpace(s.cfg.agentName) == "" { + return "", fmt.Errorf( + "%s; set %s or %s plus %s", + responsesEndpointRequirement, + envEndpoint, + envProjectEndpoint, + envAgentName, + ) + } + if !projectEndpointIsSafe(s.cfg.projectEndpoint) { + return "", fmt.Errorf("%s; project endpoint is unsafe", responsesEndpointRequirement) + } + base := strings.TrimRight( + s.cfg.projectEndpoint, + "/", + ) + "/agents/" + url.PathEscape( + s.cfg.agentName, + ) + "/endpoint/protocols/openai/responses" + return responsesEndpointWithVersion(base, s.cfg.apiVersion) +} + +func responsesEndpointWithVersion(raw, apiVersion string) (string, error) { + if !responsesEndpointIsSafe(raw) { + return "", errors.New(responsesEndpointRequirement) + } + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "", errors.New(responsesEndpointRequirement) + } + if strings.TrimSpace(apiVersion) != "" { + q := u.Query() + if q.Get("api-version") == "" { + q.Set("api-version", strings.TrimSpace(apiVersion)) + u.RawQuery = q.Encode() + } + } + return u.String(), nil +} + +func responsesEndpointIsSafe(raw string) bool { + trimmed := strings.TrimSpace(raw) + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Scheme == "" || parsed.Host == "" || strings.TrimSpace(parsed.Path) == "" { + return false + } + if parsed.User != nil || parsed.ForceQuery || parsed.Fragment != "" || strings.Contains(trimmed, "#") { + return false + } + if !strings.HasSuffix(strings.TrimRight(parsed.Path, "/"), "/responses") { + return false + } + if parsed.RawQuery != "" { + values, err := url.ParseQuery(parsed.RawQuery) + if err != nil { + return false + } + for key, vals := range values { + if key != "api-version" || len(vals) == 0 { + return false + } + for _, val := range vals { + if strings.TrimSpace(val) == "" { + return false + } + } + } + } + if strings.EqualFold(parsed.Scheme, "https") { + return true + } + if !strings.EqualFold(parsed.Scheme, "http") { + return false + } + host := strings.Trim(strings.ToLower(parsed.Hostname()), "[]") + return host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +func projectEndpointIsSafe(raw string) bool { + trimmed := strings.TrimSpace(raw) + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return false + } + if parsed.User != nil || parsed.ForceQuery || parsed.RawQuery != "" || parsed.Fragment != "" || + strings.Contains(trimmed, "#") { + return false + } + if strings.EqualFold(parsed.Scheme, "https") { + return true + } + if !strings.EqualFold(parsed.Scheme, "http") { + return false + } + host := strings.Trim(strings.ToLower(parsed.Hostname()), "[]") + return host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +func exactlyOneFoundryAuth(cfg config) bool { + hasKey := strings.TrimSpace(cfg.foundryAuth) != "" + hasBearer := strings.TrimSpace(cfg.authBearer) != "" + return hasKey != hasBearer +} + +func (s *server) validateStartRequest(req harness.StartTurnRequest) error { + if s.cfg.configError != "" { + return errors.New(s.cfg.configError) + } + if _, err := s.responsesEndpoint(); err != nil { + return err + } + if !exactlyOneFoundryAuth(s.cfg) { + return fmt.Errorf("exactly one Foundry auth mode is required") + } + if req.ToolExecutionMode == harness.ToolExecutionModeBrokered { + if len(s.cfg.brokeredToolClasses) == 0 { + return fmt.Errorf("brokered mode is not enabled for this hosted Responses adapter") + } + for _, tool := range req.Input.Tools { + if !s.supportsBrokeredClass(tool.BrokeredClass) { + return fmt.Errorf( + "brokered tool %q class %q is not advertised by this adapter", + tool.Name, + tool.BrokeredClass, + ) + } + } + } + return nil +} + +func (s *server) supportsBrokeredClass(class harness.BrokeredToolClass) bool { + return slices.Contains(s.cfg.brokeredToolClasses, class) +} + +func findToolDefinition(definitions []harness.ToolDefinition, name string) (harness.ToolDefinition, bool) { + name = strings.TrimSpace(name) + for _, definition := range definitions { + if strings.TrimSpace(definition.Name) == name { + return definition, true + } + } + return harness.ToolDefinition{}, false +} + +func normalizeResponsesToolArguments(raw json.RawMessage) (json.RawMessage, error) { + if len(raw) == 0 || strings.TrimSpace(string(raw)) == "null" { + return json.RawMessage(`{}`), nil + } + var encoded string + if err := json.Unmarshal(raw, &encoded); err == nil { + encoded = strings.TrimSpace(encoded) + if encoded == "" { + return json.RawMessage(`{}`), nil + } + return normalizeResponsesToolArguments(json.RawMessage(encoded)) + } + compacted, err := compactRawJSON(raw) + if err != nil { + return nil, fmt.Errorf("hosted response function_call arguments must be a valid JSON object") + } + trimmed := bytes.TrimSpace(compacted) + if len(trimmed) < 2 || trimmed[0] != '{' || trimmed[len(trimmed)-1] != '}' { + return nil, fmt.Errorf("hosted response function_call arguments must be a valid JSON object") + } + return json.RawMessage(append([]byte(nil), trimmed...)), nil +} + +func responsesMessageText(output []responsesOutput) string { + parts := []string{} + for _, item := range output { + typeName := strings.TrimSpace(item.Type) + if typeName == "message" || typeName == "output_text" || typeName == "text" || typeName == "" { + if text := outputItemText(item); text != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, "\n") +} + +func outputItemText(item responsesOutput) string { + if strings.TrimSpace(item.Text) != "" { + return strings.TrimSpace(item.Text) + } + switch content := item.Content.(type) { + case string: + return strings.TrimSpace(content) + case []any: + parts := []string{} + for _, entry := range content { + if m, ok := entry.(map[string]any); ok { + if text, ok := m["text"].(string); ok && strings.TrimSpace(text) != "" { + parts = append(parts, strings.TrimSpace(text)) + } + if textMap, ok := m["text"].(map[string]any); ok { + if value, ok := textMap["value"].(string); ok && strings.TrimSpace(value) != "" { + parts = append(parts, strings.TrimSpace(value)) + } + } + } + } + return strings.Join(parts, "\n") + } + return "" +} + +func isFailureStatus(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "failed", "cancelled", "expired", "incomplete": + return true + default: + return false + } +} + +func (s *server) updateTurnSessionLocked(turn *turnState) { + session := s.runtimeSessions[turn.request.RuntimeSessionID] + if session.ID != "" { + turn.foundrySessionID = session.ID + } +} + +func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { + if turn.completed { + return + } + s.appendFrameLocked( + turn, + harness.FrameTurnFailed, + "foundry hosted response failed", + func(f *harness.HarnessEventFrame) { + f.Failed = &harness.TurnFailed{Reason: reason, Message: msg} + f.Error = &harness.ErrorInfo{Code: reason, Message: msg} + }, + ) + turn.completed = true + s.scheduleTurnCleanupLocked(turn) +} + +func (s *server) scheduleTurnCleanupLocked(turn *turnState) { + turnID := turn.request.TurnID + retention := s.cfg.stateRetention + if retention <= 0 { + retention = defaultStateRetention + } + time.AfterFunc(retention, func() { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.turns, turnID) + cutoff := time.Now().UTC().Add(-retention) + for sessionID, session := range s.runtimeSessions { + if session.LastSeen.Before(cutoff) { + delete(s.runtimeSessions, sessionID) + } + } + }) +} + +func (s *server) appendFrameLocked( + turn *turnState, + typ harness.FrameType, + summary string, + mutate func(*harness.HarnessEventFrame), +) { + seq := int64(len(turn.frames) + 1) + frame := harness.HarnessEventFrame{ + Version: harness.ProtocolVersion, + Type: typ, + RuntimeSessionID: turn.request.RuntimeSessionID, + TurnID: turn.request.TurnID, + CorrelationID: turn.request.CorrelationID, + Seq: seq, + CreatedAt: time.Now().UTC(), + Summary: summary, + Metadata: map[string]string{"backend": "foundry-responses"}, + } + if mutate != nil { + mutate(&frame) + } + turn.frames = append(turn.frames, frame) +} + +func (s *server) authorized(w http.ResponseWriter, r *http.Request) bool { + if s.cfg.adapterBearer == "" { + harness.WriteError(w, http.StatusUnauthorized, "adapter bearer token is required") + return false + } + got := strings.TrimSpace(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")) + if got == "" || subtle.ConstantTimeCompare([]byte(got), []byte(s.cfg.adapterBearer)) != 1 { + harness.WriteError(w, http.StatusUnauthorized, "unauthorized") + return false + } + return true +} + +func sameStartTurnRequest(existing, retry harness.StartTurnRequest) bool { + return reflect.DeepEqual(existing, retry) +} + +func sameContinueIdentity(start harness.StartTurnRequest, cont harness.ContinueTurnRequest) bool { + return start.Namespace == cont.Namespace && + start.TaskName == cont.TaskName && + start.SessionName == cont.SessionName && + start.RuntimeSessionID == cont.RuntimeSessionID && + start.TurnID == cont.TurnID && + start.CorrelationID == cont.CorrelationID +} + +func startTurnResponse(req harness.StartTurnRequest, eventsPath string) harness.StartTurnResponse { + return harness.StartTurnResponse{ + Version: harness.ProtocolVersion, + Accepted: true, + RuntimeSessionID: req.RuntimeSessionID, + TurnID: req.TurnID, + CorrelationID: req.CorrelationID, + EventStreamPath: eventsPath, + } +} + +func continueResponse(req harness.ContinueTurnRequest, msg string) harness.ContinueTurnResponse { + return harness.ContinueTurnResponse{ + Version: harness.ProtocolVersion, + Accepted: true, + RuntimeSessionID: req.RuntimeSessionID, + TurnID: req.TurnID, + CorrelationID: req.CorrelationID, + Message: msg, + } +} + +func parseAfterSeq(value string) int64 { + parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil || parsed < 0 { + return 0 + } + return parsed +} + +func parseBrokeredToolClasses(raw string) ([]harness.BrokeredToolClass, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + classes := []harness.BrokeredToolClass{} + seen := map[harness.BrokeredToolClass]struct{}{} + for part := range strings.SplitSeq(raw, ",") { + class := harness.BrokeredToolClass(strings.TrimSpace(part)) + if class == "" { + continue + } + switch class { + case harness.BrokeredToolClassRead, harness.BrokeredToolClassWrite: + default: + return nil, fmt.Errorf( + "unsupported %s value %q; supported values are read,write", + envBrokeredToolClasses, + class, + ) + } + if _, ok := seen[class]; ok { + continue + } + seen[class] = struct{}{} + classes = append(classes, class) + } + return classes, nil +} + +func firstNonBlank(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func parseDurationEnv(name string, fallback time.Duration) time.Duration { + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + if parsed, err := time.ParseDuration(value); err == nil && parsed > 0 { + return parsed + } + } + return fallback +} + +func sanitizeEndpoint(raw string) string { + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil { + return raw + } + u.User = nil + q := u.Query() + for key := range q { + if key != "api-version" { + q.Del(key) + } + } + u.RawQuery = q.Encode() + u.Fragment = "" + return u.String() +} diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go new file mode 100644 index 000000000..bfdb1b21f --- /dev/null +++ b/examples/harness/foundry-responses/main_test.go @@ -0,0 +1,1102 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/orka-agents/orka/internal/harness" + "github.com/orka-agents/orka/internal/harness/conformance" +) + +const fakeSessionID = "session-1" + +func TestResponsesAdapterObservedTurnCompletes(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + + request := responsesStartTurnRequest("foundry-observed") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + var frames []harness.HarnessEventFrame + if err := client.StreamFrames(context.Background(), request.TurnID, 0, func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + return nil + }); err != nil { + t.Fatalf("StreamFrames: %v", err) + } + if !hasFrameType(frames, harness.FrameTurnStarted) || !hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want started and completed", frames) + } + if got := frames[len(frames)-1].Completed.Result; got != "foundry final answer" { + t.Fatalf("result = %q", got) + } + if foundry.sawRequestLevelTools.Load() { + t.Fatalf("hosted Responses request included request-level tools") + } + if got := foundry.requestHeader(0).Get("Foundry-Features"); got != "HostedAgents=V1Preview" { + t.Fatalf("Foundry-Features = %q, want HostedAgents=V1Preview", got) + } + assertJSONFileEqual(t, "testdata/golden/01_initial_hosted_request.json", foundry.requestBody(0)) +} + +func TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + + request := responsesStartTurnRequest("foundry-brokered") + request.ToolExecutionMode = harness.ToolExecutionModeBrokered + request.Input.Tools = []harness.ToolDefinition{{ + Name: "support-ticket-lookup", + Description: "Look up support ticket", + BrokeredClass: harness.BrokeredToolClassRead, + Parameters: json.RawMessage(`{"type":"object","properties":{"incident":{"type":"string"}}}`), + }} + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + if foundry.sawRequestLevelTools.Load() { + t.Fatalf("hosted Responses request included request-level tools") + } + assertJSONFileEqual(t, "testdata/golden/01_initial_hosted_request.json", foundry.requestBody(0)) + + var frames []harness.HarnessEventFrame + if err := client.StreamFrames(context.Background(), request.TurnID, 0, func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + return nil + }); err != nil { + t.Fatalf("StreamFrames before continue: %v", err) + } + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + if requested.ToolName != "support-ticket-lookup" || requested.ToolCallID != "call-1" { + t.Fatalf("tool request = %#v", requested) + } + assertJSONFileEqual(t, "testdata/golden/03_tool_call_requested_frame.json", scrubFrameForGolden(*requested)) + + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + assertJSONFileEqual(t, "testdata/golden/04_orka_continue_request.json", continueRequest) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn: %v", err) + } + assertJSONFileEqual(t, "testdata/golden/05_hosted_continuation_request.json", foundry.requestBody(1)) + if got := foundry.requestHeader(1).Get("x-agent-session-id"); got != fakeSessionID { + t.Fatalf("continuation x-agent-session-id = %q, want session-1", got) + } + + frames = nil + if err := client.StreamFrames( + context.Background(), + request.TurnID, + requested.Seq, + func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + return nil + }, + ); err != nil { + t.Fatalf("StreamFrames after continue: %v", err) + } + if !hasFrameType(frames, harness.FrameToolResultReceived) || !hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want tool result and completion", frames) + } + if foundry.postCount.Load() != 2 { + t.Fatalf("hosted post count = %d, want 2", foundry.postCount.Load()) + } +} + +func TestResponsesAdapterRuntimeSessionHeaderReuse(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + + first := responsesStartTurnRequest("foundry-session-one") + if _, err := client.StartTurn(context.Background(), first); err != nil { + t.Fatalf("StartTurn first: %v", err) + } + second := responsesStartTurnRequest("foundry-session-two") + second.RuntimeSessionID = first.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), second); err != nil { + t.Fatalf("StartTurn second: %v", err) + } + if got := foundry.requestHeader(1).Get("x-agent-session-id"); got != fakeSessionID { + t.Fatalf("same runtimeSessionID header = %q, want session-1", got) + } + if got := requestMap(t, foundry.requestBody(1))["agent_session_id"]; got != fakeSessionID { + t.Fatalf("same runtimeSessionID body agent_session_id = %#v, want %q", got, fakeSessionID) + } + + third := responsesStartTurnRequest("foundry-session-three") + if _, err := client.StartTurn(context.Background(), third); err != nil { + t.Fatalf("StartTurn third: %v", err) + } + if got := foundry.requestHeader(2).Get("x-agent-session-id"); got != "" { + t.Fatalf("new runtimeSessionID header = %q, want empty", got) + } +} + +func TestResponsesAdapterPassesObservedConformanceByDefault(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) + defer adapter.Close() + + result := conformance.Check(context.Background(), conformance.Target{ + BaseURL: adapter.URL, + BearerToken: "adapter-auth-value", + ControlTimeout: 2 * time.Second, + ProbeTurn: true, + RequireAuth: true, + }) + if !result.Passed { + t.Fatalf("observed conformance failed: %s failures=%v", result.Message, result.Failures) + } + caps := result.ObservedCapabilities + if caps == nil { + t.Fatal("ObservedCapabilities = nil") + } + if !reflect.DeepEqual(caps.ToolExecutionModes, []harness.ToolExecutionMode{harness.ToolExecutionModeObserved}) { + t.Fatalf("ToolExecutionModes = %#v, want observed only", caps.ToolExecutionModes) + } + if len(caps.BrokeredToolClasses) != 0 { + t.Fatalf("BrokeredToolClasses = %#v, want none", caps.BrokeredToolClasses) + } + if caps.SupportsContinuation { + t.Fatal("SupportsContinuation = true, want false when no brokered classes are configured") + } +} + +func TestResponsesAdapterPassesBrokeredReadConformance(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "conformance_read"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + defer adapter.Close() + + result := conformance.Check(context.Background(), conformance.Target{ + BaseURL: adapter.URL, + BearerToken: "adapter-auth-value", + ControlTimeout: 2 * time.Second, + ProbeBrokeredRead: true, + RequireAuth: true, + }) + if !result.Passed { + t.Fatalf("brokered read conformance failed: %s failures=%v", result.Message, result.Failures) + } + if foundry.sawRequestLevelTools.Load() { + t.Fatalf("hosted Responses request included request-level tools") + } +} + +func TestResponsesAdapterPassesBrokeredWriteConformance(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "conformance_write"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassWrite}, + ) + defer adapter.Close() + + result := conformance.Check(context.Background(), conformance.Target{ + BaseURL: adapter.URL, + BearerToken: "adapter-auth-value", + ControlTimeout: 2 * time.Second, + ProbeBrokeredWrite: true, + RequireAuth: true, + }) + if !result.Passed { + t.Fatalf("brokered write conformance failed: %s failures=%v", result.Message, result.Failures) + } +} + +func TestResponsesAdapterRejectsUnknownToolBeforeOrkaExecution(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "unknown-tool"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-unknown-tool") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + if hasFrameType(frames, harness.FrameToolCallRequested) { + t.Fatalf("frames = %#v, should not request Orka execution for an unknown tool", frames) + } + failed := findFrame(frames, harness.FrameTurnFailed) + if failed == nil || !strings.Contains(failed.Failed.Message, "did not expose") { + t.Fatalf("failed frame = %#v, want unknown-tool rejection", failed) + } +} + +func TestResponsesAdapterRejectsMalformedArguments(t *testing.T) { + foundry := newFakeResponses( + t, + fakeResponsesConfig{scenario: "malformed_arguments", toolName: "support-ticket-lookup"}, + ) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-malformed-args") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + if hasFrameType(frames, harness.FrameToolCallRequested) { + t.Fatalf("frames = %#v, should not request Orka execution for malformed arguments", frames) + } + failed := findFrame(frames, harness.FrameTurnFailed) + if failed == nil || !strings.Contains(failed.Failed.Message, "arguments") { + t.Fatalf("failed frame = %#v, want malformed arguments rejection", failed) + } +} + +func TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "multiple_calls", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-multiple") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requests := findFrames(frames, harness.FrameToolCallRequested) + if len(requests) != 2 { + t.Fatalf("tool request frames = %#v, want 2", requests) + } + continueRequest := harness.ContinueTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: request.Namespace, + TaskName: request.TaskName, + SessionName: request.SessionName, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + CorrelationID: request.CorrelationID, + ToolResults: []harness.ToolCallResult{ + toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true,"call":1}`), nil), + toolResultForRequest(request, "call-2", true, json.RawMessage(`{"success":true,"call":2}`), nil), + }, + } + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn: %v", err) + } + continuation := requestMap(t, foundry.requestBody(1)) + items, ok := continuation["input"].([]any) + if !ok || len(items) != 2 { + t.Fatalf("continuation input = %#v, want two function_call_output items", continuation["input"]) + } + if got := continuation["agent_session_id"]; got != fakeSessionID { + t.Fatalf("agent_session_id = %#v, want %q", got, fakeSessionID) + } + frames = streamAllFrames(t, client, request.TurnID) + if !hasFrameType(frames, harness.FrameToolResultReceived) || !hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want tool results and completion", frames) + } +} + +func TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-duplicate") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn first: %v", err) + } + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn duplicate: %v", err) + } + if foundry.postCount.Load() != 2 { + t.Fatalf("hosted post count after duplicate = %d, want 2", foundry.postCount.Load()) + } + conflicting := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":false}`)) + if _, err := client.ContinueTurn(context.Background(), conflicting); err == nil { + t.Fatalf("conflicting duplicate continue succeeded, want conflict") + } +} + +func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{ + scenario: "function_call", + toolName: "support-ticket-lookup", + continuationStatus: http.StatusInternalServerError, + }) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-continuation-failure") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err == nil { + t.Fatalf("ContinueTurn succeeded, want hosted continuation failure") + } + if foundry.postCount.Load() != 2 { + t.Fatalf("hosted post count after failed continue = %d, want 2", foundry.postCount.Load()) + } + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("duplicate ContinueTurn after fail-closed terminal state: %v", err) + } + if foundry.postCount.Load() != 2 { + t.Fatalf("hosted post count after duplicate = %d, want no second continuation", foundry.postCount.Load()) + } + frames = streamAllFrames(t, client, request.TurnID) + failed := findFrame(frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_continuation_unknown" { + t.Fatalf("failed frame = %#v, want fail-closed continuation failure", failed) + } +} + +func TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + s := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.endpoint(), + foundryAuth: "foundry-auth-value", + requestTimeout: 2 * time.Second, + stateRetention: time.Minute, + maxApprovalWait: 30 * time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + adapter := httptest.NewServer(s.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + + caps, err := client.Capabilities(context.Background()) + if err != nil { + t.Fatalf("Capabilities: %v", err) + } + if caps.MaxTurnSeconds < int((30*time.Minute + 2*time.Second).Seconds()) { + t.Fatalf("MaxTurnSeconds = %d, want approval wait included", caps.MaxTurnSeconds) + } +} + +func TestResponsesAdapterStateLossContinueFailsSafely(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-state-loss") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + + restarted := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + restartedClient, err := harness.NewClient( + restarted.URL, + harness.WithBearerToken("adapter-auth-value"), + harness.WithControlTimeout(2*time.Second), + ) + if err != nil { + t.Fatalf("NewClient restarted: %v", err) + } + _, err = restartedClient.ContinueTurn( + context.Background(), + goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)), + ) + if err == nil || !strings.Contains(err.Error(), "turn not found") { + t.Fatalf("restart continue error = %v, want clear turn not found", err) + } + if foundry.postCount.Load() != 1 { + t.Fatalf( + "hosted post count after state-loss continue = %d, want no duplicate continuation", + foundry.postCount.Load(), + ) + } +} + +func TestResponsesEndpointSafety(t *testing.T) { + tests := []struct { + name string + endpoint string + want bool + }{ + { + name: "https responses", + endpoint: "https://example.openai.azure.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", + want: true, + }, + { + name: "loopback http", + endpoint: "http://127.0.0.1:8080/agents/a/endpoint/protocols/openai/responses?api-version=v1", + want: true, + }, + { + name: "http non-loopback", + endpoint: "http://example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", + want: false, + }, + { + name: "userinfo", + endpoint: "https://user:pass@example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", + want: false, + }, + { + name: "fragment", + endpoint: "https://example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1#fragment", + want: false, + }, + { + name: "secret query", + endpoint: "https://example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1&unsafe=x", + want: false, + }, + {name: "not responses", endpoint: "https://example.com/threads", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := responsesEndpointIsSafe(tt.endpoint); got != tt.want { + t.Fatalf("responsesEndpointIsSafe(%q) = %v, want %v", tt.endpoint, got, tt.want) + } + }) + } +} + +func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { + server := newServer( + config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, + &http.Client{Timeout: time.Second}, + ) + request := brokeredReadRequest("foundry-brokered") + turn := &turnState{ + request: request, + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + var functionCall responsesResponse + decodeFixtureInto(t, "testdata/golden/02_function_call_response.json", &functionCall) + server.handleResponsesResponse(turn, functionCall) + requested := findFrame(turn.frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request from fixture", turn.frames) + } + assertJSONFileEqual(t, "testdata/golden/03_tool_call_requested_frame.json", scrubFrameForGolden(*requested)) + + finalTurn := &turnState{ + request: responsesStartTurnRequest("foundry-final"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + var finalMessage responsesResponse + decodeFixtureInto(t, "testdata/golden/06_final_message_response.json", &finalMessage) + server.handleResponsesResponse(finalTurn, finalMessage) + completed := findFrame(finalTurn.frames, harness.FrameTurnCompleted) + if completed == nil || completed.Completed.Result != "foundry final answer" { + t.Fatalf("completed frame = %#v, want final answer", completed) + } + + multipleTurn := &turnState{ + request: request, + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(multipleTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + var multiple responsesResponse + decodeFixtureInto(t, "testdata/golden/10_multiple_calls_response.json", &multiple) + server.handleResponsesResponse(multipleTurn, multiple) + requests := findFrames(multipleTurn.frames, harness.FrameToolCallRequested) + if len(requests) != 2 { + t.Fatalf("frames = %#v, want two tool requests from multiple-call fixture", multipleTurn.frames) + } +} + +func TestResponsesPreservesNumericJSONTokens(t *testing.T) { + args, err := normalizeResponsesToolArguments(json.RawMessage(`{"id":9007199254740993}`)) + if err != nil { + t.Fatalf("normalizeResponsesToolArguments: %v", err) + } + if got := string(args); got != `{"id":9007199254740993}` { + t.Fatalf("arguments = %s, want numeric token preserved", got) + } + result := baseToolResult("call-2", true, json.RawMessage(`{"id":9007199254740993}`), nil) + payload, err := canonicalToolResultOutput(result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + want := `{"approved":true,"output":{"id":9007199254740993}}` + if payload != want { + t.Fatalf("payload = %s, want %s", payload, want) + } +} + +func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: responsesStartTurnRequest("foundry-failed-status"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-failed", + Status: "failed", + Output: []responsesOutput{{ + Type: "message", + Content: "partial text should not be completed", + }}, + }) + if hasFrameType(turn.frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, failed response should not complete", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_failed" { + t.Fatalf("failed frame = %#v, want foundry_failed", failed) + } +} + +func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: brokeredReadRequest("foundry-missing-id"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + Output: []responsesOutput{{ + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }}, + }) + if hasFrameType(turn.frames, harness.FrameToolCallRequested) { + t.Fatalf("frames = %#v, should not request Orka execution without response id", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_response_id_missing" { + t.Fatalf("failed frame = %#v, want missing response id failure", failed) + } +} + +func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { + tests := []struct { + name string + result harness.ToolCallResult + fixture string + }{ + { + name: "approval declined", + result: baseToolResult("call-1", false, nil, &harness.ErrorInfo{ + Code: "approval_declined", Message: "human declined", + }), + fixture: "testdata/golden/07_approval_declined_output.json", + }, + { + name: "policy rejection", + result: baseToolResult("call-1", false, nil, &harness.ErrorInfo{ + Code: "tool_policy_rejected", Message: "tool is not allowed", + }), + fixture: "testdata/golden/08_tool_policy_rejection_output.json", + }, + { + name: "execution failure", + result: baseToolResult("call-1", true, nil, &harness.ErrorInfo{ + Code: "tool_execution_failed", Message: "downstream failed", + }), + fixture: "testdata/golden/09_tool_execution_failure_output.json", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + outputs, _, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) + if err != nil { + t.Fatalf("functionCallOutputs: %v", err) + } + assertJSONFileEqual(t, tt.fixture, outputs[0]) + }) + } +} + +type fakeResponsesConfig struct { + scenario string + toolName string + continuationStatus int +} + +type fakeResponses struct { + *httptest.Server + cfg fakeResponsesConfig + mu sync.Mutex + requests []json.RawMessage + headers []http.Header + postCount atomic.Int32 + sawRequestLevelTools atomic.Bool +} + +func newFakeResponses(t *testing.T, cfg fakeResponsesConfig) *fakeResponses { + t.Helper() + if cfg.scenario == "" { + cfg.scenario = "observed" + } + if cfg.toolName == "" { + cfg.toolName = "support-ticket-lookup" + } + f := &fakeResponses{cfg: cfg} + mux := http.NewServeMux() + mux.HandleFunc( + "/agents/test-agent/endpoint/protocols/openai/responses", + func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + if got := r.URL.Query().Get("api-version"); got != "v1" { + http.Error(w, "missing api-version", http.StatusBadRequest) + return + } + body, _ := ioReadAll(r.Body) + var decoded map[string]any + _ = json.Unmarshal(body, &decoded) + if _, ok := decoded["tools"]; ok { + f.sawRequestLevelTools.Store(true) + } + f.mu.Lock() + f.requests = append(f.requests, append(json.RawMessage(nil), body...)) + f.headers = append(f.headers, r.Header.Clone()) + f.mu.Unlock() + f.postCount.Add(1) + if _, continuing := decoded["previous_response_id"]; continuing { + if got := r.Header.Get("x-agent-session-id"); got != fakeSessionID { + http.Error(w, "missing session header", http.StatusBadRequest) + return + } + if got := decoded["agent_session_id"]; got != fakeSessionID { + http.Error(w, "missing body session", http.StatusBadRequest) + return + } + if f.cfg.continuationStatus != 0 { + http.Error(w, "continuation failed", f.cfg.continuationStatus) + return + } + writeJSON(w, finalResponsesMessage()) + return + } + w.Header().Set("x-agent-session-id", fakeSessionID) + switch f.cfg.scenario { + case "observed": + writeJSON(w, finalResponsesMessage()) + case "function_call": + writeJSON(w, functionCallResponse(f.cfg.toolName)) + case "malformed_arguments": + writeJSON( + w, + map[string]any{ + "id": "resp-1", + "output": []any{ + map[string]any{ + "type": "function_call", + "call_id": "call-1", + "name": f.cfg.toolName, + "arguments": "not-json", + }, + }, + }, + ) + case "multiple_calls": + writeJSON(w, multipleCallsResponse()) + default: + http.Error(w, "unknown scenario", http.StatusInternalServerError) + } + }, + ) + f.Server = httptest.NewServer(mux) + t.Cleanup(f.Close) + return f +} + +func (f *fakeResponses) endpoint() string { + return f.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" +} + +func (f *fakeResponses) requestBody(i int) json.RawMessage { + f.mu.Lock() + defer f.mu.Unlock() + return append(json.RawMessage(nil), f.requests[i]...) +} + +func (f *fakeResponses) requestHeader(i int) http.Header { + f.mu.Lock() + defer f.mu.Unlock() + return f.headers[i].Clone() +} + +func functionCallResponse(toolName string) map[string]any { + return map[string]any{ + "id": "resp-1", + "agent_session_id": fakeSessionID, + "output": []any{ + map[string]any{ + "type": "function_call", + "call_id": "call-1", + "name": toolName, + "arguments": `{"incident":"inc-1"}`, + }, + }, + } +} + +func multipleCallsResponse() map[string]any { + return map[string]any{"id": "resp-1", "agent_session_id": fakeSessionID, "output": []any{ + map[string]any{ + "type": "function_call", + "call_id": "call-1", + "name": "support-ticket-lookup", + "arguments": `{"incident":"inc-1"}`, + }, + map[string]any{ + "type": "function_call", + "call_id": "call-2", + "name": "support-ticket-lookup", + "arguments": `{"incident":"inc-2"}`, + }, + }} +} + +func finalResponsesMessage() map[string]any { + return map[string]any{ + "id": "resp-2", + "agent_session_id": fakeSessionID, + "output": []any{ + map[string]any{ + "type": "message", + "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": "foundry final answer"}}, + }, + }, + } +} + +func newTestResponsesAdapter(t *testing.T, endpoint string, classes []harness.BrokeredToolClass) *httptest.Server { + t.Helper() + s := newServer(config{ + addr: ":0", + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: endpoint, + foundryAuth: "foundry-auth-value", + apiVersion: "v1", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: append([]harness.BrokeredToolClass(nil), classes...), + }, &http.Client{Timeout: time.Second}) + adapter := httptest.NewServer(s.handler()) + t.Cleanup(adapter.Close) + return adapter +} + +func newHarnessClient(t *testing.T, adapter *httptest.Server) *harness.Client { + t.Helper() + client, err := harness.NewClient( + adapter.URL, + harness.WithBearerToken("adapter-auth-value"), + harness.WithControlTimeout(2*time.Second), + ) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + return client +} + +func responsesStartTurnRequest(name string) harness.StartTurnRequest { + return harness.StartTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: "default", + TaskName: name, + SessionName: name, + RuntimeSessionID: harness.RuntimeSessionID(name + "-runtime"), + TurnID: harness.HarnessTurnID(name + "-turn"), + CorrelationID: name + "-corr", + Deadline: time.Now().UTC().Add(time.Minute), + AuthIdentity: harness.AuthIdentity{Subject: "task:default/" + name}, + ToolExecutionMode: harness.ToolExecutionModeObserved, + Input: harness.TurnInput{Prompt: "Investigate incident"}, + } +} + +func brokeredReadRequest(name string) harness.StartTurnRequest { + request := responsesStartTurnRequest(name) + request.ToolExecutionMode = harness.ToolExecutionModeBrokered + request.Input.Tools = []harness.ToolDefinition{{ + Name: "support-ticket-lookup", + Description: "Look up support ticket", + BrokeredClass: harness.BrokeredToolClassRead, + Parameters: json.RawMessage(`{"type":"object","properties":{"incident":{"type":"string"}}}`), + }} + return request +} + +func goldenContinueRequest( + request harness.StartTurnRequest, + callID string, + output json.RawMessage, +) harness.ContinueTurnRequest { + return harness.ContinueTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: request.Namespace, + TaskName: request.TaskName, + SessionName: request.SessionName, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + CorrelationID: request.CorrelationID, + ToolResults: []harness.ToolCallResult{toolResultForRequest(request, callID, true, output, nil)}, + } +} + +func toolResultForRequest( + request harness.StartTurnRequest, + callID string, + approved bool, + output json.RawMessage, + errInfo *harness.ErrorInfo, +) harness.ToolCallResult { + return harness.ToolCallResult{ + Version: harness.ProtocolVersion, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + ToolCallID: callID, + IdempotencyKey: harness.ToolRequestIdempotencyKey(request.RuntimeSessionID, request.TurnID, callID), + Approved: approved, + Output: output, + Error: errInfo, + } +} + +func baseToolResult( + callID string, + approved bool, + output json.RawMessage, + errInfo *harness.ErrorInfo, +) harness.ToolCallResult { + return harness.ToolCallResult{ + Version: harness.ProtocolVersion, + RuntimeSessionID: "foundry-brokered-runtime", + TurnID: "foundry-brokered-turn", + ToolCallID: callID, + IdempotencyKey: harness.ToolRequestIdempotencyKey( + "foundry-brokered-runtime", + "foundry-brokered-turn", + callID, + ), + Approved: approved, + Output: output, + Error: errInfo, + } +} + +func streamAllFrames( + t *testing.T, + client *harness.Client, + turnID harness.HarnessTurnID, +) []harness.HarnessEventFrame { + t.Helper() + var frames []harness.HarnessEventFrame + if err := client.StreamFrames(context.Background(), turnID, 0, func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + return nil + }); err != nil { + t.Fatalf("StreamFrames: %v", err) + } + return frames +} + +func hasFrameType(frames []harness.HarnessEventFrame, typ harness.FrameType) bool { + return findFrame(frames, typ) != nil +} + +func findFrame(frames []harness.HarnessEventFrame, typ harness.FrameType) *harness.HarnessEventFrame { + for i := range frames { + if frames[i].Type == typ { + return &frames[i] + } + } + return nil +} + +func findFrames(frames []harness.HarnessEventFrame, typ harness.FrameType) []harness.HarnessEventFrame { + out := []harness.HarnessEventFrame{} + for _, frame := range frames { + if frame.Type == typ { + out = append(out, frame) + } + } + return out +} + +func scrubFrameForGolden(frame harness.HarnessEventFrame) map[string]any { + encoded, _ := json.Marshal(frame) + var decoded map[string]any + _ = json.Unmarshal(encoded, &decoded) + delete(decoded, "createdAt") + return decoded +} + +func assertJSONFileEqual(t *testing.T, path string, actual any) { + t.Helper() + expectedBytes, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + expected := decodeJSONForCompare(t, expectedBytes) + actualBytes, err := json.Marshal(actual) + if err != nil { + t.Fatalf("marshal actual: %v", err) + } + actualValue := decodeJSONForCompare(t, actualBytes) + if !reflect.DeepEqual(expected, actualValue) { + expectedPretty, _ := json.MarshalIndent(expected, "", " ") + actualPretty, _ := json.MarshalIndent(actualValue, "", " ") + t.Fatalf("JSON mismatch for %s\nexpected: %s\nactual: %s", path, expectedPretty, actualPretty) + } +} + +func requestMap(t *testing.T, data json.RawMessage) map[string]any { + t.Helper() + decoded, ok := decodeJSONForCompare(t, data).(map[string]any) + if !ok { + t.Fatalf("request body is not a JSON object: %s", string(data)) + } + return decoded +} + +func decodeJSONForCompare(t *testing.T, data []byte) any { + t.Helper() + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + t.Fatalf("decode JSON: %v\n%s", err, string(data)) + } + return value +} + +func decodeFixtureInto(t *testing.T, path string, out any) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + if err := json.Unmarshal(data, out); err != nil { + t.Fatalf("decode fixture %s: %v", path, err) + } +} + +func writeJSON(w http.ResponseWriter, value any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(value) +} + +func ioReadAll(r io.Reader) ([]byte, error) { + return io.ReadAll(r) +} diff --git a/examples/harness/foundry-responses/testdata/golden/01_initial_hosted_request.json b/examples/harness/foundry-responses/testdata/golden/01_initial_hosted_request.json new file mode 100644 index 000000000..e717d54e2 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/01_initial_hosted_request.json @@ -0,0 +1,3 @@ +{ + "input": "Investigate incident" +} diff --git a/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json b/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json new file mode 100644 index 000000000..45aaeea30 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json @@ -0,0 +1,12 @@ +{ + "id": "resp-1", + "agent_session_id": "session-1", + "output": [ + { + "type": "function_call", + "call_id": "call-1", + "name": "support-ticket-lookup", + "arguments": "{\"incident\":\"inc-1\"}" + } + ] +} diff --git a/examples/harness/foundry-responses/testdata/golden/03_tool_call_requested_frame.json b/examples/harness/foundry-responses/testdata/golden/03_tool_call_requested_frame.json new file mode 100644 index 000000000..b2bc39125 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/03_tool_call_requested_frame.json @@ -0,0 +1,13 @@ +{ + "version": "orka.harness.v1", + "type": "ToolCallRequested", + "runtimeSessionID": "foundry-brokered-runtime", + "turnID": "foundry-brokered-turn", + "correlationID": "foundry-brokered-corr", + "seq": 2, + "summary": "foundry hosted tool call requested", + "content": {"incident":"inc-1"}, + "toolName": "support-ticket-lookup", + "toolCallID": "call-1", + "metadata": {"backend":"foundry-responses"} +} diff --git a/examples/harness/foundry-responses/testdata/golden/04_orka_continue_request.json b/examples/harness/foundry-responses/testdata/golden/04_orka_continue_request.json new file mode 100644 index 000000000..8ba6d6990 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/04_orka_continue_request.json @@ -0,0 +1,20 @@ +{ + "version": "orka.harness.v1", + "namespace": "default", + "taskName": "foundry-brokered", + "sessionName": "foundry-brokered", + "runtimeSessionID": "foundry-brokered-runtime", + "turnID": "foundry-brokered-turn", + "correlationID": "foundry-brokered-corr", + "toolResults": [ + { + "version": "orka.harness.v1", + "runtimeSessionID": "foundry-brokered-runtime", + "turnID": "foundry-brokered-turn", + "toolCallID": "call-1", + "idempotencyKey": "foundry-brokered-runtime:foundry-brokered-turn:call-1", + "approved": true, + "output": {"success":true} + } + ] +} diff --git a/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json b/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json new file mode 100644 index 000000000..2107d892c --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json @@ -0,0 +1,12 @@ +{ + "previous_response_id": "resp-1", + "agent_session_id": "session-1", + "input": [ + { + "type": "function_call_output", + "call_id": "call-1", + "output": "{\"approved\":true,\"output\":{\"success\":true}}", + "status": "completed" + } + ] +} diff --git a/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json b/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json new file mode 100644 index 000000000..9f9ce60af --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json @@ -0,0 +1,13 @@ +{ + "id": "resp-2", + "agent_session_id": "session-1", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type":"output_text","text":"foundry final answer"} + ] + } + ] +} diff --git a/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json b/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json new file mode 100644 index 000000000..645b97fd1 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json @@ -0,0 +1,6 @@ +{ + "type": "function_call_output", + "call_id": "call-1", + "output": "{\"approved\":false,\"error\":{\"code\":\"approval_declined\",\"message\":\"human declined\"}}", + "status": "completed" +} diff --git a/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json b/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json new file mode 100644 index 000000000..6ae0eeee9 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json @@ -0,0 +1,6 @@ +{ + "type": "function_call_output", + "call_id": "call-1", + "output": "{\"approved\":false,\"error\":{\"code\":\"tool_policy_rejected\",\"message\":\"tool is not allowed\"}}", + "status": "completed" +} diff --git a/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json b/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json new file mode 100644 index 000000000..039e1277c --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json @@ -0,0 +1,6 @@ +{ + "type": "function_call_output", + "call_id": "call-1", + "output": "{\"approved\":false,\"error\":{\"code\":\"tool_execution_failed\",\"message\":\"downstream failed\"}}", + "status": "completed" +} diff --git a/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json b/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json new file mode 100644 index 000000000..524114c33 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json @@ -0,0 +1,8 @@ +{ + "id": "resp-1", + "agent_session_id": "session-1", + "output": [ + {"type":"function_call","call_id":"call-1","name":"support-ticket-lookup","arguments":"{\"incident\":\"inc-1\"}"}, + {"type":"function_call","call_id":"call-2","name":"support-ticket-lookup","arguments":"{\"incident\":\"inc-2\"}"} + ] +} diff --git a/examples/harness/foundry/README.md b/examples/harness/foundry/README.md index 92204a099..f30512562 100644 --- a/examples/harness/foundry/README.md +++ b/examples/harness/foundry/README.md @@ -1,6 +1,28 @@ -# Foundry AgentRuntime adapter +# Foundry Assistants/threads AgentRuntime adapter -This example adapter presents Azure AI Foundry hosted agents as an `orka.harness.v1` AgentRuntime endpoint. It keeps Foundry-specific IDs and credentials inside the adapter deployment while Orka keeps owning task lifecycle, brokered tool policy, approvals, idempotency, and result storage. +This example adapter presents the Azure AI Foundry/OpenAI **Assistants threads/runs** protocol as an `orka.harness.v1` AgentRuntime endpoint. + +It is intentionally **not** the adapter for AgentKit agents deployed as Foundry hosted agents. Hosted AgentKit uses endpoint-scoped Responses and must be driven by `examples/harness/foundry-responses` instead. + +## Protocol target + +This adapter uses the Assistants-style API shape: + +```text +POST /threads +POST /threads/{threadID}/runs with assistant_id +GET /threads/{threadID}/runs/{runID} +POST /threads/{threadID}/runs/{runID}/submit_tool_outputs +``` + +Because it targets Assistants/runs, it may send safe Orka tool schemas in the run request and it requires a real Assistants `assistant_id`. Do not set `ORKA_FOUNDRY_AGENT_ID` to a hosted AgentKit agent name; hosted agents are not `asst_*` Assistants resources. + +For Foundry hosted AgentKit over Responses: + +- use `examples/harness/foundry-responses`; +- do not send request-level `tools`; +- statically configure safe brokered schemas in AgentKit; +- resume with `function_call_output` and `previous_response_id`. ## Configuration @@ -9,20 +31,20 @@ This example adapter presents Azure AI Foundry hosted agents as an `orka.harness | `ORKA_FOUNDRY_ADAPTER_ADDR` | HTTP listen address, default `:8090`. | | `ORKA_FOUNDRY_RUNTIME_NAME` | Runtime name advertised in `/v1/capabilities`. | | `ORKA_FOUNDRY_ADAPTER_BEARER_TOKEN` | Bearer token Orka uses for mutating harness endpoints. | -| `ORKA_FOUNDRY_ENDPOINT` | Foundry agents endpoint base URL. Must be HTTPS in production; plain HTTP is accepted only for loopback/local tests. Do not include userinfo, query strings, or fragments. Put API versions in `ORKA_FOUNDRY_API_VERSION`. | -| `ORKA_FOUNDRY_AGENT_ID` | Foundry hosted-agent ID. | +| `ORKA_FOUNDRY_ENDPOINT` | Foundry Assistants endpoint base URL. Must be HTTPS in production; plain HTTP is accepted only for loopback/local tests. Do not include userinfo, query strings, or fragments. Put API versions in `ORKA_FOUNDRY_API_VERSION`. | +| `ORKA_FOUNDRY_AGENT_ID` | Assistants `assistant_id` for the thread run. | | `ORKA_FOUNDRY_API_KEY` | Foundry API key, sent as `api-key`. | | `ORKA_FOUNDRY_AUTH_BEARER` | Optional bearer auth alternative. | | `ORKA_FOUNDRY_API_VERSION` | API version query parameter, default `v1`. | Use Kubernetes Secrets for every token/key value. Do not put Foundry credentials in `AgentRuntime` CRDs. -For `AgentRuntime` readiness/conformance, the hosted Foundry agent must follow Orka brokered-tool probe prompts by calling exactly one provided function tool (for example `conformance_read` or `conformance_write`) and completing after Orka returns the brokered result. The adapter enforces that Foundry can only request tools supplied in the current `StartTurnRequest.input.tools` payload. +For `AgentRuntime` readiness/conformance, the Assistants runtime must follow Orka brokered-tool probe prompts by calling exactly one provided function tool (for example `conformance_read` or `conformance_write`) and completing after Orka returns the brokered result. The adapter enforces that Foundry can only request tools supplied in the current `StartTurnRequest.input.tools` payload. ## Protocol mapping - `StartTurnRequest` creates a Foundry thread and run. -- Orka safe `input.tools` schemas are passed to Foundry as function definitions only; Orka Tool URLs and credentials are never sent. +- In brokered mode, Orka safe `input.tools` schemas are passed to the Assistants run as function definitions only; Orka Tool URLs and credentials are never sent. - Foundry `requires_action.submit_tool_outputs.tool_calls[]` is mapped to `ToolCallRequested` frames. - `/v1/turns/{turnID}/continue` submits Orka-brokered tool outputs back to Foundry. - Foundry completion messages are mapped to `TurnCompleted`. diff --git a/website/docs/guides/bring-your-own-agent-runtime.md b/website/docs/guides/bring-your-own-agent-runtime.md index 740cd8300..1dcc570dc 100644 --- a/website/docs/guides/bring-your-own-agent-runtime.md +++ b/website/docs/guides/bring-your-own-agent-runtime.md @@ -84,6 +84,36 @@ metadata: orka.ai/agent-runtime-endpoint: http://support-http-runtime.default.svc.cluster.local:8080 ``` + +## Foundry hosted AgentKit over Responses + +For AgentKit agents deployed as Foundry hosted agents, use the `examples/harness/foundry-responses` adapter rather than the Assistants/threads adapter. Hosted Responses requests are endpoint-scoped and must not include request-level `tools`; AgentKit must be statically configured with the safe function schemas it may request. Orka still validates every `function_call` against Task policy and Tool CRDs, performs approval/idempotency, executes the tool, and resumes the hosted response with `function_call_output` plus `previous_response_id`. + +Advertise only the brokered classes that the hosted AgentKit deployment is statically configured and conformance-tested to request: + +```yaml +apiVersion: core.orka.ai/v1alpha1 +kind: AgentRuntime +metadata: + name: foundry-agentkit-responses +spec: + contractVersion: orka.harness.v1 + deployment: + mode: external-endpoint + endpoint: http://foundry-agentkit-responses.default.svc.cluster.local:8080 + clientAuth: + bearerTokenSecretRef: + name: foundry-agentkit-responses-token + key: token + capabilities: + toolExecutionModes: [observed, brokered] + brokeredToolClasses: [read] + supportsRuntimeSessions: true + supportsContinuation: true +``` + +Add `write` only after the hosted AgentKit static write schema and brokered-write conformance pass. The adapter's MVP state is in-memory and fail-safe: duplicate identical continuations are no-ops, conflicting duplicates are rejected, and a restart while waiting for approval returns `turn not found` without sending a hosted continuation. + ## Expose a brokered tool ```yaml From 94a199aba131de9c2ead3e2c516ba42442bbe0b7 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 20:46:30 -0700 Subject: [PATCH 02/51] fix: harden Foundry Responses adapter logging Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 6 +++--- examples/harness/foundry-responses/main.go | 2 +- examples/harness/foundry-responses/main_test.go | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 66c4e7250..20cc71f72 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -70,7 +70,7 @@ Hosted AgentKit function call: {"type":"function_call","call_id":"call_1","name":"check-network-telemetry","arguments":"{\"site\":\"quincy-north\"}"} ``` -Adapter emits `ToolCallRequested` with the exact `call_id`, function name, and compact JSON object arguments. +Adapter emits `ToolCallRequested` with the exact `call_id`, function name, and compact JSON object arguments. If a hosted response returns multiple `function_call` items, the adapter emits one `ToolCallRequested` frame per call and waits until Orka returns every pending result before continuing. Orka continuation: @@ -78,7 +78,7 @@ Orka continuation: {"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}","status":"completed"} ``` -The hosted continuation request includes `previous_response_id` and the `function_call_output` item. +The hosted continuation request includes `previous_response_id`, `agent_session_id` when a Foundry session is known, and one or more `function_call_output` items. Raw REST calls include the hosted-agent feature header required by Foundry hosted-agent endpoints. ## Output and error encoding @@ -98,7 +98,7 @@ This MVP stores turn state in memory. That is intentionally fail-safe: - if the adapter restarts while a tool approval is pending, `/continue` returns `turn not found` and does not call Foundry, so the adapter itself does not duplicate a side effect; - Orka's broker/idempotency ledger remains the source of truth for actual write execution. -The adapter captures Foundry session headers such as `x-agent-session-id` and reuses them for later calls with the same Orka `runtimeSessionID`. Session/auth headers are stored only in memory and are not logged. +The adapter captures Foundry session identifiers from the hosted response body (`agent_session_id`) and compatibility headers such as `x-agent-session-id`, then reuses the session in the `agent_session_id` request field for later calls with the same Orka `runtimeSessionID`. Session/auth values are stored only in memory and are not logged. ## Local build diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index b5ad4d300..69dab0c63 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -1275,7 +1275,7 @@ func sanitizeEndpoint(raw string) string { } u, err := url.Parse(raw) if err != nil { - return raw + return "[invalid endpoint]" } u.User = nil q := u.Query() diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index bfdb1b21f..78924ede9 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -642,6 +642,13 @@ func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { } } +func TestSanitizeEndpointDoesNotReturnRawMalformedURL(t *testing.T) { + raw := "http://[::1" + "?unsafe=do-not-log" + if got := sanitizeEndpoint(raw); got == raw || strings.Contains(got, "do-not-log") { + t.Fatalf("sanitizeEndpoint(%q) = %q, want redacted placeholder", raw, got) + } +} + func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing.T) { server := newServer(config{ runtimeName: "test", From a10546c8348eac4b02cd28d336fb36852e9d5d65 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 20:53:20 -0700 Subject: [PATCH 03/51] fix: clarify Foundry Responses endpoint safety Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 69dab0c63..cd28b5ae7 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -138,7 +138,7 @@ type pendingFunctionCall struct { } const responsesEndpointRequirement = "foundry hosted Responses endpoint must use https " + - "(http allowed only for loopback), target /responses, and must not include foundryAuths, " + + "(http allowed only for loopback), target /responses, and must not include credentials, " + "fragments, or query parameters other than api-version" func main() { From b8f722c84c2ba81662973085f3801ece146378e9 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 20:58:10 -0700 Subject: [PATCH 04/51] fix: harden Foundry Responses continuations Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 1 + examples/harness/foundry-responses/main.go | 31 +++++++ .../harness/foundry-responses/main_test.go | 82 +++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 20cc71f72..0a868fc58 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -28,6 +28,7 @@ Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `exa | `ORKA_FOUNDRY_RESPONSES_API_KEY` | Static API-key auth mode. Tests/demo only unless your deployment standard permits it. | | `ORKA_FOUNDRY_RESPONSES_AUTH_BEARER` | Static bearer auth mode. Tests/demo only unless supplied by a production token refresher sidecar. | | `ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE` | Reserved for future workload-identity token refresh support; currently not used. | +| `ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF` | Optional Orka-only proof value sent as `X-AgentKit-Brokered-Continuation-Proof` on hosted Responses continuations. Set it to match AgentKit's `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` when that guard is enabled. | | `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` | Comma-separated static classes the hosted AgentKit deployment has been configured and conformance-tested to request, e.g. `read` or `read,write`. Empty means observed-only. | | `ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT` | Per-request timeout for hosted Responses calls, default `20s`. | | `ORKA_FOUNDRY_RESPONSES_STATE_RETENTION` | How long terminal in-memory turn/session state is retained, default `10m`. | diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index cd28b5ae7..d6226e89f 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -45,6 +45,7 @@ const ( envRequestTimeout = "ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT" envStateRetention = "ORKA_FOUNDRY_RESPONSES_STATE_RETENTION" envMaxApprovalWait = "ORKA_FOUNDRY_RESPONSES_MAX_APPROVAL_WAIT" + envContinuationProof = "ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF" envBrokeredToolClasses = "ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES" envAudience = "ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE" ) @@ -62,6 +63,7 @@ type config struct { requestTimeout time.Duration stateRetention time.Duration maxApprovalWait time.Duration + continuationProof string brokeredToolClasses []harness.BrokeredToolClass configError string } @@ -170,6 +172,7 @@ func loadConfig() config { requestTimeout: parseDurationEnv(envRequestTimeout, defaultRequestTimeout), stateRetention: parseDurationEnv(envStateRetention, defaultStateRetention), maxApprovalWait: parseDurationEnv(envMaxApprovalWait, defaultMaxApprovalWait), + continuationProof: strings.TrimSpace(os.Getenv(envContinuationProof)), brokeredToolClasses: classes, } _ = os.Getenv(envAudience) // Reserved for a future workload-identity token provider; never logged. @@ -556,6 +559,7 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp } turn.pendingTools[call.callID] = call.name turn.pendingSince[call.callID] = now + s.schedulePendingToolTimeoutLocked(turn, call.callID) s.appendFrameLocked( turn, harness.FrameToolCallRequested, @@ -844,6 +848,9 @@ func (s *server) postResponses( if s.cfg.authBearer != "" { req.Header.Set("Authorization", "Bearer "+s.cfg.authBearer) } + if body.PreviousResponseID != "" && s.cfg.continuationProof != "" { + req.Header.Set("X-AgentKit-Brokered-Continuation-Proof", s.cfg.continuationProof) + } if session.ID != "" { req.Header.Set("x-agent-session-id", session.ID) } @@ -1105,6 +1112,30 @@ func (s *server) updateTurnSessionLocked(turn *turnState) { } } +func (s *server) schedulePendingToolTimeoutLocked(turn *turnState, toolCallID string) { + wait := s.cfg.maxApprovalWait + if wait <= 0 { + return + } + turnID := turn.request.TurnID + time.AfterFunc(wait, func() { + s.mu.Lock() + defer s.mu.Unlock() + current := s.turns[turnID] + if current != turn || turn.completed { + return + } + if _, pending := turn.pendingTools[toolCallID]; !pending { + return + } + s.appendFailedLocked( + turn, + "approval_wait_exceeded", + "maximum brokered tool wait exceeded", + ) + }) +} + func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { if turn.completed { return diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 78924ede9..c1552e808 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -359,6 +359,48 @@ func TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject(t *test } } +func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{ + scenario: "function_call", + toolName: "support-ticket-lookup", + requiredProof: "proof-for-test", + }) + s := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.endpoint(), + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + continuationProof: "proof-for-test", + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + adapter := httptest.NewServer(s.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-continuation-proof") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn: %v", err) + } + if got := foundry.requestHeader(1).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "proof-for-test" { + t.Fatalf("continuation proof header = %q, want proof-for-test", got) + } + if got := foundry.requestHeader(0).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "" { + t.Fatalf("initial proof header = %q, want empty", got) + } +} + func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{ scenario: "function_call", @@ -401,6 +443,45 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t } } +func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: 10 * time.Millisecond, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: brokeredReadRequest("foundry-timeout"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.turns[turn.request.TurnID] = turn + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-1", + Output: []responsesOutput{{ + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }}, + }) + time.Sleep(50 * time.Millisecond) + server.mu.Lock() + defer server.mu.Unlock() + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "approval_wait_exceeded" { + t.Fatalf("failed frame = %#v, want approval_wait_exceeded", failed) + } +} + func TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) s := newServer(config{ @@ -729,6 +810,7 @@ type fakeResponsesConfig struct { scenario string toolName string continuationStatus int + requiredProof string } type fakeResponses struct { From 25f066525fdc0c05ae652ee6e1abde69645dcbc3 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:05:08 -0700 Subject: [PATCH 05/51] fix: fail repeated Foundry Responses tool calls Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 36 +++++- .../harness/foundry-responses/main_test.go | 122 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index d6226e89f..710fa1875 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -535,6 +535,10 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp ) return } + if isFailureStatus(response.Status) { + s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) + return + } calls, err := s.extractFunctionCalls(turn.request, response.Output) if err != nil { s.appendFailedLocked(turn, "foundry_function_call_invalid", err.Error()) @@ -549,14 +553,36 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp ) return } - now := time.Now().UTC() + seenInResponse := map[string]struct{}{} for _, call := range calls { + if _, seen := seenInResponse[call.callID]; seen { + s.appendFailedLocked( + turn, + "foundry_repeated_function_call", + "hosted response repeated a function call id", + ) + return + } + seenInResponse[call.callID] = struct{}{} if _, submitted := turn.submittedPayloads[call.callID]; submitted { - continue + s.appendFailedLocked( + turn, + "foundry_repeated_function_call", + "hosted response repeated an already-submitted function call", + ) + return } if _, pending := turn.pendingTools[call.callID]; pending { - continue + s.appendFailedLocked( + turn, + "foundry_repeated_function_call", + "hosted response repeated an already-pending function call", + ) + return } + } + now := time.Now().UTC() + for _, call := range calls { turn.pendingTools[call.callID] = call.name turn.pendingSince[call.callID] = now s.schedulePendingToolTimeoutLocked(turn, call.callID) @@ -573,10 +599,6 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp } return } - if isFailureStatus(response.Status) { - s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) - return - } result := responsesMessageText(response.Output) if len([]byte(result)) > maxFoundryOutputBytes { s.appendFailedLocked(turn, "foundry_output_too_large", "foundry completion exceeded advertised output limit") diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index c1552e808..c53d65d25 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -443,6 +443,89 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t } } +func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-repeated-call") + turn := &turnState{ + request: request, + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{"call-1": `{"approved":true}`}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-repeat", + Output: []responsesOutput{{ + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }}, + }) + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_repeated_function_call" { + t.Fatalf("failed frame = %#v, want foundry_repeated_function_call", failed) + } +} + +func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-mixed-repeated-call") + turn := &turnState{ + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-repeat", + Output: []responsesOutput{ + { + Type: "function_call", + CallID: "call-2", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-2"}`), + }, + { + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }, + }, + }) + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_repeated_function_call" { + t.Fatalf("failed frame = %#v, want foundry_repeated_function_call", failed) + } + if _, exists := turn.pendingTools["call-2"]; exists { + t.Fatal("new call was accepted after repeated pending call") + } +} + func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -723,6 +806,45 @@ func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { } } +func TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: brokeredReadRequest("foundry-failed-function-call"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-failed", + Status: "incomplete", + Output: []responsesOutput{{ + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }}, + }) + if hasFrameType(turn.frames, harness.FrameToolCallRequested) { + t.Fatalf("frames = %#v, failed response should not request a tool", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_incomplete" { + t.Fatalf("failed frame = %#v, want foundry_incomplete", failed) + } +} + func TestSanitizeEndpointDoesNotReturnRawMalformedURL(t *testing.T) { raw := "http://[::1" + "?unsafe=do-not-log" if got := sanitizeEndpoint(raw); got == raw || strings.Contains(got, "do-not-log") { From 0ea9a420255e801b32465125ad6ea183fa2dd0b4 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:19:07 -0700 Subject: [PATCH 06/51] test: consume AgentKit Foundry brokered fixtures Signed-off-by: Sertac Ozercan --- .../harness/foundry-responses/main_test.go | 149 ++++++++++++++++++ .../approval_declined_payload.json | 7 + .../continuation_request.json | 11 ++ .../final_message_response.json | 25 +++ .../function_call_response.json | 19 +++ .../initial_request.json | 3 + .../tool_execution_failure_payload.json | 7 + .../tool_policy_rejection_payload.json | 7 + 8 files changed, 228 insertions(+) create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/continuation_request.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/final_message_response.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/function_call_response.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/initial_request.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json create mode 100644 examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index c53d65d25..2de49f66c 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -751,6 +751,155 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { } } +func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { + server := newServer( + config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, + &http.Client{Timeout: time.Second}, + ) + request := responsesStartTurnRequest("agentkit-brokered-fixture") + request.ToolExecutionMode = harness.ToolExecutionModeBrokered + request.Input.Tools = []harness.ToolDefinition{{ + Name: "conformance_read", + Description: "Synthetic AgentKit brokered fixture tool", + BrokeredClass: harness.BrokeredToolClassRead, + Parameters: json.RawMessage(`{"type":"object","properties":{"probe":{"type":"boolean"}}}`), + }} + turn := &turnState{ + request: request, + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + var functionCall responsesResponse + decodeFixtureInto( + t, + "testdata/agentkit-foundry-brokered/function_call_response.json", + &functionCall, + ) + server.handleResponsesResponse(turn, functionCall) + requested := findFrame(turn.frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request from AgentKit fixture", turn.frames) + } + if requested.ToolName != "conformance_read" || requested.ToolCallID != "call_caresp_test_1" { + t.Fatalf("tool request = %#v, want AgentKit fixture call", requested) + } + assertJSONFileEqual(t, "testdata/agentkit-foundry-brokered/initial_request.json", responsesRequest{ + Input: "please read telemetry", + }) + + outputs, _, err := functionCallOutputs([]harness.ToolCallResult{{ + Version: harness.ProtocolVersion, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + ToolCallID: "call_caresp_test_1", + IdempotencyKey: harness.ToolRequestIdempotencyKey(request.RuntimeSessionID, request.TurnID, "call_caresp_test_1"), + Approved: true, + Output: json.RawMessage(`{"success":true}`), + }}) + if err != nil { + t.Fatalf("functionCallOutputs: %v", err) + } + assertJSONFileEqual(t, "testdata/agentkit-foundry-brokered/continuation_request.json", responsesRequest{ + PreviousResponseID: "caresp_test", + Input: outputs, + }) + + finalTurn := &turnState{ + request: responsesStartTurnRequest("agentkit-final-fixture"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + var finalMessage responsesResponse + decodeFixtureInto(t, "testdata/agentkit-foundry-brokered/final_message_response.json", &finalMessage) + server.handleResponsesResponse(finalTurn, finalMessage) + completed := findFrame(finalTurn.frames, harness.FrameTurnCompleted) + want := `Brokered tool conformance_read completed with output: {"success":true}` + if completed == nil || completed.Completed.Result != want { + t.Fatalf("completed frame = %#v, want %q", completed, want) + } +} + +func TestResponsesAgentKitErrorPayloadFixturesMatchCanonicalEncoding(t *testing.T) { + tests := []struct { + name string + result harness.ToolCallResult + fixture string + }{ + { + name: "approval declined", + result: baseToolResult("call-agentkit-1", false, nil, &harness.ErrorInfo{ + Code: "approval_declined", Message: "Human declined dispatch-work-order", + }), + fixture: "testdata/agentkit-foundry-brokered/approval_declined_payload.json", + }, + { + name: "policy rejection", + result: baseToolResult("call-agentkit-2", false, nil, &harness.ErrorInfo{ + Code: "tool_policy_rejected", Message: "writes are disabled", + }), + fixture: "testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json", + }, + { + name: "execution failure", + result: baseToolResult("call-agentkit-3", true, nil, &harness.ErrorInfo{ + Code: "tool_execution_failed", Message: "downstream timed out", + }), + fixture: "testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := canonicalToolResultOutput(tt.result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + assertJSONFileEqual(t, tt.fixture, json.RawMessage(payload)) + }) + } +} + +func TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets(t *testing.T) { + for _, root := range []string{"testdata/golden", "testdata/agentkit-foundry-brokered"} { + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("ReadDir(%s): %v", root, err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := root + "/" + entry.Name() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s): %v", path, err) + } + text := strings.ToLower(string(data)) + for _, forbidden := range []string{"http://", "https://", "authorization", "api_key", "api-key", "bearer "} { + if strings.Contains(text, forbidden) { + t.Fatalf("fixture %s contains forbidden %q", path, forbidden) + } + } + } + } +} + func TestResponsesPreservesNumericJSONTokens(t *testing.T) { args, err := normalizeResponsesToolArguments(json.RawMessage(`{"id":9007199254740993}`)) if err != nil { diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json new file mode 100644 index 000000000..facb8ed7b --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "approval_declined", + "message": "Human declined dispatch-work-order" + } +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/continuation_request.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/continuation_request.json new file mode 100644 index 000000000..e5a8e9128 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/continuation_request.json @@ -0,0 +1,11 @@ +{ + "previous_response_id": "caresp_test", + "input": [ + { + "type": "function_call_output", + "call_id": "call_caresp_test_1", + "output": "{\"approved\":true,\"output\":{\"success\":true}}", + "status": "completed" + } + ] +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/final_message_response.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/final_message_response.json new file mode 100644 index 000000000..6d4786fef --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/final_message_response.json @@ -0,0 +1,25 @@ +{ + "id": "caresp_final", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_final", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Brokered tool conformance_read completed with output: {\"success\":true}", + "annotations": [] + } + ], + "response_id": "caresp_final" + } + ], + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "previous_response_id": "caresp_test" +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/function_call_response.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/function_call_response.json new file mode 100644 index 000000000..adc0d9dc4 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/function_call_response.json @@ -0,0 +1,19 @@ +{ + "id": "caresp_test", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "fc_test", + "type": "function_call", + "call_id": "call_caresp_test_1", + "name": "conformance_read", + "arguments": "{\"probe\":true}", + "status": "completed", + "response_id": "caresp_test" + } + ], + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/initial_request.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/initial_request.json new file mode 100644 index 000000000..02c472731 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/initial_request.json @@ -0,0 +1,3 @@ +{ + "input": "please read telemetry" +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json new file mode 100644 index 000000000..b0955fef2 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "tool_execution_failed", + "message": "downstream timed out" + } +} diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json new file mode 100644 index 000000000..d32368fb6 --- /dev/null +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "tool_policy_rejected", + "message": "writes are disabled" + } +} From c74bcd650f87fb7c896036bf7e9bb841a09721cd Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:28:31 -0700 Subject: [PATCH 07/51] fix: harden Foundry Responses HTTP session handling Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 36 ++++++++-- .../harness/foundry-responses/main_test.go | 68 ++++++++++++++++++- 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 710fa1875..dcf060757 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -186,9 +186,15 @@ func newServer(cfg config, client *http.Client) *server { if client == nil { client = &http.Client{Timeout: cfg.requestTimeout} } + clientCopy := *client + if clientCopy.CheckRedirect == nil { + clientCopy.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + } return &server{ cfg: cfg, - client: client, + client: &clientCopy, turns: map[harness.HarnessTurnID]*turnState{}, runtimeSessions: map[harness.RuntimeSessionID]foundrySession{}, } @@ -436,6 +442,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn } s.mu.Lock() previousResponseID := turn.responseID + foundrySessionID := turn.foundrySessionID s.mu.Unlock() if strings.TrimSpace(previousResponseID) == "" { harness.WriteError(w, http.StatusConflict, "cannot continue before Foundry response id is known") @@ -446,6 +453,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn var response responsesResponse continuation := responsesRequest{ PreviousResponseID: previousResponseID, + AgentSessionID: foundrySessionID, Input: outputs, } s.markSubmittedPayloads(turn, payloadByCall) @@ -850,8 +858,9 @@ func (s *server) postResponses( s.mu.Lock() session := s.runtimeSessions[runtimeSessionID] s.mu.Unlock() - if body.AgentSessionID == "" && session.ID != "" { - body.AgentSessionID = session.ID + sessionID := firstNonBlank(body.AgentSessionID, session.ID) + if body.AgentSessionID == "" && sessionID != "" { + body.AgentSessionID = sessionID } payload, err := json.Marshal(body) if err != nil { @@ -873,17 +882,18 @@ func (s *server) postResponses( if body.PreviousResponseID != "" && s.cfg.continuationProof != "" { req.Header.Set("X-AgentKit-Brokered-Continuation-Proof", s.cfg.continuationProof) } - if session.ID != "" { - req.Header.Set("x-agent-session-id", session.ID) + if sessionID != "" { + req.Header.Set("x-agent-session-id", sessionID) } resp, err := s.client.Do(req) if err != nil { return err } defer resp.Body.Close() //nolint:errcheck - sessionID := firstNonBlank( + sessionID = firstNonBlank( resp.Header.Get("x-agent-session-id"), resp.Header.Get("x-ms-agent-session-id"), + sessionID, ) if resp.StatusCode < 200 || resp.StatusCode >= 300 { data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) @@ -1185,8 +1195,12 @@ func (s *server) scheduleTurnCleanupLocked(turn *turnState) { s.mu.Lock() defer s.mu.Unlock() delete(s.turns, turnID) + activeSessions := s.activeRuntimeSessionsLocked() cutoff := time.Now().UTC().Add(-retention) for sessionID, session := range s.runtimeSessions { + if activeSessions[sessionID] { + continue + } if session.LastSeen.Before(cutoff) { delete(s.runtimeSessions, sessionID) } @@ -1194,6 +1208,16 @@ func (s *server) scheduleTurnCleanupLocked(turn *turnState) { }) } +func (s *server) activeRuntimeSessionsLocked() map[harness.RuntimeSessionID]bool { + active := map[harness.RuntimeSessionID]bool{} + for _, turn := range s.turns { + if turn != nil && !turn.completed { + active[turn.request.RuntimeSessionID] = true + } + } + return active +} + func (s *server) appendFrameLocked( turn *turnState, typ harness.FrameType, diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 2de49f66c..e796cfbd4 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -52,6 +52,30 @@ func TestResponsesAdapterObservedTurnCompletes(t *testing.T) { assertJSONFileEqual(t, "testdata/golden/01_initial_hosted_request.json", foundry.requestBody(0)) } +func TestResponsesAdapterDoesNotFollowCredentialedRedirects(t *testing.T) { + redirectTargetHit := atomic.Bool{} + redirectTarget := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + redirectTargetHit.Store(true) + })) + t.Cleanup(redirectTarget.Close) + + redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, redirectTarget.URL+"/capture", http.StatusTemporaryRedirect) + })) + t.Cleanup(redirector.Close) + + endpoint := redirector.URL + + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter := newTestResponsesAdapter(t, endpoint, nil) + client := newHarnessClient(t, adapter) + if _, err := client.StartTurn(context.Background(), responsesStartTurnRequest("foundry-redirect")); err == nil { + t.Fatalf("StartTurn followed redirect and succeeded, want rejection") + } + if redirectTargetHit.Load() { + t.Fatal("redirect target was called; credentialed Foundry request followed an unvalidated redirect") + } +} + func TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) adapter := newTestResponsesAdapter( @@ -153,6 +177,38 @@ func TestResponsesAdapterRuntimeSessionHeaderReuse(t *testing.T) { } } +func TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter, server := newTestResponsesAdapterWithServer( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-session-cleanup") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + + server.mu.Lock() + delete(server.runtimeSessions, request.RuntimeSessionID) + server.mu.Unlock() + + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { + t.Fatalf("ContinueTurn after runtime session cleanup: %v", err) + } + if got := requestMap(t, foundry.requestBody(1))["agent_session_id"]; got != fakeSessionID { + t.Fatalf("agent_session_id after runtime session cleanup = %#v, want %q", got, fakeSessionID) + } +} + func TestResponsesAdapterPassesObservedConformanceByDefault(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) @@ -1238,6 +1294,16 @@ func finalResponsesMessage() map[string]any { } func newTestResponsesAdapter(t *testing.T, endpoint string, classes []harness.BrokeredToolClass) *httptest.Server { + t.Helper() + adapter, _ := newTestResponsesAdapterWithServer(t, endpoint, classes) + return adapter +} + +func newTestResponsesAdapterWithServer( + t *testing.T, + endpoint string, + classes []harness.BrokeredToolClass, +) (*httptest.Server, *server) { t.Helper() s := newServer(config{ addr: ":0", @@ -1253,7 +1319,7 @@ func newTestResponsesAdapter(t *testing.T, endpoint string, classes []harness.Br }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) t.Cleanup(adapter.Close) - return adapter + return adapter, s } func newHarnessClient(t *testing.T, adapter *httptest.Server) *harness.Client { From 1ca8f83bff3f27fe2ec3690262d988744c94c922 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:28:31 -0700 Subject: [PATCH 08/51] docs: add Foundry Responses Kubernetes smoke skeleton Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 12 ++ .../foundry-responses/kubernetes.example.yaml | 116 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 examples/harness/foundry-responses/kubernetes.example.yaml diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 0a868fc58..391d512e0 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -107,6 +107,18 @@ The adapter captures Foundry session identifiers from the hosted response body ( docker build -t ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest -f examples/harness/foundry-responses/Dockerfile . ``` + +## Kubernetes smoke skeleton + +`kubernetes.example.yaml` contains a credentials-free Deployment, Service, Secret placeholders, and matching `AgentRuntime` facade for a read-profile hosted Responses smoke. Replace the `REDACTED` values through your secret-management flow, set the hosted Responses endpoint or project/agent-name pair, and keep `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` narrowed to classes whose static AgentKit schemas passed conformance. + +```bash +kubectl apply -f examples/harness/foundry-responses/kubernetes.example.yaml +kubectl wait --for=condition=Ready agentruntime/sample-foundry-responses-runtime --timeout=60s +``` + +For write-profile smoke, first prove the hosted AgentKit deployment has a static write schema, then add `write` to both the adapter env and the `AgentRuntime.spec.capabilities.brokeredToolClasses`. Orka still performs approval and idempotent write execution; the hosted endpoint receives only `function_call_output` continuations. + ## Tests ```bash diff --git a/examples/harness/foundry-responses/kubernetes.example.yaml b/examples/harness/foundry-responses/kubernetes.example.yaml new file mode 100644 index 000000000..452dfefb3 --- /dev/null +++ b/examples/harness/foundry-responses/kubernetes.example.yaml @@ -0,0 +1,116 @@ +# Example Kubernetes deployment for the Foundry hosted Responses adapter. +# Replace REDACTED placeholders through your secret-management flow before use. +# Do not commit real Foundry credentials or adapter bearer values. +apiVersion: v1 +kind: Secret +metadata: + name: sample-foundry-responses-runtime-token + annotations: + orka.ai/agent-runtime-endpoint: http://sample-foundry-responses-runtime.default.svc.cluster.local:8080 + labels: + orka.ai/agent-runtime-auth: "true" + orka.ai/agent-runtime-name: sample-foundry-responses-runtime +stringData: + harness-bearer: REDACTED +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample-foundry-responses-adapter-config +stringData: + foundry-auth: REDACTED + continuation-proof: REDACTED +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sample-foundry-responses-runtime + labels: + app.kubernetes.io/name: sample-foundry-responses-runtime +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sample-foundry-responses-runtime + template: + metadata: + labels: + app.kubernetes.io/name: sample-foundry-responses-runtime + spec: + containers: + - name: adapter + image: ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest + imagePullPolicy: IfNotPresent + env: + - name: ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR + value: :8090 + - name: ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME + value: sample-foundry-responses-runtime + - name: ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sample-foundry-responses-runtime-token + key: harness-bearer + # Use ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT plus AGENT_NAME instead of + # the full endpoint when you want the adapter to compose the hosted URL. + - name: ORKA_FOUNDRY_RESPONSES_ENDPOINT + value: https://example.services.ai.azure.com/agents/sample-agent/endpoint/protocols/openai/responses + - name: ORKA_FOUNDRY_RESPONSES_API_VERSION + value: v1 + - name: ORKA_FOUNDRY_RESPONSES_API_KEY + valueFrom: + secretKeyRef: + name: sample-foundry-responses-adapter-config + key: foundry-auth + - name: ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF + valueFrom: + secretKeyRef: + name: sample-foundry-responses-adapter-config + key: continuation-proof + optional: true + # Advertise only classes whose static AgentKit brokered schemas passed conformance. + - name: ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES + value: read + ports: + - name: http + containerPort: 8090 + readinessProbe: + httpGet: + path: /v1/health + port: http + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: sample-foundry-responses-runtime +spec: + selector: + app.kubernetes.io/name: sample-foundry-responses-runtime + ports: + - name: http + port: 8080 + targetPort: http +--- +apiVersion: core.orka.ai/v1alpha1 +kind: AgentRuntime +metadata: + name: sample-foundry-responses-runtime +spec: + contractVersion: orka.harness.v1 + deployment: + mode: external-endpoint + endpoint: http://sample-foundry-responses-runtime.default.svc.cluster.local:8080 + clientAuth: + bearerTokenSecretRef: + name: sample-foundry-responses-runtime-token + key: harness-bearer + capabilities: + toolExecutionModes: + - observed + - brokered + brokeredToolClasses: + - read + supportsCancel: true + supportsRuntimeSessions: true + supportsContinuation: true From 5755bdf8d45785222ae988c110cee6e365c4d5da Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:36:25 -0700 Subject: [PATCH 09/51] test: cover Foundry Responses platform edge cases Signed-off-by: Sertac Ozercan --- .../harness/foundry-responses/main_test.go | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index e796cfbd4..e43d12a4f 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -975,6 +975,57 @@ func TestResponsesPreservesNumericJSONTokens(t *testing.T) { } } +func TestResponsesLargeOutputFails(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: responsesStartTurnRequest("foundry-large-output"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-large", + Output: []responsesOutput{{ + Type: "message", + Content: strings.Repeat("x", maxFoundryOutputBytes+1), + }}, + }) + if hasFrameType(turn.frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, oversized response should not complete", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_output_too_large" { + t.Fatalf("failed frame = %#v, want foundry_output_too_large", failed) + } +} + +func TestResponsesInitialPlatformErrorDoesNotRetainTurn(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "platform_error"}) + adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-platform-error") + + if _, err := client.StartTurn(context.Background(), request); err == nil { + t.Fatal("StartTurn succeeded, want platform error") + } + server.mu.Lock() + defer server.mu.Unlock() + if _, exists := server.turns[request.TurnID]; exists { + t.Fatalf("turn %q retained after failed initial hosted response", request.TurnID) + } +} + func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { server := newServer(config{ runtimeName: "test", From 90bc61a674a43f8a5f44b19e91ebc90035da210b Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:41:31 -0700 Subject: [PATCH 10/51] fix: guard Foundry Responses startup races Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 13 ++++ .../harness/foundry-responses/main_test.go | 68 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index dcf060757..7d36b8361 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -84,6 +84,7 @@ type foundrySession struct { type turnState struct { request harness.StartTurnRequest + initializing bool responseID string foundrySessionID string pendingTools map[string]string @@ -306,12 +307,18 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteError(w, http.StatusConflict, "turn already exists") return } + if existing.initializing { + s.mu.Unlock() + harness.WriteError(w, http.StatusConflict, "turn initialization in progress") + return + } s.mu.Unlock() harness.WriteJSON(w, http.StatusAccepted, response) return } turn := &turnState{ request: req, + initializing: true, pendingTools: map[string]string{}, pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, @@ -337,6 +344,9 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { s.updateTurnSessionLocked(turn) s.mu.Unlock() s.handleResponsesResponse(turn, response) + s.mu.Lock() + turn.initializing = false + s.mu.Unlock() harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) } @@ -1160,6 +1170,9 @@ func (s *server) schedulePendingToolTimeoutLocked(turn *turnState, toolCallID st if _, pending := turn.pendingTools[toolCallID]; !pending { return } + if _, submitted := turn.submittedPayloads[toolCallID]; submitted { + return + } s.appendFailedLocked( turn, "approval_wait_exceeded", diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index e43d12a4f..835f51432 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -209,6 +209,44 @@ func TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup(t *te } } +func TestResponsesAdapterDuplicateStartDuringInitializationRejected(t *testing.T) { + received := make(chan struct{}) + release := make(chan struct{}) + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + select { + case <-received: + default: + close(received) + } + <-release + writeJSON(w, finalResponsesMessage()) + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter := newTestResponsesAdapter(t, endpoint, nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-duplicate-start") + + firstErr := make(chan error, 1) + go func() { + _, err := client.StartTurn(context.Background(), request) + firstErr <- err + }() + <-received + if _, err := client.StartTurn(context.Background(), request); err == nil || + !strings.Contains(err.Error(), "initialization in progress") { + t.Fatalf("duplicate StartTurn error = %v, want initialization conflict", err) + } + close(release) + if err := <-firstErr; err != nil { + t.Fatalf("initial StartTurn: %v", err) + } +} + func TestResponsesAdapterPassesObservedConformanceByDefault(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) @@ -621,6 +659,36 @@ func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { } } +func TestResponsesAdapterPendingTimeoutSkipsSubmittedCall(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: 10 * time.Millisecond, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: brokeredReadRequest("foundry-timeout-submitted"), + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{"call-1": `{"approved":true}`}, + } + server.turns[turn.request.TurnID] = turn + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.schedulePendingToolTimeoutLocked(turn, "call-1") + time.Sleep(50 * time.Millisecond) + server.mu.Lock() + defer server.mu.Unlock() + if failed := findFrame(turn.frames, harness.FrameTurnFailed); failed != nil { + t.Fatalf("failed frame = %#v, submitted call should not time out", failed) + } +} + func TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) s := newServer(config{ From e16e3f6639e10fc67ba096f4feeaa4ad7d4035af Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:48:32 -0700 Subject: [PATCH 11/51] test: cover Foundry Responses write approval decline Signed-off-by: Sertac Ozercan --- .../harness/foundry-responses/main_test.go | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 835f51432..c23a0a108 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -322,6 +322,76 @@ func TestResponsesAdapterPassesBrokeredWriteConformance(t *testing.T) { } } +func TestResponsesAdapterWriteParksUntilDeclinedApprovalContinue(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "dispatch-work-order"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassWrite}, + ) + client := newHarnessClient(t, adapter) + request := brokeredWriteRequest("foundry-write-declined") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want write tool request", frames) + } + if requested.ToolName != "dispatch-work-order" { + t.Fatalf("ToolName = %q, want dispatch-work-order", requested.ToolName) + } + if hasFrameType(frames, harness.FrameToolResultReceived) || hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, write should park until Orka continuation", frames) + } + if foundry.postCount.Load() != 1 { + t.Fatalf("hosted post count before approval = %d, want 1", foundry.postCount.Load()) + } + + declined := harness.ContinueTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: request.Namespace, + TaskName: request.TaskName, + SessionName: request.SessionName, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + CorrelationID: request.CorrelationID, + ToolResults: []harness.ToolCallResult{toolResultForRequest( + request, + requested.ToolCallID, + false, + nil, + &harness.ErrorInfo{Code: "approval_declined", Message: "human declined"}, + )}, + } + if _, err := client.ContinueTurn(context.Background(), declined); err != nil { + t.Fatalf("ContinueTurn declined approval: %v", err) + } + continuation := requestMap(t, foundry.requestBody(1)) + items, ok := continuation["input"].([]any) + if !ok || len(items) != 1 { + t.Fatalf("continuation input = %#v, want one item", continuation["input"]) + } + item, ok := items[0].(map[string]any) + if !ok { + t.Fatalf("continuation item = %#v, want object", items[0]) + } + wantOutput := `{"approved":false,"error":{"code":"approval_declined","message":"human declined"}}` + if got := item["output"]; got != wantOutput { + t.Fatalf("declined output = %#v, want %s", got, wantOutput) + } + frames = streamAllFrames(t, client, request.TurnID) + toolResult := findFrame(frames, harness.FrameToolResultReceived) + if toolResult == nil || toolResult.Error == nil || toolResult.Error.Code != "approval_declined" { + t.Fatalf("tool result frame = %#v, want approval_declined", toolResult) + } + if !hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want final completion after declined continuation", frames) + } +} + func TestResponsesAdapterRejectsUnknownToolBeforeOrkaExecution(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "unknown-tool"}) adapter := newTestResponsesAdapter( @@ -1482,6 +1552,20 @@ func brokeredReadRequest(name string) harness.StartTurnRequest { return request } +func brokeredWriteRequest(name string) harness.StartTurnRequest { + request := responsesStartTurnRequest(name) + request.ToolExecutionMode = harness.ToolExecutionModeBrokered + request.Input.Tools = []harness.ToolDefinition{{ + Name: "dispatch-work-order", + Description: "Dispatch a work order", + BrokeredClass: harness.BrokeredToolClassWrite, + Parameters: json.RawMessage( + `{"type":"object","properties":{"incident":{"type":"string"}},"required":["incident"]}`, + ), + }} + return request +} + func goldenContinueRequest( request harness.StartTurnRequest, callID string, From 8fc3a91a995f7336fb106d0e12bb537f95c95ce2 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 21:56:29 -0700 Subject: [PATCH 12/51] fix: avoid duplicate Foundry Responses continuations Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 37 ++++++----- .../harness/foundry-responses/main_test.go | 65 ++++++++++++++++++- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 7d36b8361..52d5eadcf 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "log" - "maps" "net/http" "net/url" "os" @@ -445,7 +444,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) return } - outputs, payloadByCall, err := functionCallOutputs(resultsToSubmit) + outputs, err := functionCallOutputs(resultsToSubmit) if err != nil { harness.WriteError(w, http.StatusBadRequest, err.Error()) return @@ -466,7 +465,6 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn AgentSessionID: foundrySessionID, Input: outputs, } - s.markSubmittedPayloads(turn, payloadByCall) if err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response); err != nil { s.mu.Lock() s.appendFailedLocked( @@ -744,7 +742,17 @@ func (s *server) recordContinueResults( turn.bufferedResults[result.ToolCallID] = result turn.bufferedPayloads[result.ToolCallID] = payload } - if len(turn.bufferedResults) < len(turn.pendingTools) { + readyCount := 0 + for id := range turn.pendingTools { + if _, submitted := turn.submittedPayloads[id]; submitted { + readyCount++ + continue + } + if _, buffered := turn.bufferedResults[id]; buffered { + readyCount++ + } + } + if readyCount < len(turn.pendingTools) { return nil, nil } ids := make([]string, 0, len(turn.pendingTools)) @@ -754,17 +762,18 @@ func (s *server) recordContinueResults( sort.Strings(ids) toSubmit := make([]harness.ToolCallResult, 0, len(ids)) for _, id := range ids { + if _, submitted := turn.submittedPayloads[id]; submitted { + continue + } toSubmit = append(toSubmit, turn.bufferedResults[id]) + turn.submittedPayloads[id] = turn.bufferedPayloads[id] + } + if len(toSubmit) == 0 { + return nil, nil } return toSubmit, nil } -func (s *server) markSubmittedPayloads(turn *turnState, payloadByCall map[string]string) { - s.mu.Lock() - defer s.mu.Unlock() - maps.Copy(turn.submittedPayloads, payloadByCall) -} - func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []harness.ToolCallResult) error { s.mu.Lock() defer s.mu.Unlock() @@ -787,13 +796,12 @@ func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []ha return nil } -func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, map[string]string, error) { +func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, error) { outputs := make([]responsesFunctionCallOutput, 0, len(results)) - payloadByCall := map[string]string{} for _, result := range results { payload, err := canonicalToolResultOutput(result) if err != nil { - return nil, nil, err + return nil, err } outputs = append( outputs, @@ -804,9 +812,8 @@ func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionC Status: "completed", }, ) - payloadByCall[result.ToolCallID] = payload } - return outputs, payloadByCall, nil + return outputs, nil } func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index c23a0a108..cab94f0b4 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -565,6 +565,40 @@ func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { } } +func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-already-submitted") + result := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + payload, err := canonicalToolResultOutput(result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + turn := &turnState{ + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{"call-1": result}, + bufferedPayloads: map[string]string{"call-1": payload}, + submittedPayloads: map[string]string{"call-1": payload}, + } + toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}) + if err != nil { + t.Fatalf("recordContinueResults: %v", err) + } + if toSubmit != nil { + t.Fatalf("toSubmit = %#v, want nil for already submitted duplicate", toSubmit) + } +} + func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{ scenario: "function_call", @@ -729,6 +763,33 @@ func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { } } +func TestResponsesAdapterAlreadySubmittedPendingResultIsNoop(t *testing.T) { + server := newServer(config{maxApprovalWait: time.Minute}, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-submitted-noop") + turn := &turnState{ + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + result := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + payload, err := canonicalToolResultOutput(result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + turn.submittedPayloads["call-1"] = payload + + toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}) + if err != nil { + t.Fatalf("recordContinueResults: %v", err) + } + if len(toSubmit) != 0 { + t.Fatalf("toSubmit = %#v, want duplicate submitted result to be a no-op", toSubmit) + } +} + func TestResponsesAdapterPendingTimeoutSkipsSubmittedCall(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -994,7 +1055,7 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { Input: "please read telemetry", }) - outputs, _, err := functionCallOutputs([]harness.ToolCallResult{{ + outputs, err := functionCallOutputs([]harness.ToolCallResult{{ Version: harness.ProtocolVersion, RuntimeSessionID: request.RuntimeSessionID, TurnID: request.TurnID, @@ -1313,7 +1374,7 @@ func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - outputs, _, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) + outputs, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) if err != nil { t.Fatalf("functionCallOutputs: %v", err) } From 85d7831323f58c13ba778fbea24b57bbe6a83d4f Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:07:00 -0700 Subject: [PATCH 13/51] fix: validate Foundry continuation before submit marker Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 19 ++++++++++++++----- .../harness/foundry-responses/main_test.go | 4 ++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 52d5eadcf..ebf4a8a7d 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "log" + "maps" "net/http" "net/url" "os" @@ -444,7 +445,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) return } - outputs, err := functionCallOutputs(resultsToSubmit) + outputs, payloadByCall, err := functionCallOutputs(resultsToSubmit) if err != nil { harness.WriteError(w, http.StatusBadRequest, err.Error()) return @@ -465,6 +466,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn AgentSessionID: foundrySessionID, Input: outputs, } + s.markSubmittedPayloads(turn, payloadByCall) if err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response); err != nil { s.mu.Lock() s.appendFailedLocked( @@ -766,7 +768,6 @@ func (s *server) recordContinueResults( continue } toSubmit = append(toSubmit, turn.bufferedResults[id]) - turn.submittedPayloads[id] = turn.bufferedPayloads[id] } if len(toSubmit) == 0 { return nil, nil @@ -774,6 +775,12 @@ func (s *server) recordContinueResults( return toSubmit, nil } +func (s *server) markSubmittedPayloads(turn *turnState, payloadByCall map[string]string) { + s.mu.Lock() + defer s.mu.Unlock() + maps.Copy(turn.submittedPayloads, payloadByCall) +} + func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []harness.ToolCallResult) error { s.mu.Lock() defer s.mu.Unlock() @@ -796,12 +803,13 @@ func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []ha return nil } -func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, error) { +func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, map[string]string, error) { outputs := make([]responsesFunctionCallOutput, 0, len(results)) + payloadByCall := map[string]string{} for _, result := range results { payload, err := canonicalToolResultOutput(result) if err != nil { - return nil, err + return nil, nil, err } outputs = append( outputs, @@ -812,8 +820,9 @@ func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionC Status: "completed", }, ) + payloadByCall[result.ToolCallID] = payload } - return outputs, nil + return outputs, payloadByCall, nil } func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index cab94f0b4..8b40c35be 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -1055,7 +1055,7 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { Input: "please read telemetry", }) - outputs, err := functionCallOutputs([]harness.ToolCallResult{{ + outputs, _, err := functionCallOutputs([]harness.ToolCallResult{{ Version: harness.ProtocolVersion, RuntimeSessionID: request.RuntimeSessionID, TurnID: request.TurnID, @@ -1374,7 +1374,7 @@ func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - outputs, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) + outputs, _, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) if err != nil { t.Fatalf("functionCallOutputs: %v", err) } From 7b1618545a062750339b4c9f27beeb15eae935b1 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:11:59 -0700 Subject: [PATCH 14/51] fix: align Foundry Responses continuation payload Signed-off-by: Sertac Ozercan --- .../harness/foundry-responses/main_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 8b40c35be..d16a788cb 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -599,6 +599,66 @@ func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { } } +func TestResponsesAdapterContinuesToolExecutionFailurePayload(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-tool-failure") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamAllFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + failure := harness.ContinueTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: request.Namespace, + TaskName: request.TaskName, + SessionName: request.SessionName, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + CorrelationID: request.CorrelationID, + ToolResults: []harness.ToolCallResult{toolResultForRequest( + request, + requested.ToolCallID, + true, + nil, + &harness.ErrorInfo{Code: "tool_execution_failed", Message: "downstream failed"}, + )}, + } + if _, err := client.ContinueTurn(context.Background(), failure); err != nil { + t.Fatalf("ContinueTurn tool failure: %v", err) + } + continuation := requestMap(t, foundry.requestBody(1)) + items, ok := continuation["input"].([]any) + if !ok || len(items) != 1 { + t.Fatalf("continuation input = %#v, want one item", continuation["input"]) + } + item, ok := items[0].(map[string]any) + if !ok { + t.Fatalf("continuation item = %#v, want object", items[0]) + } + wantOutput := `{"approved":false,"error":{"code":"tool_execution_failed","message":"downstream failed"}}` + if got := item["output"]; got != wantOutput { + t.Fatalf("failure output = %#v, want %s", got, wantOutput) + } + frames = streamAllFrames(t, client, request.TurnID) + toolResult := findFrame(frames, harness.FrameToolResultReceived) + if toolResult == nil || toolResult.Error == nil || toolResult.Error.Code != "tool_execution_failed" { + t.Fatalf("tool result frame = %#v, want tool_execution_failed", toolResult) + } + if !hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want final completion after tool failure continuation", frames) + } +} + func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{ scenario: "function_call", From d96c0fa73ca725546700eb159b854662c914c925 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:23:34 -0700 Subject: [PATCH 15/51] fix: harden Foundry Responses readiness and continuation claims Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 4 +- .../foundry-responses/kubernetes.example.yaml | 2 +- examples/harness/foundry-responses/main.go | 52 +++++++++++-------- .../harness/foundry-responses/main_test.go | 33 +++++++++++- 4 files changed, 63 insertions(+), 28 deletions(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 391d512e0..d66a75ae1 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -45,7 +45,7 @@ The hosted Responses endpoint must: - end in `/responses`; - not include username/password, fragments, or query parameters other than `api-version`. -The adapter returns degraded health and rejects starts when the endpoint is unsafe. +The adapter returns degraded health and rejects starts when the endpoint is unsafe. `GET /v1/health` always returns a harness health body; `GET /v1/ready` is a Kubernetes readiness helper that returns HTTP 503 until the same configuration is ready. ## Capability discipline @@ -76,7 +76,7 @@ Adapter emits `ToolCallRequested` with the exact `call_id`, function name, and c Orka continuation: ```json -{"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}","status":"completed"} +{"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}"} ``` The hosted continuation request includes `previous_response_id`, `agent_session_id` when a Foundry session is known, and one or more `function_call_output` items. Raw REST calls include the hosted-agent feature header required by Foundry hosted-agent endpoints. diff --git a/examples/harness/foundry-responses/kubernetes.example.yaml b/examples/harness/foundry-responses/kubernetes.example.yaml index 452dfefb3..70f095ee7 100644 --- a/examples/harness/foundry-responses/kubernetes.example.yaml +++ b/examples/harness/foundry-responses/kubernetes.example.yaml @@ -76,7 +76,7 @@ spec: containerPort: 8090 readinessProbe: httpGet: - path: /v1/health + path: /v1/ready port: http periodSeconds: 5 --- diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index ebf4a8a7d..a4afae3b6 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "log" - "maps" "net/http" "net/url" "os" @@ -32,6 +31,7 @@ const ( defaultMaxApprovalWait = 30 * time.Minute maxFoundryOutputBytes = 1 << 20 maxFoundryBodyBytes = 4 << 20 + readinessPath = "/v1/ready" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -204,6 +204,7 @@ func newServer(cfg config, client *http.Client) *server { func (s *server) handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc(harness.HealthPath, s.health) + mux.HandleFunc(readinessPath, s.ready) mux.HandleFunc(harness.CapabilitiesPath, s.capabilities) mux.HandleFunc(harness.TurnsPath, s.startTurn) mux.HandleFunc(harness.TurnsPath+"/", s.turn) @@ -242,6 +243,20 @@ func (s *server) health(w http.ResponseWriter, r *http.Request) { }) } +func (s *server) ready(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + _, endpointErr := s.responsesEndpoint() + ready := s.cfg.configError == "" && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + if !ready { + harness.WriteError(w, http.StatusServiceUnavailable, "adapter is not ready") + return + } + harness.WriteJSON(w, http.StatusOK, map[string]string{"status": "ready"}) +} + func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") @@ -435,6 +450,14 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn ) return } + s.mu.Lock() + previousResponseID := turn.responseID + foundrySessionID := turn.foundrySessionID + s.mu.Unlock() + if strings.TrimSpace(previousResponseID) == "" { + harness.WriteError(w, http.StatusConflict, "cannot continue before Foundry response id is known") + return + } resultsToSubmit, err := s.recordContinueResults(turn, req.ToolResults) if err != nil { @@ -445,19 +468,11 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) return } - outputs, payloadByCall, err := functionCallOutputs(resultsToSubmit) + outputs, err := functionCallOutputs(resultsToSubmit) if err != nil { harness.WriteError(w, http.StatusBadRequest, err.Error()) return } - s.mu.Lock() - previousResponseID := turn.responseID - foundrySessionID := turn.foundrySessionID - s.mu.Unlock() - if strings.TrimSpace(previousResponseID) == "" { - harness.WriteError(w, http.StatusConflict, "cannot continue before Foundry response id is known") - return - } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.requestTimeout) defer cancel() var response responsesResponse @@ -466,7 +481,6 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn AgentSessionID: foundrySessionID, Input: outputs, } - s.markSubmittedPayloads(turn, payloadByCall) if err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response); err != nil { s.mu.Lock() s.appendFailedLocked( @@ -768,6 +782,7 @@ func (s *server) recordContinueResults( continue } toSubmit = append(toSubmit, turn.bufferedResults[id]) + turn.submittedPayloads[id] = turn.bufferedPayloads[id] } if len(toSubmit) == 0 { return nil, nil @@ -775,12 +790,6 @@ func (s *server) recordContinueResults( return toSubmit, nil } -func (s *server) markSubmittedPayloads(turn *turnState, payloadByCall map[string]string) { - s.mu.Lock() - defer s.mu.Unlock() - maps.Copy(turn.submittedPayloads, payloadByCall) -} - func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []harness.ToolCallResult) error { s.mu.Lock() defer s.mu.Unlock() @@ -803,13 +812,12 @@ func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []ha return nil } -func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, map[string]string, error) { +func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionCallOutput, error) { outputs := make([]responsesFunctionCallOutput, 0, len(results)) - payloadByCall := map[string]string{} for _, result := range results { payload, err := canonicalToolResultOutput(result) if err != nil { - return nil, nil, err + return nil, err } outputs = append( outputs, @@ -817,12 +825,10 @@ func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionC Type: "function_call_output", CallID: result.ToolCallID, Output: payload, - Status: "completed", }, ) - payloadByCall[result.ToolCallID] = payload } - return outputs, payloadByCall, nil + return outputs, nil } func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index d16a788cb..fc4f488ce 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -247,6 +247,35 @@ func TestResponsesAdapterDuplicateStartDuringInitializationRejected(t *testing.T } } +func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { + unready := httptest.NewServer(newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: "https://example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", + requestTimeout: time.Second, + }, &http.Client{Timeout: time.Second}).handler()) + t.Cleanup(unready.Close) + resp, err := http.Get(unready.URL + readinessPath) //nolint:gosec,noctx // local test server + if err != nil { + t.Fatalf("GET unready: %v", err) + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("unready status = %d, want %d", resp.StatusCode, http.StatusServiceUnavailable) + } + + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + ready := newTestResponsesAdapter(t, foundry.endpoint(), nil) + readyResp, err := http.Get(ready.URL + readinessPath) //nolint:gosec,noctx // local test server + if err != nil { + t.Fatalf("GET ready: %v", err) + } + defer readyResp.Body.Close() //nolint:errcheck + if readyResp.StatusCode != http.StatusOK { + t.Fatalf("ready status = %d, want %d", readyResp.StatusCode, http.StatusOK) + } +} + func TestResponsesAdapterPassesObservedConformanceByDefault(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) @@ -1115,7 +1144,7 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { Input: "please read telemetry", }) - outputs, _, err := functionCallOutputs([]harness.ToolCallResult{{ + outputs, err := functionCallOutputs([]harness.ToolCallResult{{ Version: harness.ProtocolVersion, RuntimeSessionID: request.RuntimeSessionID, TurnID: request.TurnID, @@ -1434,7 +1463,7 @@ func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - outputs, _, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) + outputs, err := functionCallOutputs([]harness.ToolCallResult{tt.result}) if err != nil { t.Fatalf("functionCallOutputs: %v", err) } From 0a16837b1bc8e912f17bca749044d56ce5091701 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:26:09 -0700 Subject: [PATCH 16/51] fix: include Foundry continuation output status Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index a4afae3b6..8f1b19b40 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -825,6 +825,7 @@ func functionCallOutputs(results []harness.ToolCallResult) ([]responsesFunctionC Type: "function_call_output", CallID: result.ToolCallID, Output: payload, + Status: "completed", }, ) } From e73058d993b4e10bc094e58a927935dc77f1a94d Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:34:50 -0700 Subject: [PATCH 17/51] docs: show Foundry continuation output status Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index d66a75ae1..a6def7e16 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -76,7 +76,7 @@ Adapter emits `ToolCallRequested` with the exact `call_id`, function name, and c Orka continuation: ```json -{"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}"} +{"type":"function_call_output","call_id":"call_1","output":"{\"approved\":true,\"output\":{\"success\":true}}","status":"completed"} ``` The hosted continuation request includes `previous_response_id`, `agent_session_id` when a Foundry session is known, and one or more `function_call_output` items. Raw REST calls include the hosted-agent feature header required by Foundry hosted-agent endpoints. From ba9979763b5868960b5c72e63e2d3713dcce00ac Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:40:39 -0700 Subject: [PATCH 18/51] fix: fail incomplete Foundry Responses safely Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 29 ++++++++- .../harness/foundry-responses/main_test.go | 65 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 8f1b19b40..6770a5be4 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -571,6 +571,10 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) return } + if !isCompletionStatus(response.Status) { + s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) + return + } calls, err := s.extractFunctionCalls(turn.request, response.Output) if err != nil { s.appendFailedLocked(turn, "foundry_function_call_invalid", err.Error()) @@ -714,6 +718,8 @@ func (s *server) recordContinueResults( return nil, fmt.Errorf("no tool calls are pending for this turn") } now := time.Now().UTC() + newResults := map[string]harness.ToolCallResult{} + newPayloads := map[string]string{} for _, result := range results { payload, err := canonicalToolResultOutput(result) if err != nil { @@ -755,8 +761,18 @@ func (s *server) recordContinueResults( } return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) } - turn.bufferedResults[result.ToolCallID] = result - turn.bufferedPayloads[result.ToolCallID] = payload + if buffered, exists := newPayloads[result.ToolCallID]; exists { + if buffered == payload { + continue + } + return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) + } + newResults[result.ToolCallID] = result + newPayloads[result.ToolCallID] = payload + } + for id, result := range newResults { + turn.bufferedResults[id] = result + turn.bufferedPayloads[id] = newPayloads[id] } readyCount := 0 for id := range turn.pendingTools { @@ -1161,6 +1177,15 @@ func outputItemText(item responsesOutput) string { return "" } +func isCompletionStatus(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "", "completed": + return true + default: + return false + } +} + func isFailureStatus(status string) bool { switch strings.ToLower(strings.TrimSpace(status)) { case "failed", "cancelled", "expired", "incomplete": diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index fc4f488ce..e3ab6761d 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -594,6 +594,36 @@ func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { } } +func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-partial-reject") + valid := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + unknown := toolResultForRequest(request, "call-missing", true, json.RawMessage(`{"success":true}`), nil) + turn := &turnState{ + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{valid, unknown}); err == nil { + t.Fatalf("recordContinueResults succeeded, want unknown tool result error") + } + if len(turn.bufferedResults) != 0 || len(turn.bufferedPayloads) != 0 { + t.Fatalf("buffered state = %#v/%#v, want no partial buffering", turn.bufferedResults, turn.bufferedPayloads) + } +} + func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -1314,6 +1344,41 @@ func TestResponsesInitialPlatformErrorDoesNotRetainTurn(t *testing.T) { } } +//nolint:dupl // Mirrors failure-status regression with a distinct non-terminal status. +func TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: responsesStartTurnRequest("foundry-in-progress"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-in-progress", + Status: "in_progress", + Output: []responsesOutput{{Type: "message", Content: "partial text"}}, + }) + if hasFrameType(turn.frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, in-progress response should not complete", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_in_progress" { + t.Fatalf("failed frame = %#v, want foundry_in_progress", failed) + } +} + +//nolint:dupl // Mirrors non-terminal-status regression with a distinct failed status. func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { server := newServer(config{ runtimeName: "test", From 7b1239b2f177b745a088b36ceb8a48f1be777344 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:49:28 -0700 Subject: [PATCH 19/51] fix: require completed Foundry Responses status Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 11 ++-- .../harness/foundry-responses/main_test.go | 52 ++++++++++++++++--- .../golden/02_function_call_response.json | 1 + .../golden/06_final_message_response.json | 1 + .../golden/10_multiple_calls_response.json | 1 + 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 6770a5be4..33bc59d0f 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -571,6 +571,10 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) return } + if strings.TrimSpace(response.Status) == "" { + s.appendFailedLocked(turn, "foundry_status_missing", "Foundry hosted Responses status is missing") + return + } if !isCompletionStatus(response.Status) { s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) return @@ -1178,12 +1182,7 @@ func outputItemText(item responsesOutput) string { } func isCompletionStatus(status string) bool { - switch strings.ToLower(strings.TrimSpace(status)) { - case "", "completed": - return true - default: - return false - } + return strings.EqualFold(strings.TrimSpace(status), "completed") } func isFailureStatus(status string) bool { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index e3ab6761d..9390092a3 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -782,7 +782,8 @@ func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) server.handleResponsesResponse(turn, responsesResponse{ - ID: "resp-repeat", + ID: "resp-repeat", + Status: "completed", Output: []responsesOutput{{ Type: "function_call", CallID: "call-1", @@ -818,7 +819,8 @@ func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) server.handleResponsesResponse(turn, responsesResponse{ - ID: "resp-repeat", + ID: "resp-repeat", + Status: "completed", Output: []responsesOutput{ { Type: "function_call", @@ -865,7 +867,8 @@ func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { server.turns[turn.request.TurnID] = turn server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) server.handleResponsesResponse(turn, responsesResponse{ - ID: "resp-1", + ID: "resp-1", + Status: "completed", Output: []responsesOutput{{ Type: "function_call", CallID: "call-1", @@ -1313,7 +1316,8 @@ func TestResponsesLargeOutputFails(t *testing.T) { } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) server.handleResponsesResponse(turn, responsesResponse{ - ID: "resp-large", + ID: "resp-large", + Status: "completed", Output: []responsesOutput{{ Type: "message", Content: strings.Repeat("x", maxFoundryOutputBytes+1), @@ -1461,6 +1465,38 @@ func TestSanitizeEndpointDoesNotReturnRawMalformedURL(t *testing.T) { } } +func TestResponsesMissingStatusDoesNotCompleteWithPartialText(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: responsesStartTurnRequest("foundry-missing-status"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-missing-status", + Output: []responsesOutput{{Type: "message", Content: "partial text"}}, + }) + if hasFrameType(turn.frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, missing status response should not complete", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_status_missing" { + t.Fatalf("failed frame = %#v, want foundry_status_missing", failed) + } +} + func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -1482,6 +1518,7 @@ func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) server.handleResponsesResponse(turn, responsesResponse{ + Status: "completed", Output: []responsesOutput{{ Type: "function_call", CallID: "call-1", @@ -1612,7 +1649,8 @@ func newFakeResponses(t *testing.T, cfg fakeResponsesConfig) *fakeResponses { writeJSON( w, map[string]any{ - "id": "resp-1", + "id": "resp-1", + "status": "completed", "output": []any{ map[string]any{ "type": "function_call", @@ -1655,6 +1693,7 @@ func functionCallResponse(toolName string) map[string]any { return map[string]any{ "id": "resp-1", "agent_session_id": fakeSessionID, + "status": "completed", "output": []any{ map[string]any{ "type": "function_call", @@ -1667,7 +1706,7 @@ func functionCallResponse(toolName string) map[string]any { } func multipleCallsResponse() map[string]any { - return map[string]any{"id": "resp-1", "agent_session_id": fakeSessionID, "output": []any{ + return map[string]any{"id": "resp-1", "agent_session_id": fakeSessionID, "status": "completed", "output": []any{ map[string]any{ "type": "function_call", "call_id": "call-1", @@ -1687,6 +1726,7 @@ func finalResponsesMessage() map[string]any { return map[string]any{ "id": "resp-2", "agent_session_id": fakeSessionID, + "status": "completed", "output": []any{ map[string]any{ "type": "message", diff --git a/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json b/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json index 45aaeea30..dd61e135f 100644 --- a/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json +++ b/examples/harness/foundry-responses/testdata/golden/02_function_call_response.json @@ -1,5 +1,6 @@ { "id": "resp-1", + "status": "completed", "agent_session_id": "session-1", "output": [ { diff --git a/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json b/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json index 9f9ce60af..8086699ac 100644 --- a/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json +++ b/examples/harness/foundry-responses/testdata/golden/06_final_message_response.json @@ -1,5 +1,6 @@ { "id": "resp-2", + "status": "completed", "agent_session_id": "session-1", "output": [ { diff --git a/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json b/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json index 524114c33..cf2ba79c3 100644 --- a/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json +++ b/examples/harness/foundry-responses/testdata/golden/10_multiple_calls_response.json @@ -1,5 +1,6 @@ { "id": "resp-1", + "status": "completed", "agent_session_id": "session-1", "output": [ {"type":"function_call","call_id":"call-1","name":"support-ticket-lookup","arguments":"{\"incident\":\"inc-1\"}"}, From bb4c69d86eb61e3bae7a997115b6aed816f4467b Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 22:57:06 -0700 Subject: [PATCH 20/51] docs: add Foundry Responses validation matrix Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 2 + .../harness/foundry-responses/VALIDATION.md | 87 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 examples/harness/foundry-responses/VALIDATION.md diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index a6def7e16..ad02e5232 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -127,6 +127,8 @@ go test ./examples/harness/foundry-responses The tests use a fake hosted Responses server and golden fixtures for initial requests, function calls, `ToolCallRequested`, continuations, final messages, error encoding, and buffered multiple-call behavior. +See [`VALIDATION.md`](VALIDATION.md) for the brokered-plan evidence matrix, local commands, and remaining live Foundry/Fibey gates. + ## Troubleshooting | Symptom | Likely cause | Fix | diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md new file mode 100644 index 000000000..1038201ce --- /dev/null +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -0,0 +1,87 @@ +# Foundry hosted Responses validation matrix + +This document maps the brokered Foundry hosted Responses plan to deterministic +local evidence and the remaining live gates. It intentionally contains no live +endpoints, credentials, tokens, or Foundry project identifiers. + +## Local deterministic validation + +Run from the Orka repository root: + +```bash +go test ./examples/harness/foundry-responses -count=1 + +go test \ + ./examples/harness/foundry \ + ./examples/harness/foundry-responses \ + ./examples/harness/echo \ + ./internal/harness \ + ./internal/harness/conformance + +go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)' + +bash -n examples/**/*.sh +``` + +Full non-e2e validation: + +```bash +make test +``` + +When the sibling AgentKit checkout is available, run the shared deterministic +Foundry brokered protocol suite from the AgentKit checkout's `runtimes/common` directory: + +```bash +uv run --extra dev pytest -q \ + tests/test_foundry_brokered_protocol.py \ + tests/test_brokered_schema.py \ + tests/test_foundry_protocol.py +``` + +## Requirement evidence + +| Plan requirement | Local evidence | +| --- | --- | +| Separate hosted Responses adapter path | `examples/harness/foundry-responses/{main.go,main_test.go,README.md,Dockerfile}` | +| Existing Assistants adapter remains Assistants-only | `examples/harness/foundry/README.md` distinguishes Assistants/threads from hosted Responses. | +| Hosted `/responses` initial request does not send request-level `tools` | `TestResponsesAdapterObservedTurnCompletes`, `TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures`, and conformance tests assert no `tools` field reaches the fake hosted endpoint. | +| Endpoint URL safety | `TestResponsesEndpointSafety`, `TestResponsesAdapterDoesNotFollowCredentialedRedirects`, and `TestSanitizeEndpointDoesNotReturnRawMalformedURL`. | +| Mutating/streaming harness endpoints require bearer auth | `TestResponsesAdapterPassesObservedConformanceByDefault`, `TestResponsesAdapterPassesBrokeredReadConformance`, and `TestResponsesAdapterPassesBrokeredWriteConformance` run harness conformance with `RequireAuth`. | +| Observed hosted response maps to `TurnCompleted` | `TestResponsesAdapterObservedTurnCompletes` and observed conformance. | +| Responses `function_call` maps to `ToolCallRequested` | `TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures`, `TestResponsesConsumesAgentKitBrokeredFixtures`, and brokered read/write conformance. | +| Orka `ToolCallResult` maps to `function_call_output` | `TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures`, `TestResponsesAdapterWriteParksUntilDeclinedApprovalContinue`, `TestResponsesAdapterContinuesToolExecutionFailurePayload`, golden fixtures, and AgentKit fixture tests. | +| Canonical error/decline encoding is stable | `TestCanonicalErrorAndDeclineOutputFixtures` and `TestResponsesAgentKitErrorPayloadFixturesMatchCanonicalEncoding`. | +| Unknown tool and malformed arguments are rejected before Orka execution | `TestResponsesAdapterRejectsUnknownToolBeforeOrkaExecution` and `TestResponsesAdapterRejectsMalformedArguments`. | +| Multiple hosted calls are buffered until all results arrive | `TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued`. | +| Duplicate/replayed function calls fail closed | `TestResponsesRepeatedSubmittedFunctionCallFailsTurn` and `TestResponsesMixedRepeatedFunctionCallFailsTurn`. | +| Duplicate identical `/continue` is idempotent and conflicting duplicates reject | `TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject`, `TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit`, and `TestResponsesAdapterAlreadySubmittedPendingResultIsNoop`. | +| Continuation failures fail closed rather than duplicating hosted progress | `TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost`. | +| Pending approval/tool waits are bounded | `TestResponsesAdapterPendingToolTimesOutWithoutContinuation`, `TestResponsesAdapterPendingTimeoutSkipsSubmittedCall`, and `TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait`. | +| Restart/state loss fails safely without hosted continuation | `TestResponsesAdapterStateLossContinueFailsSafely`. | +| Runtime session continuity | `TestResponsesAdapterRuntimeSessionHeaderReuse` and `TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup`. | +| Hosted response status handling is fail-closed | `TestResponsesFailureStatusDoesNotCompleteWithPartialText`, `TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest`, `TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText`, and `TestResponsesMissingStatusDoesNotCompleteWithPartialText`. | +| Large hosted output and platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorDoesNotRetainTurn`. | +| No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | +| Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | +| Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | + +## Live gates that cannot be satisfied by local fixtures + +The following remain required before declaring the full hosted Foundry/Fibey plan +complete: + +1. Deploy an AgentKit prototype as a real Foundry hosted agent with static safe + brokered schemas and a configured brokered continuation proof. +2. Deploy the Orka `foundry-responses` adapter with real Foundry auth and a real + hosted `/responses` endpoint. +3. Verify `AgentRuntime` readiness for the read profile. +4. Run a read task and verify `ToolCallRequested` then successful completion. +5. Enable write only after the hosted AgentKit deployment has a static write + schema and passes write conformance. +6. Run a write task and verify `ApprovalRequested`, no downstream execution + before approval, approved continuation, idempotency key delivery, and no + credential/tool URL leakage in logs. +7. Run the Fibey read/write scenario with `check-network-telemetry`, + `get-active-incidents`, `dispatch-work-order`, and optional + `escalate-incident`, then verify replay produces no second dispatch. From 3b9bebcfc9f1298f37d4befac8ba20024b3e01f4 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 23:04:58 -0700 Subject: [PATCH 21/51] docs: add Foundry Responses validation helper Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 6 ++ .../harness/foundry-responses/VALIDATION.md | 11 ++- .../harness/foundry-responses/validate.sh | 96 +++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100755 examples/harness/foundry-responses/validate.sh diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index ad02e5232..23d58526b 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -125,6 +125,12 @@ For write-profile smoke, first prove the hosted AgentKit deployment has a static go test ./examples/harness/foundry-responses ``` +For the full deterministic local validation bundle, including the focused Orka harness/controller suites and optional sibling AgentKit fixture tests when available: + +```bash +examples/harness/foundry-responses/validate.sh +``` + The tests use a fake hosted Responses server and golden fixtures for initial requests, function calls, `ToolCallRequested`, continuations, final messages, error encoding, and buffered multiple-call behavior. See [`VALIDATION.md`](VALIDATION.md) for the brokered-plan evidence matrix, local commands, and remaining live Foundry/Fibey gates. diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 1038201ce..e8786a275 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -6,7 +6,13 @@ endpoints, credentials, tokens, or Foundry project identifiers. ## Local deterministic validation -Run from the Orka repository root: +Run the bundled deterministic validator from the Orka repository root: + +```bash +examples/harness/foundry-responses/validate.sh +``` + +Or run the component commands manually: ```bash go test ./examples/harness/foundry-responses -count=1 @@ -20,7 +26,8 @@ go test \ go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)' -bash -n examples/**/*.sh +find examples -type f -name '*.sh' -print0 | sort -z | \ + xargs -0 -n1 bash -n ``` Full non-e2e validation: diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh new file mode 100755 index 000000000..77b171a6b --- /dev/null +++ b/examples/harness/foundry-responses/validate.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: examples/harness/foundry-responses/validate.sh [--agentkit PATH] [--full] + +Runs deterministic local validation for the Orka Foundry hosted Responses adapter. +No live Foundry endpoint, token, API key, or Kubernetes cluster is required. + +Options: + --agentkit PATH Also run AgentKit's deterministic Foundry brokered protocol + tests from PATH/runtimes/common. If omitted, the script uses + a sibling ../agentkit checkout when present. + --full Also run `make test` for the full non-e2e Orka suite. + -h, --help Show this help. +USAGE +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +agentkit_root="" +run_full=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --agentkit) + [[ $# -ge 2 ]] || { echo "--agentkit requires a path" >&2; exit 2; } + agentkit_root="$2" + shift 2 + ;; + --full) + run_full=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +run() { + printf '\n==> %s\n' "$*" >&2 + "$@" +} + +cd "$repo_root" + +run go test ./examples/harness/foundry-responses -count=1 +run go test \ + ./examples/harness/foundry \ + ./examples/harness/foundry-responses \ + ./examples/harness/echo \ + ./internal/harness \ + ./internal/harness/conformance +run go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)' +while IFS= read -r -d '' script; do + run bash -n "$script" +done < <(find examples -type f -name '*.sh' -print0 | sort -z) + +if [[ "$run_full" == "1" ]]; then + run make test +fi + +if [[ -z "$agentkit_root" ]]; then + sibling_agentkit="$(cd "$repo_root/.." && pwd)/agentkit" + if [[ -d "$sibling_agentkit/runtimes/common" ]]; then + agentkit_root="$sibling_agentkit" + fi +fi + +if [[ -n "$agentkit_root" ]]; then + common_dir="$agentkit_root/runtimes/common" + if [[ ! -d "$common_dir" ]]; then + echo "AgentKit common runtime directory not found: $common_dir" >&2 + exit 2 + fi + if ! command -v uv >/dev/null 2>&1; then + echo "uv is required for AgentKit validation but was not found in PATH" >&2 + exit 2 + fi + ( + cd "$common_dir" + run uv run --extra dev pytest -q \ + tests/test_foundry_brokered_protocol.py \ + tests/test_brokered_schema.py \ + tests/test_foundry_protocol.py + ) +fi + +printf '\nFoundry hosted Responses local validation passed.\n' >&2 From 99f50bfe58e6f72ec775b970778c31d519b175da Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 23:30:55 -0700 Subject: [PATCH 22/51] docs: add Foundry Responses live smoke helper Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 7 + .../harness/foundry-responses/VALIDATION.md | 13 + .../harness/foundry-responses/live-smoke.sh | 354 ++++++++++++++++++ 3 files changed, 374 insertions(+) create mode 100755 examples/harness/foundry-responses/live-smoke.sh diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 23d58526b..556ef902e 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -131,6 +131,13 @@ For the full deterministic local validation bundle, including the focused Orka h examples/harness/foundry-responses/validate.sh ``` +For the live Foundry hosted AgentKit smoke gate, first run the credentials-safe preflight and then apply only when your current Kubernetes context is the intended Orka cluster: + +```bash +examples/harness/foundry-responses/live-smoke.sh +examples/harness/foundry-responses/live-smoke.sh --apply --wait +``` + The tests use a fake hosted Responses server and golden fixtures for initial requests, function calls, `ToolCallRequested`, continuations, final messages, error encoding, and buffered multiple-call behavior. See [`VALIDATION.md`](VALIDATION.md) for the brokered-plan evidence matrix, local commands, and remaining live Foundry/Fibey gates. diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index e8786a275..6912889e1 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -75,6 +75,19 @@ uv run --extra dev pytest -q \ ## Live gates that cannot be satisfied by local fixtures +Use the credentials-safe live smoke helper as the first live preflight/deploy step: + +```bash +examples/harness/foundry-responses/live-smoke.sh +examples/harness/foundry-responses/live-smoke.sh --apply --wait +``` + +The helper validates required environment without printing secret values and can +deploy the adapter `Deployment`, `Service`, `Secret` placeholders, and matching +`AgentRuntime` facade into the selected namespace. It does not replace the +required real hosted AgentKit deployment, downstream tools, human approvals, or +Fibey scenario verification. + The following remain required before declaring the full hosted Foundry/Fibey plan complete: diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh new file mode 100755 index 000000000..481bd3a03 --- /dev/null +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: examples/harness/foundry-responses/live-smoke.sh [--apply] [--wait] [--namespace NAME] + +Credentials-safe preflight/deploy helper for the live Foundry hosted AgentKit +Responses smoke gate. By default this performs preflight only. With --apply it +creates/updates the namespace-local adapter Deployment, Service, Secrets, and +AgentRuntime facade, then optionally waits for readiness with --wait. + +Required environment for preflight/apply: + ORKA_FOUNDRY_RESPONSES_ENDPOINT + Full hosted Responses URL, OR set both: + ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT + ORKA_FOUNDRY_RESPONSES_AGENT_NAME + Exactly one Foundry auth value: + ORKA_FOUNDRY_RESPONSES_API_KEY + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER + +Optional environment: + ORKA_FOUNDRY_RESPONSES_NAMESPACE default: foundry-responses-smoke + ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME default: sample-foundry-responses-runtime + ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE default: ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest + ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN generated if absent for this run + ORKA_FOUNDRY_RESPONSES_API_VERSION default: v1 + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES default: read + ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF optional + +The script never prints secret values. Do not run with shell tracing (set -x). +USAGE +} + +apply=0 +wait_ready=0 +namespace="${ORKA_FOUNDRY_RESPONSES_NAMESPACE:-foundry-responses-smoke}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --apply) + apply=1 + shift + ;; + --wait) + wait_ready=1 + shift + ;; + --namespace) + [[ $# -ge 2 ]] || { echo "--namespace requires a value" >&2; exit 2; } + namespace="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +if [[ "$wait_ready" == "1" && "$apply" != "1" ]]; then + echo "error: --wait requires --apply" >&2 + exit 2 +fi + +runtime_name="${ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME:-sample-foundry-responses-runtime}" +service_url="http://${runtime_name}.${namespace}.svc.cluster.local:8080" +adapter_image="${ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE:-ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest}" +api_version="${ORKA_FOUNDRY_RESPONSES_API_VERSION:-v1}" +brokered_classes="${ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES:-read}" +endpoint="${ORKA_FOUNDRY_RESPONSES_ENDPOINT:-}" +project_endpoint="${ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT:-}" +agent_name="${ORKA_FOUNDRY_RESPONSES_AGENT_NAME:-}" +api_key="${ORKA_FOUNDRY_RESPONSES_API_KEY:-}" +auth_bearer="${ORKA_FOUNDRY_RESPONSES_AUTH_BEARER:-}" +adapter_bearer="${ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN:-}" +continuation_proof="${ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF:-}" +rollout_nonce="$(date -u +%Y%m%dT%H%M%SZ)" + +fail() { + echo "error: $*" >&2 + exit 2 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "$1 is required" +} + +is_https_or_loopback_http() { + local value="$1" + local rest authority + [[ -z "$value" ]] && return 0 + [[ "$value" != *$'\n'* && "$value" != *$'\r'* && "$value" != *$'\t'* ]] || return 1 + if [[ "$value" == https://* ]]; then + rest="${value#https://}" + authority="${rest%%/*}" + authority="${authority%%\?*}" + authority="${authority%%#*}" + [[ -n "$authority" && "$authority" != *@* && "$authority" != *[[:space:]]* ]] + return + fi + [[ "$value" == http://* ]] || return 1 + rest="${value#http://}" + authority="${rest%%/*}" + authority="${authority%%\?*}" + authority="${authority%%#*}" + [[ -n "$authority" && "$authority" != *@* ]] || return 1 + if [[ "$authority" == \[* ]]; then + [[ "$authority" =~ ^\[::1\](:[0-9]+)?$ ]] + else + [[ "$authority" =~ ^(localhost|127\.0\.0\.1)(:[0-9]+)?$ ]] + fi +} + +require_nonempty_name() { + local label="$1" + local value="$2" + [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || \ + fail "$label must be a Kubernetes DNS label, got '$value'" +} + +indent_block() { + sed 's/^/ /' +} + +preflight() { + require_nonempty_name namespace "$namespace" + require_nonempty_name "runtime name" "$runtime_name" + + if [[ -z "$endpoint" ]]; then + [[ -n "$project_endpoint" && -n "$agent_name" ]] || \ + fail "set ORKA_FOUNDRY_RESPONSES_ENDPOINT or PROJECT_ENDPOINT plus AGENT_NAME" + fi + is_https_or_loopback_http "$endpoint" || fail "ORKA_FOUNDRY_RESPONSES_ENDPOINT must be https (or loopback http for tests)" + is_https_or_loopback_http "$project_endpoint" || fail "ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT must be https (or loopback http for tests)" + + if [[ -n "$api_key" && -n "$auth_bearer" ]]; then + fail "set exactly one of ORKA_FOUNDRY_RESPONSES_API_KEY or ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" + fi + if [[ -z "$api_key" && -z "$auth_bearer" ]]; then + fail "set one Foundry auth value: ORKA_FOUNDRY_RESPONSES_API_KEY or ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" + fi + + IFS=',' read -r -a classes <<<"$brokered_classes" + for class in "${classes[@]}"; do + class="${class//[[:space:]]/}" + [[ "$class" == "read" || "$class" == "write" ]] || fail "unsupported brokered class '$class' (expected read/write)" + done + + if [[ "$apply" == "1" || "$wait_ready" == "1" ]]; then + require_cmd kubectl + fi +} + +random_bearer() { + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 32 + else + od -An -N32 -tx1 /dev/urandom | tr -d ' \n' + printf '\n' + fi +} + + +decode_base64() { + if printf '' | base64 --decode >/dev/null 2>&1; then + base64 --decode + else + base64 -D + fi +} + +encode_base64() { + printf '%s' "$1" | base64 | tr -d '[:space:]' +} + +existing_adapter_bearer() { + local encoded + encoded="$(kubectl -n "$namespace" get secret "${runtime_name}-token" -o jsonpath='{.data.harness-bearer}' 2>/dev/null || true)" + [[ -n "$encoded" ]] || return 1 + printf '%s' "$encoded" | decode_base64 +} + +emit_secret_yaml() { + local foundry_key_name="foundry-auth" + local foundry_value="$api_key" + if [[ -n "$auth_bearer" ]]; then + foundry_key_name="foundry-bearer" + foundry_value="$auth_bearer" + fi + + cat <&2 + exit 0 +fi + +kubectl get namespace "$namespace" >/dev/null 2>&1 || kubectl create namespace "$namespace" >/dev/null +emit_secret_yaml | kubectl -n "$namespace" apply --server-side --field-manager=orka-foundry-responses-live-smoke -f - >/dev/null +kubectl -n "$namespace" annotate secret "${runtime_name}-token" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true +kubectl -n "$namespace" annotate secret "${runtime_name}-adapter-config" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true +emit_runtime_yaml | kubectl -n "$namespace" apply -f - + +echo "Applied Foundry Responses adapter smoke resources in namespace '$namespace'." >&2 + +if [[ "$wait_ready" == "1" ]]; then + kubectl -n "$namespace" rollout status "deployment/${runtime_name}" --timeout=120s + kubectl -n "$namespace" wait --for=condition=Ready "agentruntime/${runtime_name}" --timeout=120s +fi From 25e313d7745aee0ce31cff22fd51b5346dfb916a Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 23:35:26 -0700 Subject: [PATCH 23/51] fix: omit empty Foundry continuation proof secret Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/live-smoke.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 481bd3a03..448aef49f 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -213,8 +213,10 @@ metadata: name: "${runtime_name}-adapter-config" data: ${foundry_key_name}: $(encode_base64 "$foundry_value") - continuation-proof: $(encode_base64 "$continuation_proof") YAML + if [[ -n "$continuation_proof" ]]; then + printf ' continuation-proof: %s\n' "$(encode_base64 "$continuation_proof")" + fi } emit_runtime_yaml() { From 8315213934c9b1f3185dc2d3a1ebf0a20de115b1 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 23:40:12 -0700 Subject: [PATCH 24/51] fix: preflight Foundry Responses live manifests before secrets Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/live-smoke.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 448aef49f..30d77830a 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -343,10 +343,15 @@ if [[ "$apply" != "1" ]]; then fi kubectl get namespace "$namespace" >/dev/null 2>&1 || kubectl create namespace "$namespace" >/dev/null +# Validate CRDs/admission for non-secret resources before writing live credentials. +emit_runtime_yaml | kubectl -n "$namespace" apply --server-side --dry-run=server -f - >/dev/null emit_secret_yaml | kubectl -n "$namespace" apply --server-side --field-manager=orka-foundry-responses-live-smoke -f - >/dev/null kubectl -n "$namespace" annotate secret "${runtime_name}-token" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true kubectl -n "$namespace" annotate secret "${runtime_name}-adapter-config" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true emit_runtime_yaml | kubectl -n "$namespace" apply -f - +if kubectl -n "$namespace" get "deployment/${runtime_name}" >/dev/null 2>&1; then + kubectl -n "$namespace" rollout restart "deployment/${runtime_name}" >/dev/null +fi echo "Applied Foundry Responses adapter smoke resources in namespace '$namespace'." >&2 From 0258876f6eff3e3b34a14965c66c5f739e376c07 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 8 Jul 2026 23:46:11 -0700 Subject: [PATCH 25/51] fix: align Foundry Responses live apply mode Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/live-smoke.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 30d77830a..efa7bcf64 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -344,11 +344,11 @@ fi kubectl get namespace "$namespace" >/dev/null 2>&1 || kubectl create namespace "$namespace" >/dev/null # Validate CRDs/admission for non-secret resources before writing live credentials. -emit_runtime_yaml | kubectl -n "$namespace" apply --server-side --dry-run=server -f - >/dev/null +emit_runtime_yaml | kubectl -n "$namespace" apply --server-side --field-manager=orka-foundry-responses-live-smoke --dry-run=server -f - >/dev/null emit_secret_yaml | kubectl -n "$namespace" apply --server-side --field-manager=orka-foundry-responses-live-smoke -f - >/dev/null kubectl -n "$namespace" annotate secret "${runtime_name}-token" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true kubectl -n "$namespace" annotate secret "${runtime_name}-adapter-config" kubectl.kubernetes.io/last-applied-configuration- --overwrite >/dev/null 2>&1 || true -emit_runtime_yaml | kubectl -n "$namespace" apply -f - +emit_runtime_yaml | kubectl -n "$namespace" apply --server-side --field-manager=orka-foundry-responses-live-smoke -f - if kubectl -n "$namespace" get "deployment/${runtime_name}" >/dev/null 2>&1; then kubectl -n "$namespace" rollout restart "deployment/${runtime_name}" >/dev/null fi From 1cbd65f48254b2c1c10773766f48e81928127739 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 00:17:51 -0700 Subject: [PATCH 26/51] test: add Fibey Foundry Responses verifier Signed-off-by: Sertac Ozercan --- examples/fibey-custom-agent-demo/README.md | 5 + .../verify-foundry-responses.sh | 260 ++++++++++++++++++ examples/harness/foundry-responses/README.md | 2 +- .../harness/foundry-responses/VALIDATION.md | 19 +- .../harness/foundry-responses/validate.sh | 11 +- 5 files changed, 284 insertions(+), 13 deletions(-) create mode 100755 examples/fibey-custom-agent-demo/verify-foundry-responses.sh diff --git a/examples/fibey-custom-agent-demo/README.md b/examples/fibey-custom-agent-demo/README.md index cebbc3985..11be51647 100644 --- a/examples/fibey-custom-agent-demo/README.md +++ b/examples/fibey-custom-agent-demo/README.md @@ -116,6 +116,11 @@ kubectl wait --for=condition=Ready agentruntime/fibey-agentkit-foundry-responses # Optional literal brokered Fibey read/write scenario once downstream services exist. kubectl apply -f examples/fibey-custom-agent-demo/tools-foundry-responses.yaml kubectl apply -f examples/fibey-custom-agent-demo/task-foundry-responses.yaml + +# After the task runs through read, write approval, and completion, verify evidence. +examples/fibey-custom-agent-demo/verify-foundry-responses.sh \ + --task fibey-foundry-responses-quincy-north-alert \ + --namespace default ``` Run the same task against another backend by changing only `spec.agentRef.name`, for example: diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh new file mode 100755 index 000000000..90f467ffb --- /dev/null +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: examples/fibey-custom-agent-demo/verify-foundry-responses.sh [--task NAME] [--namespace NAME] [--json EVENTS.json] + +Checks the live Fibey Foundry hosted AgentKit Responses scenario evidence from +Orka task events. By default it calls `orka task events --output json`. Use +--json to verify a previously captured event payload without contacting Orka. + +Expected evidence: + - read brokered tool request for check-network-telemetry or get-active-incidents + - write brokered tool request for dispatch-work-order or escalate-incident + - ApprovalRequested is present before write ToolCallStarted + - an idempotency key is present in write ToolCallStarted content + - terminal TaskSucceeded/AgentRuntimeCompleted/TurnCompleted-style event exists + +This verifier does not approve tasks and never reads Foundry credentials. +USAGE +} + +task="fibey-foundry-responses-quincy-north-alert" +namespace="default" +json_file="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --task) + [[ $# -ge 2 ]] || { echo "--task requires a value" >&2; exit 2; } + task="$2" + shift 2 + ;; + --namespace) + [[ $# -ge 2 ]] || { echo "--namespace requires a value" >&2; exit 2; } + namespace="$2" + shift 2 + ;; + --json) + [[ $# -ge 2 ]] || { echo "--json requires a path" >&2; exit 2; } + json_file="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { echo "error: $1 is required" >&2; exit 2; } +} + +require_cmd python3 + +json_tmp="" +cleanup() { + [[ -z "$json_tmp" ]] || rm -f "$json_tmp" +} +trap cleanup EXIT + +if [[ -z "$json_file" ]]; then + require_cmd orka + json_tmp="$(mktemp)" + orka task events "$task" --namespace "$namespace" --output json >"$json_tmp" + json_file="$json_tmp" +fi + +python3 - "$json_file" <<'PY' +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +try: + payload = json.loads(path.read_text()) +except Exception as exc: # noqa: BLE001 - user-facing validation script + raise SystemExit(f"error: read event JSON: {exc}") + +if isinstance(payload, dict): + if isinstance(payload.get("events"), list): + events = payload["events"] + elif isinstance(payload.get("items"), list): + events = payload["items"] + else: + events = [payload] +elif isinstance(payload, list): + events = payload +else: + raise SystemExit("error: event JSON must be an object or list") + +READ_TOOLS = {"check-network-telemetry", "get-active-incidents"} +WRITE_TOOLS = {"dispatch-work-order", "escalate-incident"} +TERMINAL_TYPES = { + "TaskSucceeded", + "AgentRuntimeCompleted", + "TurnCompleted", + "TaskCompleted", +} +WRITE_EXEC_TYPES = {"ToolCallStarted"} + + +def field(event, name): + if not isinstance(event, dict): + return None + if name in event: + return event[name] + content = event.get("content") + if isinstance(content, dict) and name in content: + return content[name] + if isinstance(content, str): + try: + decoded = json.loads(content) + except Exception: # noqa: BLE001 + decoded = None + if isinstance(decoded, dict) and name in decoded: + return decoded[name] + return None + + +def event_type(event): + if not isinstance(event, dict): + return "" + return str(event.get("type") or event.get("eventType") or "") + + +def tool_name(event): + for key in ("toolName", "tool", "name"): + value = field(event, key) + if isinstance(value, str) and value: + return value + return "" + + +def seq(event): + if not isinstance(event, dict): + return None + for name in ("seq", "sequence", "_verifyOrder", "id"): + value = event.get(name) + if value is None: + continue + try: + return int(value) + except Exception: # noqa: BLE001 + continue + return None + + +def idempotency_value(value): + if isinstance(value, dict): + for key, nested in value.items(): + if key in {"idempotencyKey", "Idempotency-Key"}: + if isinstance(nested, str) and nested.strip(): + return nested.strip() + found = idempotency_value(nested) + if found: + return found + elif isinstance(value, list): + for item in value: + found = idempotency_value(item) + if found: + return found + elif isinstance(value, str): + try: + decoded = json.loads(value) + except Exception: # noqa: BLE001 + return "" + return idempotency_value(decoded) + return "" + + +def contains_idempotency(event): + return bool(idempotency_value(event)) + + +ordered_events = [] +for index, event in enumerate(events, start=1): + if isinstance(event, dict) and seq(event) is None: + copied = dict(event) + copied["_verifyOrder"] = index + ordered_events.append(copied) + else: + ordered_events.append(event) + +events = ordered_events +read_events = [e for e in events if tool_name(e) in READ_TOOLS] +write_events = [e for e in events if tool_name(e) in WRITE_TOOLS] +approval_events = [e for e in events if event_type(e) == "ApprovalRequested"] +write_exec_events = [e for e in write_events if event_type(e) in WRITE_EXEC_TYPES] +write_start_events = [e for e in write_events if event_type(e) == "ToolCallStarted"] +terminal_events = [e for e in events if event_type(e) in TERMINAL_TYPES] +idempotency_events = [e for e in write_exec_events if idempotency_value(e)] + +failures = [] +if not read_events: + failures.append("missing read brokered tool event for check-network-telemetry/get-active-incidents") +if not write_events: + failures.append("missing write brokered tool event for dispatch-work-order/escalate-incident") +if not approval_events: + failures.append("missing ApprovalRequested event") +if not write_exec_events: + failures.append("missing write ToolCallStarted event after approval") +if write_exec_events and approval_events: + for event in write_exec_events: + write_tool = tool_name(event) + write_order = seq(event) + matching_approvals = [ + approval for approval in approval_events + if tool_name(approval) == write_tool and seq(approval) < write_order + ] + if not matching_approvals: + failures.append(f"write execution for {write_tool} has no preceding approval") +if not idempotency_events: + failures.append("missing write ToolCallStarted idempotency key evidence") + +missing_idempotency_tools = sorted( + tool for tool in {tool_name(event) for event in write_exec_events} + if tool not in {tool_name(event) for event in idempotency_events} +) +if missing_idempotency_tools: + failures.append( + "missing write execution idempotency key evidence for: " + ", ".join(missing_idempotency_tools) + ) + +starts_by_tool = {} +for event in write_start_events: + starts_by_tool.setdefault(tool_name(event), 0) + starts_by_tool[tool_name(event)] += 1 +for write_tool, count in starts_by_tool.items(): + if count > 1: + failures.append(f"duplicate write execution starts for {write_tool}") + +idempotency_by_tool = {} +for event in idempotency_events: + idempotency_by_tool.setdefault(tool_name(event), set()).add(idempotency_value(event)) +for write_tool, keys in idempotency_by_tool.items(): + if len(keys) > 1: + failures.append(f"multiple write idempotency keys for {write_tool}") +if not terminal_events: + failures.append("missing terminal completion event") + +if failures: + print("Fibey Foundry Responses verification failed:", file=sys.stderr) + for failure in failures: + print(f"- {failure}", file=sys.stderr) + raise SystemExit(1) + +print("Fibey Foundry Responses verification passed:") +print(f"- read events: {len(read_events)}") +print(f"- write events: {len(write_events)}") +print(f"- approvals: {len(approval_events)}") +print(f"- idempotency evidence events: {len(idempotency_events)}") +print(f"- terminal events: {len(terminal_events)}") +PY diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 556ef902e..1c283fa7f 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -125,7 +125,7 @@ For write-profile smoke, first prove the hosted AgentKit deployment has a static go test ./examples/harness/foundry-responses ``` -For the full deterministic local validation bundle, including the focused Orka harness/controller suites and optional sibling AgentKit fixture tests when available: +For the full deterministic local validation bundle, including the focused Orka harness/controller suites and explicit AgentKit fixture tests when --agentkit is provided: ```bash examples/harness/foundry-responses/validate.sh diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 6912889e1..b35b94f94 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -36,8 +36,14 @@ Full non-e2e validation: make test ``` -When the sibling AgentKit checkout is available, run the shared deterministic -Foundry brokered protocol suite from the AgentKit checkout's `runtimes/common` directory: +When a clean AgentKit checkout is available, run the shared deterministic +Foundry brokered protocol suite explicitly: + +```bash +examples/harness/foundry-responses/validate.sh --agentkit ../agentkit +``` + +Equivalent manual command from the AgentKit checkout's `runtimes/common` directory: ```bash uv run --extra dev pytest -q \ @@ -104,4 +110,11 @@ complete: credential/tool URL leakage in logs. 7. Run the Fibey read/write scenario with `check-network-telemetry`, `get-active-incidents`, `dispatch-work-order`, and optional - `escalate-incident`, then verify replay produces no second dispatch. + `escalate-incident`, then verify replay produces no second dispatch. Capture + Orka task events and run: + + ```bash + examples/fibey-custom-agent-demo/verify-foundry-responses.sh \ + --task fibey-foundry-responses-quincy-north-alert \ + --namespace + ``` diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 77b171a6b..aa3f20717 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -10,8 +10,8 @@ No live Foundry endpoint, token, API key, or Kubernetes cluster is required. Options: --agentkit PATH Also run AgentKit's deterministic Foundry brokered protocol - tests from PATH/runtimes/common. If omitted, the script uses - a sibling ../agentkit checkout when present. + tests from PATH/runtimes/common. This is explicit because a + sibling AgentKit checkout may contain unrelated local changes. --full Also run `make test` for the full non-e2e Orka suite. -h, --help Show this help. USAGE @@ -67,13 +67,6 @@ if [[ "$run_full" == "1" ]]; then run make test fi -if [[ -z "$agentkit_root" ]]; then - sibling_agentkit="$(cd "$repo_root/.." && pwd)/agentkit" - if [[ -d "$sibling_agentkit/runtimes/common" ]]; then - agentkit_root="$sibling_agentkit" - fi -fi - if [[ -n "$agentkit_root" ]]; then common_dir="$agentkit_root/runtimes/common" if [[ ! -d "$common_dir" ]]; then From 29b0daf56857bb1e8e0605247feea709fb7a645a Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 00:21:45 -0700 Subject: [PATCH 27/51] fix: require Fibey read request evidence Signed-off-by: Sertac Ozercan --- examples/fibey-custom-agent-demo/verify-foundry-responses.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 90f467ffb..242a43ca5 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -103,6 +103,7 @@ TERMINAL_TYPES = { "TurnCompleted", "TaskCompleted", } +READ_REQUEST_TYPES = {"ToolCallRequested", "ToolCallStarted"} WRITE_EXEC_TYPES = {"ToolCallStarted"} @@ -189,7 +190,7 @@ for index, event in enumerate(events, start=1): ordered_events.append(event) events = ordered_events -read_events = [e for e in events if tool_name(e) in READ_TOOLS] +read_events = [e for e in events if tool_name(e) in READ_TOOLS and event_type(e) in READ_REQUEST_TYPES] write_events = [e for e in events if tool_name(e) in WRITE_TOOLS] approval_events = [e for e in events if event_type(e) == "ApprovalRequested"] write_exec_events = [e for e in write_events if event_type(e) in WRITE_EXEC_TYPES] From 3ea86c53368526f6e5008e0197b1e605ef70721b Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 00:58:36 -0700 Subject: [PATCH 28/51] test: add Foundry Responses live evidence verifier Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 6 + .../harness/foundry-responses/VALIDATION.md | 9 + .../foundry-responses/live-evidence.sh | 395 ++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100755 examples/harness/foundry-responses/live-evidence.sh diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 1c283fa7f..e9dda8ad1 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -136,6 +136,12 @@ For the live Foundry hosted AgentKit smoke gate, first run the credentials-safe ```bash examples/harness/foundry-responses/live-smoke.sh examples/harness/foundry-responses/live-smoke.sh --apply --wait + +# After the live task completes, capture redacted evidence. +examples/harness/foundry-responses/live-evidence.sh \ + --namespace \ + --runtime fibey-agentkit-foundry-responses \ + --task fibey-foundry-responses-quincy-north-alert ``` The tests use a fake hosted Responses server and golden fixtures for initial requests, function calls, `ToolCallRequested`, continuations, final messages, error encoding, and buffered multiple-call behavior. diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index b35b94f94..001e5737a 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -118,3 +118,12 @@ complete: --task fibey-foundry-responses-quincy-north-alert \ --namespace ``` + + To collect a credentials-safe evidence bundle after the run: + + ```bash + examples/harness/foundry-responses/live-evidence.sh \ + --namespace \ + --runtime fibey-agentkit-foundry-responses \ + --task fibey-foundry-responses-quincy-north-alert + ``` diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh new file mode 100755 index 000000000..b3ffdd6fe --- /dev/null +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: examples/harness/foundry-responses/live-evidence.sh [--namespace NAME] [--runtime NAME] [--task NAME] [--out DIR] [--logs-since DURATION] + +Capture a credentials-safe evidence bundle for the live Foundry hosted +Responses/Fibey gate after the task has run. The bundle stores Kubernetes +AgentRuntime metadata, Orka task events/approvals, verifier output, and a +summary-only adapter log scan. It intentionally does not store raw adapter logs. + +Defaults: + --namespace default + --runtime fibey-agentkit-foundry-responses + --task fibey-foundry-responses-quincy-north-alert + --out ./foundry-responses-live-evidence- + --logs-since empty, meaning scan all available deployment logs. Set a kubectl + duration such as 30m only when the run start time is known. + +Required commands: kubectl, orka, python3. +USAGE +} + +namespace="default" +runtime="fibey-agentkit-foundry-responses" +task="fibey-foundry-responses-quincy-north-alert" +out_dir="" +logs_since="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) + [[ $# -ge 2 ]] || { echo "--namespace requires a value" >&2; exit 2; } + namespace="$2" + shift 2 + ;; + --runtime) + [[ $# -ge 2 ]] || { echo "--runtime requires a value" >&2; exit 2; } + runtime="$2" + shift 2 + ;; + --task) + [[ $# -ge 2 ]] || { echo "--task requires a value" >&2; exit 2; } + task="$2" + shift 2 + ;; + --out) + [[ $# -ge 2 ]] || { echo "--out requires a value" >&2; exit 2; } + out_dir="$2" + shift 2 + ;; + --logs-since) + [[ $# -ge 2 ]] || { echo "--logs-since requires a value" >&2; exit 2; } + logs_since="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { echo "error: $1 is required" >&2; exit 2; } +} + +require_cmd kubectl +require_cmd orka +require_cmd python3 + +if [[ -z "$out_dir" ]]; then + out_dir="foundry-responses-live-evidence-$(date -u +%Y%m%dT%H%M%SZ)" +fi + +umask 077 +mkdir -p "$out_dir" +runtime_tmp="" +events_tmp="" +approvals_tmp="" +pods_tmp="" +cleanup() { + [[ -z "${runtime_tmp:-}" ]] || rm -f "$runtime_tmp" + [[ -z "${events_tmp:-}" ]] || rm -f "$events_tmp" + [[ -z "${approvals_tmp:-}" ]] || rm -f "$approvals_tmp" + [[ -z "${pods_tmp:-}" ]] || rm -f "$pods_tmp" +} +trap cleanup EXIT + +report="$out_dir/README.md" +agentruntime_json="$out_dir/agentruntime.json" +events_json="$out_dir/task-events.json" +approvals_json="$out_dir/task-approvals.json" +verifier_out="$out_dir/fibey-verifier.txt" +log_scan="$out_dir/adapter-log-scan.txt" +artifact_forbidden_pattern="(api[-_]?key|authorization|bearer|secret|password|credential|txn-token|ORKA_FOUNDRY_RESPONSES_|https?://[^[:space:]\"<>]*(fibey-telemetry|fibey-incidents|fibey-dispatch|support-tool))" +log_forbidden_pattern="(api[-_]?key|authorization|bearer|secret|password|credential|txn-token|ORKA_FOUNDRY_RESPONSES_|https?://[^[:space:]\"<>]*(fibey-telemetry|fibey-incidents|fibey-dispatch|support-tool))" + +{ + echo "# Foundry Responses live evidence" + echo + echo "- Captured at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "- Namespace: ${namespace}" + echo "- AgentRuntime: ${runtime}" + echo "- Task: ${task}" + echo + echo "Raw adapter logs are not stored by this script." +} >"$report" + +scan_saved_artifact() { + local file="$1" + local label="$2" + if [[ ! -s "$file" ]]; then + return + fi + if grep -Eiq "$artifact_forbidden_pattern" "$file"; then + rm -f "$file" + echo "error: forbidden credential/tool-url pattern detected in ${label}; removed ${file}" >&2 + exit 1 + fi +} + +runtime_tmp="$(mktemp)" +kubectl -n "$namespace" get agentruntime "$runtime" -o json >"$runtime_tmp" +python3 - "$runtime_tmp" "$agentruntime_json" <<'PY' +import json +import sys +from pathlib import Path + +runtime = json.loads(Path(sys.argv[1]).read_text()) +status = runtime.get("status") or {} +metadata = runtime.get("metadata") or {} +metadata_generation = metadata.get("generation") +observed_generation = status.get("observedGeneration") +if metadata_generation is not None and observed_generation != metadata_generation: + raise SystemExit("AgentRuntime status.observedGeneration does not match metadata.generation") +conditions = status.get("conditions") or [] +ready = [c for c in conditions if c.get("type") == "Ready"] +if not ready or str(ready[-1].get("status", "")).lower() != "true": + raise SystemExit("AgentRuntime Ready=True was not observed") +condition_generation = ready[-1].get("observedGeneration") +if condition_generation is not None and metadata_generation is not None and condition_generation != metadata_generation: + raise SystemExit("AgentRuntime Ready condition observedGeneration does not match metadata.generation") +observed = status.get("observedCapabilities") or status.get("observedCapabilitiesRaw") or {} +if isinstance(observed, dict): + modes = observed.get("toolExecutionModes") or [] + classes = observed.get("brokeredToolClasses") or [] + continuation = observed.get("supportsContinuation") +else: + modes, classes, continuation = [], [], None +if "brokered" not in modes: + raise SystemExit("AgentRuntime observed capabilities do not include brokered mode") +if not {"read", "write"}.issubset(set(classes)): + raise SystemExit("AgentRuntime observed capabilities do not include both read and write brokered classes") +if continuation is not True: + raise SystemExit("AgentRuntime observed capabilities do not include supportsContinuation=true") +summary = { + "kind": runtime.get("kind", "AgentRuntime"), + "metadata": { + "name": (runtime.get("metadata") or {}).get("name"), + "namespace": (runtime.get("metadata") or {}).get("namespace"), + "generation": (runtime.get("metadata") or {}).get("generation"), + }, + "status": { + "ready": ready[-1], + "observedCapabilities": observed, + "observedGeneration": status.get("observedGeneration"), + }, +} +Path(sys.argv[2]).write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") +PY +rm -f "$runtime_tmp" +scan_saved_artifact "$agentruntime_json" "AgentRuntime summary" + +events_tmp="$(mktemp)" +approvals_tmp="$(mktemp)" +orka task events "$task" --namespace "$namespace" --output json >"$events_tmp" +orka task approvals "$task" --namespace "$namespace" --output json >"$approvals_tmp" +python3 - "$events_tmp" "$events_json" <<'PY' +import json +import sys +from pathlib import Path + +payload = json.loads(Path(sys.argv[1]).read_text()) +if isinstance(payload, dict): + if isinstance(payload.get("events"), list): + events = payload["events"] + elif isinstance(payload.get("items"), list): + events = payload["items"] + else: + events = [payload] +elif isinstance(payload, list): + events = payload +else: + events = [] + + +def content_dict(event): + content = event.get("content") if isinstance(event, dict) else None + if isinstance(content, dict): + return content + if isinstance(content, str): + try: + decoded = json.loads(content) + except Exception: # noqa: BLE001 - best-effort evidence summarizer + return {} + return decoded if isinstance(decoded, dict) else {} + return {} + + +def event_field(event, *names): + if not isinstance(event, dict): + return None + content = content_dict(event) + for name in names: + value = event.get(name) + if value not in (None, ""): + return value + value = content.get(name) + if value not in (None, ""): + return value + return None + + +def has_idempotency(value): + if isinstance(value, dict): + if any(k in value for k in ("idempotencyKey", "Idempotency-Key")): + return True + return any(has_idempotency(v) for v in value.values()) + if isinstance(value, list): + return any(has_idempotency(v) for v in value) + if isinstance(value, str): + try: + decoded = json.loads(value) + except Exception: # noqa: BLE001 + return False + return has_idempotency(decoded) + return False + +summary = [] +for index, event in enumerate(events, start=1): + event = event if isinstance(event, dict) else {} + summary.append({ + "index": index, + "type": event.get("type") or event.get("eventType"), + "toolName": event_field(event, "toolName", "tool", "name"), + "hasIdempotencyEvidence": has_idempotency(event), + "hasError": bool(event_field(event, "error", "errorCode")), + }) +Path(sys.argv[2]).write_text(json.dumps({"eventCount": len(events), "events": summary}, indent=2, sort_keys=True) + " +") +PY +scan_saved_artifact "$events_json" "task events summary" +python3 - "$approvals_tmp" "$approvals_json" <<'PY' +import json +import sys +from pathlib import Path + +payload = json.loads(Path(sys.argv[1]).read_text()) +if isinstance(payload, dict): + items = payload.get("approvals") or payload.get("items") or payload.get("events") or [] + if not isinstance(items, list): + items = [payload] +elif isinstance(payload, list): + items = payload +else: + items = [] +summary = [] +for index, item in enumerate(items, start=1): + item = item if isinstance(item, dict) else {} + summary.append({ + "index": index, + "type": item.get("type") or item.get("eventType"), + "status": item.get("status") or item.get("decision"), + "toolName": item.get("toolName") or item.get("tool"), + }) +Path(sys.argv[2]).write_text(json.dumps({"approvalCount": len(items), "approvals": summary}, indent=2, sort_keys=True) + " +") +PY +scan_saved_artifact "$approvals_json" "task approvals summary" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fibey_verifier="${script_dir}/../fibey-custom-agent-demo/verify-foundry-responses.sh" +if [[ ! -x "$fibey_verifier" ]]; then + fibey_verifier="${script_dir}/../../fibey-custom-agent-demo/verify-foundry-responses.sh" +fi +"$fibey_verifier" --json "$events_tmp" >"$verifier_out" +scan_saved_artifact "$verifier_out" "Fibey verifier output" + +pods_tmp="$(mktemp)" +kubectl -n "$namespace" get pods -l "app.kubernetes.io/name=${runtime}" -o json >"$pods_tmp" +python3 - "$pods_tmp" <<'PY' +import json +import sys +from pathlib import Path + +pods = json.loads(Path(sys.argv[1]).read_text()).get("items") or [] +if not pods: + raise SystemExit("no adapter pods found for log evidence") +restarted = [] +for pod in pods: + pod_name = (pod.get("metadata") or {}).get("name", "") + statuses = (pod.get("status") or {}).get("containerStatuses") or [] + for status in statuses: + if int(status.get("restartCount") or 0) > 0: + restarted.append(f"{pod_name}/{status.get('name', '')}") +if restarted: + raise SystemExit("adapter pod restarts observed; previous logs must be inspected before evidence can pass: " + ", ".join(restarted)) +PY +rm -f "$pods_tmp" + +log_err="$out_dir/adapter-log-error.txt" +pods="$(kubectl -n "$namespace" get pods \ + -l "app.kubernetes.io/name=${runtime}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>"$log_err")" +if [[ -z "$pods" ]]; then + { + echo "adapter log scan: FAILED" + echo "No adapter pods were found for app.kubernetes.io/name=${runtime}." + echo "kubectl error: $(tr '\n' ' ' <"$log_err")" + } >"$log_scan" + rm -f "$log_err" + cat "$log_scan" >&2 + exit 1 +fi +log_text="" +log_pods=0 +while IFS= read -r pod; do + [[ -n "$pod" ]] || continue + log_pods=$((log_pods + 1)) + log_args=(logs "pod/${pod}" --all-containers) + if [[ -n "$logs_since" ]]; then + log_args+=(--since "$logs_since") + fi + if ! pod_logs="$(kubectl -n "$namespace" "${log_args[@]}" 2>>"$log_err")"; then + { + echo "adapter log scan: FAILED" + echo "Could not retrieve adapter logs from pod/${pod}. Raw logs were not stored." + echo "kubectl error: $(tr '\n' ' ' <"$log_err")" + } >"$log_scan" + rm -f "$log_err" + cat "$log_scan" >&2 + exit 1 + fi + log_text+=$'\n'"${pod_logs}" +done <<<"$pods" +rm -f "$log_err" +if [[ -z "${log_text//[[:space:]]/}" ]]; then + { + echo "adapter log scan: FAILED" + echo "No adapter logs were returned from pods for deployment/${runtime}; evidence is indeterminate." + } >"$log_scan" + cat "$log_scan" >&2 + exit 1 +fi +if grep -Eiq "$log_forbidden_pattern" <<<"$log_text"; then + { + echo "adapter log scan: FAILED" + echo "A forbidden credential/tool-url pattern was detected in adapter logs. Raw logs were not stored." + } >"$log_scan" + cat "$log_scan" >&2 + exit 1 +fi +{ + echo "adapter log scan: passed" + echo "scanned pods: ${log_pods}" + echo "scanned tail lines: $(wc -l <<<"$log_text" | tr -d ' ')" +} >"$log_scan" + +{ + echo + echo "## Evidence files" + echo + echo "- AgentRuntime JSON: $(basename "$agentruntime_json")" + echo "- Task events JSON: $(basename "$events_json")" + echo "- Task approvals JSON: $(basename "$approvals_json")" + echo "- Fibey verifier output: $(basename "$verifier_out")" + echo "- Adapter log scan summary: $(basename "$log_scan")" + echo + echo "## Verifier output" + echo + sed 's/^/> /' "$verifier_out" + echo + echo "## Adapter log scan" + echo + sed 's/^/> /' "$log_scan" +} >>"$report" + +echo "Foundry Responses live evidence captured in: $out_dir" >&2 From 3eca9527db7b9ed98fa0ef1b1e3cb46eb9128de8 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 01:01:05 -0700 Subject: [PATCH 29/51] fix: repair Foundry Responses evidence summaries Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/live-evidence.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index b3ffdd6fe..8752bad58 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -253,8 +253,7 @@ for index, event in enumerate(events, start=1): "hasIdempotencyEvidence": has_idempotency(event), "hasError": bool(event_field(event, "error", "errorCode")), }) -Path(sys.argv[2]).write_text(json.dumps({"eventCount": len(events), "events": summary}, indent=2, sort_keys=True) + " -") +Path(sys.argv[2]).write_text(json.dumps({"eventCount": len(events), "events": summary}, indent=2, sort_keys=True) + "\n") PY scan_saved_artifact "$events_json" "task events summary" python3 - "$approvals_tmp" "$approvals_json" <<'PY' @@ -280,8 +279,7 @@ for index, item in enumerate(items, start=1): "status": item.get("status") or item.get("decision"), "toolName": item.get("toolName") or item.get("tool"), }) -Path(sys.argv[2]).write_text(json.dumps({"approvalCount": len(items), "approvals": summary}, indent=2, sort_keys=True) + " -") +Path(sys.argv[2]).write_text(json.dumps({"approvalCount": len(items), "approvals": summary}, indent=2, sort_keys=True) + "\n") PY scan_saved_artifact "$approvals_json" "task approvals summary" From 3bbc5d186c6944d709d4dbb92f0a51e9502af224 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 01:20:16 -0700 Subject: [PATCH 30/51] test: harden Foundry Responses live verification Signed-off-by: Sertac Ozercan --- ...ndry-responses-events-duplicate-write.json | 18 +++++++ ...y-responses-events-missing-write-exec.json | 8 ++++ .../foundry-responses-events-pass.json | 30 ++++++++++++ .../harness/foundry-responses/VALIDATION.md | 1 + .../harness/foundry-responses/live-smoke.sh | 47 ++++++++++++++++++- .../harness/foundry-responses/validate.sh | 37 +++++++++++++++ 6 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json new file mode 100644 index 000000000..3d90d2c10 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json @@ -0,0 +1,18 @@ +{ + "events": [ + {"eventType": "ToolCallRequested", "toolName": "check-network-telemetry"}, + {"eventType": "ToolCallRequested", "toolName": "dispatch-work-order"}, + {"eventType": "ApprovalRequested", "toolName": "dispatch-work-order"}, + { + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": {"idempotencyKey": "dispatch-1"} + }, + { + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": {"idempotencyKey": "dispatch-2"} + }, + {"eventType": "AgentRuntimeCompleted"} + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json new file mode 100644 index 000000000..7891a7e0e --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json @@ -0,0 +1,8 @@ +{ + "events": [ + {"eventType": "ToolCallRequested", "toolName": "check-network-telemetry"}, + {"eventType": "ToolCallRequested", "toolName": "dispatch-work-order"}, + {"eventType": "ApprovalRequested", "toolName": "dispatch-work-order"}, + {"eventType": "AgentRuntimeCompleted"} + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json new file mode 100644 index 000000000..3b248f21b --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json @@ -0,0 +1,30 @@ +{ + "events": [ + { + "id": "read-request", + "eventType": "ToolCallRequested", + "toolName": "check-network-telemetry", + "content": {"type": "business_payload"} + }, + { + "id": "write-request", + "eventType": "ToolCallRequested", + "toolName": "dispatch-work-order" + }, + { + "id": "approval", + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order" + }, + { + "id": "write-started", + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": {"idempotencyKey": "dispatch-1"} + }, + { + "id": "done", + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 001e5737a..8166f8608 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -77,6 +77,7 @@ uv run --extra dev pytest -q \ | Large hosted output and platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorDoesNotRetainTurn`. | | No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | | Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | +| Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`. | | Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | ## Live gates that cannot be satisfied by local fixtures diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index efa7bcf64..da41fe524 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -124,6 +124,49 @@ require_nonempty_name() { fail "$label must be a Kubernetes DNS label, got '$value'" } +url_authority() { + local value="$1" + local rest="${value#*://}" + rest="${rest%%#*}" + rest="${rest%%\?*}" + printf '%s' "${rest%%/*}" +} + +responses_endpoint_is_safe() { + local value="$1" + local rest authority after path query pair key val + [[ -z "$value" ]] && return 0 + is_https_or_loopback_http "$value" || return 1 + [[ "$value" != *#* ]] || return 1 + authority="$(url_authority "$value")" + [[ -n "$authority" && "$authority" != *@* ]] || return 1 + rest="${value#*://}" + after="${rest#"$authority"}" + path="${after%%\?*}" + [[ "${path%/}" == */responses ]] || return 1 + if [[ "$after" == *\?* ]]; then + query="${after#*\?}" + [[ -n "$query" ]] || return 1 + IFS='&' read -r -a pairs <<<"$query" + for pair in "${pairs[@]}"; do + [[ "$pair" == *=* ]] || return 1 + key="${pair%%=*}" + val="${pair#*=}" + [[ "$key" == "api-version" && -n "$val" ]] || return 1 + done + fi +} + +project_endpoint_is_safe() { + local value="$1" + local authority + [[ -z "$value" ]] && return 0 + is_https_or_loopback_http "$value" || return 1 + [[ "$value" != *#* && "$value" != *\?* ]] || return 1 + authority="$(url_authority "$value")" + [[ -n "$authority" && "$authority" != *@* ]] +} + indent_block() { sed 's/^/ /' } @@ -136,8 +179,8 @@ preflight() { [[ -n "$project_endpoint" && -n "$agent_name" ]] || \ fail "set ORKA_FOUNDRY_RESPONSES_ENDPOINT or PROJECT_ENDPOINT plus AGENT_NAME" fi - is_https_or_loopback_http "$endpoint" || fail "ORKA_FOUNDRY_RESPONSES_ENDPOINT must be https (or loopback http for tests)" - is_https_or_loopback_http "$project_endpoint" || fail "ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT must be https (or loopback http for tests)" + responses_endpoint_is_safe "$endpoint" || fail "ORKA_FOUNDRY_RESPONSES_ENDPOINT must be a safe /responses URL (https, or loopback http for tests; only api-version query allowed)" + project_endpoint_is_safe "$project_endpoint" || fail "ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT must be https (or loopback http for tests) without userinfo, query, or fragment" if [[ -n "$api_key" && -n "$auth_bearer" ]]; then fail "set exactly one of ORKA_FOUNDRY_RESPONSES_API_KEY or ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index aa3f20717..5875bb49c 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -63,6 +63,43 @@ while IFS= read -r -d '' script; do run bash -n "$script" done < <(find examples -type f -name '*.sh' -print0 | sort -z) +expect_verifier_failure() { + local fixture="$1" + local expected="$2" + local label="$3" + local out_file err_file code + out_file="$(mktemp)" + err_file="$(mktemp)" + set +e + examples/fibey-custom-agent-demo/verify-foundry-responses.sh --json "$fixture" >"$out_file" 2>"$err_file" + code=$? + set -e + if [[ "$code" == "0" ]]; then + cat "$out_file" >&2 + rm -f "$out_file" "$err_file" + echo "expected ${label} verifier fixture to fail" >&2 + exit 1 + fi + if ! grep -q "$expected" "$err_file"; then + cat "$err_file" >&2 + rm -f "$out_file" "$err_file" + echo "${label} fixture failed for the wrong reason" >&2 + exit 1 + fi + rm -f "$out_file" "$err_file" +} + +run examples/fibey-custom-agent-demo/verify-foundry-responses.sh \ + --json examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json \ + "missing write ToolCallStarted event after approval" \ + "missing-write" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json \ + "duplicate write execution starts for dispatch-work-order" \ + "duplicate-write" + if [[ "$run_full" == "1" ]]; then run make test fi From 87b670021ae03d032a58d1fed7115f5789e9554f Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 01:37:44 -0700 Subject: [PATCH 31/51] fix: update goldmark for vulnerability scan Signed-off-by: Sertac Ozercan --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d4eeedcf7..f9f6099b9 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( github.com/valyala/fastjson v1.6.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark v1.7.17 // indirect github.com/yuin/goldmark-emoji v1.0.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect diff --git a/go.sum b/go.sum index fdde1bef0..a37a79a16 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= -github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark v1.7.17 h1:p36OVWwRb246iHxA/U4p8OPEpOTESm4n+g+8t0EE5uA= +github.com/yuin/goldmark v1.7.17/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= From 170729a98e419326a2fc06ddd330d8706c9a3d6a Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 14:15:03 -0700 Subject: [PATCH 32/51] fix: harden Foundry Responses request handling Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 2 +- examples/harness/foundry-responses/main.go | 97 +++++++++--- .../harness/foundry-responses/main_test.go | 148 +++++++++++++----- 3 files changed, 187 insertions(+), 60 deletions(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index e9dda8ad1..3ea46d422 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -28,7 +28,7 @@ Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `exa | `ORKA_FOUNDRY_RESPONSES_API_KEY` | Static API-key auth mode. Tests/demo only unless your deployment standard permits it. | | `ORKA_FOUNDRY_RESPONSES_AUTH_BEARER` | Static bearer auth mode. Tests/demo only unless supplied by a production token refresher sidecar. | | `ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE` | Reserved for future workload-identity token refresh support; currently not used. | -| `ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF` | Optional Orka-only proof value sent as `X-AgentKit-Brokered-Continuation-Proof` on hosted Responses continuations. Set it to match AgentKit's `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` when that guard is enabled. | +| `ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF` | Optional Orka-only proof value sent on hosted Responses continuations in both the `X-AgentKit-Brokered-Continuation-Proof` header and `brokered_continuation_proof` request-body field, so gateways that strip custom headers can still forward it. Set it to match AgentKit's `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` when that guard is enabled. | | `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` | Comma-separated static classes the hosted AgentKit deployment has been configured and conformance-tested to request, e.g. `read` or `read,write`. Empty means observed-only. | | `ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT` | Per-request timeout for hosted Responses calls, default `20s`. | | `ORKA_FOUNDRY_RESPONSES_STATE_RETENTION` | How long terminal in-memory turn/session state is retained, default `10m`. | diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 33bc59d0f..679668d7d 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -24,14 +24,17 @@ import ( ) const ( - defaultAddr = ":8090" - defaultAPIVersion = "v1" - defaultRequestTimeout = 20 * time.Second - defaultStateRetention = 10 * time.Minute - defaultMaxApprovalWait = 30 * time.Minute - maxFoundryOutputBytes = 1 << 20 - maxFoundryBodyBytes = 4 << 20 - readinessPath = "/v1/ready" + defaultAddr = ":8090" + defaultAPIVersion = "v1" + defaultRequestTimeout = 20 * time.Second + defaultStateRetention = 10 * time.Minute + defaultMaxApprovalWait = 30 * time.Minute + defaultReadHeaderTimeout = 5 * time.Second + defaultReadTimeout = 30 * time.Second + defaultIdleTimeout = 60 * time.Second + maxFoundryOutputBytes = 1 << 20 + maxFoundryBodyBytes = 4 << 20 + readinessPath = "/v1/ready" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -94,6 +97,7 @@ type turnState struct { submittedPayloads map[string]string frames []harness.HarnessEventFrame completed bool + frameUpdates chan struct{} continueMu sync.Mutex } @@ -101,6 +105,14 @@ type responsesRequest struct { Input any `json:"input"` PreviousResponseID string `json:"previous_response_id,omitempty"` AgentSessionID string `json:"agent_session_id,omitempty"` + // BrokeredContinuationProof carries the continuation proof in the request + // BODY in addition to the X-AgentKit-Brokered-Continuation-Proof header. + // Some hosted-agent gateways (e.g. Microsoft Foundry) strip custom request + // headers before forwarding to the container, which would reject the + // function_call_output continuation; the body survives, so a + // gateway-tolerant runtime can recover the proof from here. Only set on + // continuation requests (PreviousResponseID present). + BrokeredContinuationProof string `json:"brokered_continuation_proof,omitempty"` } type responsesFunctionCallOutput struct { @@ -153,11 +165,21 @@ func main() { cfg.runtimeName, sanitizeEndpoint(cfg.endpoint), ) - if err := http.ListenAndServe(cfg.addr, s.handler()); err != nil { + if err := newAdapterHTTPServer(cfg.addr, s.handler()).ListenAndServe(); err != nil { log.Fatal(err) } } +func newAdapterHTTPServer(addr string, handler http.Handler) *http.Server { + return &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: defaultReadHeaderTimeout, + ReadTimeout: defaultReadTimeout, + IdleTimeout: defaultIdleTimeout, + } +} + func loadConfig() config { classes, classErr := parseBrokeredToolClasses(os.Getenv(envBrokeredToolClasses)) cfg := config{ @@ -339,12 +361,13 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, + frameUpdates: make(chan struct{}), } s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) s.turns[req.TurnID] = turn s.mu.Unlock() - ctx, cancel := context.WithTimeout(r.Context(), s.cfg.requestTimeout) + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.cfg.requestTimeout) defer cancel() var response responsesResponse initialRequest := responsesRequest{Input: req.Input.Prompt} @@ -404,17 +427,39 @@ func (s *server) turn(w http.ResponseWriter, r *http.Request) { func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turnState) { afterSeq := parseAfterSeq(r.URL.Query().Get("afterSeq")) w.Header().Set("Content-Type", "text/event-stream") - s.mu.Lock() - frames := append([]harness.HarnessEventFrame(nil), turn.frames...) - completed := turn.completed - s.mu.Unlock() - for _, frame := range frames { - if frame.Seq > afterSeq { - _ = harness.WriteSSEFrame(w, frame) - } + w.WriteHeader(http.StatusOK) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() } - if completed { - _ = harness.WriteSSEDone(w) + nextSeq := afterSeq + for { + s.mu.Lock() + frames := append([]harness.HarnessEventFrame(nil), turn.frames...) + completed := turn.completed + updates := turn.frameUpdates + if updates == nil { + updates = make(chan struct{}) + turn.frameUpdates = updates + } + s.mu.Unlock() + for _, frame := range frames { + if frame.Seq <= nextSeq { + continue + } + if err := harness.WriteSSEFrame(w, frame); err != nil { + return + } + nextSeq = frame.Seq + } + if completed { + _ = harness.WriteSSEDone(w) + return + } + select { + case <-r.Context().Done(): + return + case <-updates: + } } } @@ -473,7 +518,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteError(w, http.StatusBadRequest, err.Error()) return } - ctx, cancel := context.WithTimeout(r.Context(), s.cfg.requestTimeout) + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.cfg.requestTimeout) defer cancel() var response responsesResponse continuation := responsesRequest{ @@ -915,6 +960,12 @@ func (s *server) postResponses( if body.AgentSessionID == "" && sessionID != "" { body.AgentSessionID = sessionID } + // Carry the continuation proof in the body as well as the header, so it + // survives hosted-agent gateways (e.g. Foundry) that strip custom request + // headers before forwarding to the runtime container. Only on continuations. + if body.PreviousResponseID != "" && s.cfg.continuationProof != "" { + body.BrokeredContinuationProof = s.cfg.continuationProof + } payload, err := json.Marshal(body) if err != nil { return err @@ -1300,6 +1351,10 @@ func (s *server) appendFrameLocked( mutate(&frame) } turn.frames = append(turn.frames, frame) + if turn.frameUpdates != nil { + close(turn.frameUpdates) + } + turn.frameUpdates = make(chan struct{}) } func (s *server) authorized(w http.ResponseWriter, r *http.Request) bool { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 9390092a3..0ec4bcded 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -102,11 +102,19 @@ func TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures(t *testing.T) assertJSONFileEqual(t, "testdata/golden/01_initial_hosted_request.json", foundry.requestBody(0)) var frames []harness.HarnessEventFrame - if err := client.StreamFrames(context.Background(), request.TurnID, 0, func(frame harness.HarnessEventFrame) error { + var continueRequest harness.ContinueTurnRequest + streamCtx, streamCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer streamCancel() + if err := client.StreamFrames(streamCtx, request.TurnID, 0, func(frame harness.HarnessEventFrame) error { frames = append(frames, frame) - return nil + if frame.Type != harness.FrameToolCallRequested { + return nil + } + continueRequest = goldenContinueRequest(request, frame.ToolCallID, json.RawMessage(`{"success":true}`)) + _, err := client.ContinueTurn(context.Background(), continueRequest) + return err }); err != nil { - t.Fatalf("StreamFrames before continue: %v", err) + t.Fatalf("StreamFrames through continue: %v", err) } requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { @@ -117,28 +125,11 @@ func TestResponsesAdapterBrokeredReadContinuationAndGoldenFixtures(t *testing.T) } assertJSONFileEqual(t, "testdata/golden/03_tool_call_requested_frame.json", scrubFrameForGolden(*requested)) - continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) assertJSONFileEqual(t, "testdata/golden/04_orka_continue_request.json", continueRequest) - if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { - t.Fatalf("ContinueTurn: %v", err) - } assertJSONFileEqual(t, "testdata/golden/05_hosted_continuation_request.json", foundry.requestBody(1)) if got := foundry.requestHeader(1).Get("x-agent-session-id"); got != fakeSessionID { t.Fatalf("continuation x-agent-session-id = %q, want session-1", got) } - - frames = nil - if err := client.StreamFrames( - context.Background(), - request.TurnID, - requested.Seq, - func(frame harness.HarnessEventFrame) error { - frames = append(frames, frame) - return nil - }, - ); err != nil { - t.Fatalf("StreamFrames after continue: %v", err) - } if !hasFrameType(frames, harness.FrameToolResultReceived) || !hasFrameType(frames, harness.FrameTurnCompleted) { t.Fatalf("frames = %#v, want tool result and completion", frames) } @@ -190,7 +181,7 @@ func TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup(t *te if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -247,6 +238,65 @@ func TestResponsesAdapterDuplicateStartDuringInitializationRejected(t *testing.T } } +func TestResponsesAdapterInitialPostSurvivesControlDisconnect(t *testing.T) { + received := make(chan struct{}) + release := make(chan struct{}) + var postCount atomic.Int32 + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + postCount.Add(1) + select { + case <-received: + default: + close(received) + } + select { + case <-release: + writeJSON(w, finalResponsesMessage()) + case <-r.Context().Done(): + return + } + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter, adapterServer := newTestResponsesAdapterWithServer(t, endpoint, nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-control-disconnect") + + ctx, cancel := context.WithCancel(context.Background()) + startErr := make(chan error, 1) + go func() { + _, err := client.StartTurn(ctx, request) + startErr <- err + }() + <-received + cancel() + if err := <-startErr; err == nil { + t.Fatal("StartTurn after control disconnect error = nil, want client cancellation") + } + close(release) + + deadline := time.Now().Add(time.Second) + for { + adapterServer.mu.Lock() + turn := adapterServer.turns[request.TurnID] + completed := turn != nil && turn.completed && !turn.initializing + adapterServer.mu.Unlock() + if completed { + break + } + if time.Now().After(deadline) { + t.Fatal("adapter did not retain and complete the initial hosted response after control disconnect") + } + time.Sleep(10 * time.Millisecond) + } + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("idempotent StartTurn retry: %v", err) + } + if got := postCount.Load(); got != 1 { + t.Fatalf("hosted post count = %d, want 1 after control disconnect and retry", got) + } +} + func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { unready := httptest.NewServer(newServer(config{ runtimeName: "foundry-responses-test", @@ -276,6 +326,19 @@ func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { } } +func TestResponsesAdapterHTTPServerHasBoundedReadTimeouts(t *testing.T) { + httpServer := newAdapterHTTPServer(":0", http.NewServeMux()) + if got := httpServer.ReadHeaderTimeout; got != defaultReadHeaderTimeout { + t.Fatalf("ReadHeaderTimeout = %v, want %v", got, defaultReadHeaderTimeout) + } + if got := httpServer.ReadTimeout; got != defaultReadTimeout { + t.Fatalf("ReadTimeout = %v, want %v", got, defaultReadTimeout) + } + if got := httpServer.IdleTimeout; got != defaultIdleTimeout { + t.Fatalf("IdleTimeout = %v, want %v", got, defaultIdleTimeout) + } +} + func TestResponsesAdapterPassesObservedConformanceByDefault(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) adapter := newTestResponsesAdapter(t, foundry.endpoint(), nil) @@ -364,7 +427,7 @@ func TestResponsesAdapterWriteParksUntilDeclinedApprovalContinue(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want write tool request", frames) @@ -411,7 +474,7 @@ func TestResponsesAdapterWriteParksUntilDeclinedApprovalContinue(t *testing.T) { if got := item["output"]; got != wantOutput { t.Fatalf("declined output = %#v, want %s", got, wantOutput) } - frames = streamAllFrames(t, client, request.TurnID) + frames = streamCurrentFrames(t, client, request.TurnID) toolResult := findFrame(frames, harness.FrameToolResultReceived) if toolResult == nil || toolResult.Error == nil || toolResult.Error.Code != "approval_declined" { t.Fatalf("tool result frame = %#v, want approval_declined", toolResult) @@ -434,7 +497,7 @@ func TestResponsesAdapterRejectsUnknownToolBeforeOrkaExecution(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) if hasFrameType(frames, harness.FrameToolCallRequested) { t.Fatalf("frames = %#v, should not request Orka execution for an unknown tool", frames) } @@ -460,7 +523,7 @@ func TestResponsesAdapterRejectsMalformedArguments(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) if hasFrameType(frames, harness.FrameToolCallRequested) { t.Fatalf("frames = %#v, should not request Orka execution for malformed arguments", frames) } @@ -483,7 +546,7 @@ func TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued(t *testing.T) if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requests := findFrames(frames, harness.FrameToolCallRequested) if len(requests) != 2 { t.Fatalf("tool request frames = %#v, want 2", requests) @@ -512,7 +575,7 @@ func TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued(t *testing.T) if got := continuation["agent_session_id"]; got != fakeSessionID { t.Fatalf("agent_session_id = %#v, want %q", got, fakeSessionID) } - frames = streamAllFrames(t, client, request.TurnID) + frames = streamCurrentFrames(t, client, request.TurnID) if !hasFrameType(frames, harness.FrameToolResultReceived) || !hasFrameType(frames, harness.FrameTurnCompleted) { t.Fatalf("frames = %#v, want tool results and completion", frames) } @@ -531,7 +594,7 @@ func TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject(t *test if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -552,7 +615,7 @@ func TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject(t *test } } -func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { +func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{ scenario: "function_call", toolName: "support-ticket-lookup", @@ -577,7 +640,7 @@ func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -589,9 +652,15 @@ func TestResponsesAdapterSendsBrokeredContinuationProofHeader(t *testing.T) { if got := foundry.requestHeader(1).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "proof-for-test" { t.Fatalf("continuation proof header = %q, want proof-for-test", got) } + if got := requestMap(t, foundry.requestBody(1))["brokered_continuation_proof"]; got != "proof-for-test" { + t.Fatalf("continuation proof body = %#v, want proof-for-test", got) + } if got := foundry.requestHeader(0).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "" { t.Fatalf("initial proof header = %q, want empty", got) } + if _, ok := requestMap(t, foundry.requestBody(0))["brokered_continuation_proof"]; ok { + t.Fatal("initial request included brokered continuation proof body") + } } func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing.T) { @@ -671,7 +740,7 @@ func TestResponsesAdapterContinuesToolExecutionFailurePayload(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -708,7 +777,7 @@ func TestResponsesAdapterContinuesToolExecutionFailurePayload(t *testing.T) { if got := item["output"]; got != wantOutput { t.Fatalf("failure output = %#v, want %s", got, wantOutput) } - frames = streamAllFrames(t, client, request.TurnID) + frames = streamCurrentFrames(t, client, request.TurnID) toolResult := findFrame(frames, harness.FrameToolResultReceived) if toolResult == nil || toolResult.Error == nil || toolResult.Error.Code != "tool_execution_failed" { t.Fatalf("tool result frame = %#v, want tool_execution_failed", toolResult) @@ -735,7 +804,7 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -753,7 +822,7 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t if foundry.postCount.Load() != 2 { t.Fatalf("hosted post count after duplicate = %d, want no second continuation", foundry.postCount.Load()) } - frames = streamAllFrames(t, client, request.TurnID) + frames = streamCurrentFrames(t, client, request.TurnID) failed := findFrame(frames, harness.FrameTurnFailed) if failed == nil || failed.Failed.Reason != "foundry_continuation_unknown" { t.Fatalf("failed frame = %#v, want fail-closed continuation failure", failed) @@ -979,7 +1048,7 @@ func TestResponsesAdapterStateLossContinueFailsSafely(t *testing.T) { if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamAllFrames(t, client, request.TurnID) + frames := streamCurrentFrames(t, client, request.TurnID) requested := findFrame(frames, harness.FrameToolCallRequested) if requested == nil { t.Fatalf("frames = %#v, want tool request", frames) @@ -1879,17 +1948,20 @@ func baseToolResult( } } -func streamAllFrames( +func streamCurrentFrames( t *testing.T, client *harness.Client, turnID harness.HarnessTurnID, ) []harness.HarnessEventFrame { t.Helper() var frames []harness.HarnessEventFrame - if err := client.StreamFrames(context.Background(), turnID, 0, func(frame harness.HarnessEventFrame) error { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + err := client.StreamFrames(ctx, turnID, 0, func(frame harness.HarnessEventFrame) error { frames = append(frames, frame) return nil - }); err != nil { + }) + if err != nil && ctx.Err() == nil { t.Fatalf("StreamFrames: %v", err) } return frames From dbe0757d99a18319580079b243d0761601e53ba8 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 16:12:00 -0700 Subject: [PATCH 33/51] fix: address Foundry Responses review findings Signed-off-by: Sertac Ozercan --- .../core_v1alpha1_agentruntime_foundry.yaml | 8 +- .../agentruntime-foundry.yaml | 7 +- ...ponses-events-decision-before-request.json | 62 ++++ ...undry-responses-events-declined-write.json | 62 ++++ ...ndry-responses-events-duplicate-write.json | 68 ++++- ...onses-events-mismatched-write-request.json | 62 ++++ ...nses-events-missing-approval-decision.json | 53 ++++ ...-responses-events-missing-approval-id.json | 55 ++++ ...y-responses-events-missing-write-exec.json | 50 +++- ...onses-events-overlapping-write-marker.json | 52 ++++ ...-responses-events-partial-idempotency.json | 86 ++++++ .../foundry-responses-events-pass.json | 55 +++- .../verify-foundry-responses.sh | 130 +++++++-- examples/harness/foundry-responses/README.md | 4 +- .../harness/foundry-responses/VALIDATION.md | 4 +- .../foundry-responses/kubernetes.example.yaml | 4 +- .../foundry-responses/live-evidence.sh | 8 +- .../harness/foundry-responses/live-smoke.sh | 2 + examples/harness/foundry-responses/main.go | 187 +++++++++--- .../harness/foundry-responses/main_test.go | 271 ++++++++++++++++++ .../harness/foundry-responses/validate.sh | 28 ++ internal/harness/client.go | 5 +- .../guides/bring-your-own-agent-runtime.md | 2 + 23 files changed, 1172 insertions(+), 93 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json diff --git a/config/samples/core_v1alpha1_agentruntime_foundry.yaml b/config/samples/core_v1alpha1_agentruntime_foundry.yaml index e4bb0c12e..fdd8cd181 100644 --- a/config/samples/core_v1alpha1_agentruntime_foundry.yaml +++ b/config/samples/core_v1alpha1_agentruntime_foundry.yaml @@ -17,11 +17,11 @@ spec: clientAuth: bearerTokenSecretRef: name: sample-foundry-responses-runtime-token - key: token + key: harness-bearer capabilities: - # Mirror only the classes the hosted AgentKit deployment is statically - # configured and conformance-tested to request. Add write only after the - # hosted AgentKit write schema and Orka brokered-write conformance pass. + # Mirror only classes whose hosted AgentKit deployment statically includes + # the matching probe-only conformance_read/conformance_write schema and has + # passed live AgentRuntime conformance. Add write only after that gate. toolExecutionModes: - observed - brokered diff --git a/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml b/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml index ddb549dd4..1b42bb41c 100644 --- a/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml +++ b/examples/fibey-custom-agent-demo/agentruntime-foundry.yaml @@ -16,9 +16,10 @@ spec: name: fibey-agentkit-foundry-responses-token key: token capabilities: - # This sample assumes the hosted AgentKit deployment has static read/write - # schemas and has passed the adapter's fake-server read/write conformance. - # Narrow this list if the deployment only supports read. + # This sample assumes the hosted AgentKit deployment has static Fibey tool + # schemas plus probe-only conformance_read/conformance_write schemas, and + # has passed live AgentRuntime read/write conformance. Fake-server adapter + # tests alone are not sufficient. Narrow this list until each gate passes. toolExecutionModes: - observed - brokered diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json new file mode 100644 index 000000000..4e2309522 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json @@ -0,0 +1,62 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 3, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 4, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json new file mode 100644 index 000000000..9cfc14fad --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json @@ -0,0 +1,62 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalDeclined", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "decline" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json index 3d90d2c10..319af3c23 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json @@ -1,18 +1,74 @@ { "events": [ - {"eventType": "ToolCallRequested", "toolName": "check-network-telemetry"}, - {"eventType": "ToolCallRequested", "toolName": "dispatch-work-order"}, - {"eventType": "ApprovalRequested", "toolName": "dispatch-work-order"}, { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", - "content": {"idempotencyKey": "dispatch-1"} + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" }, { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", - "content": {"idempotencyKey": "dispatch-2"} + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" }, - {"eventType": "AgentRuntimeCompleted"} + { + "seq": 7, + "eventType": "AgentRuntimeCompleted" + } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json new file mode 100644 index 000000000..c248385b4 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json @@ -0,0 +1,62 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + } + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "toolCallID": "dispatch-call-1", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + } + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "escalate-incident", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "escalate-incident", + "toolCallID": "escalate-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "escalate-incident", + "toolCallID": "escalate-call-1", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + } + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json new file mode 100644 index 000000000..6eb246fd8 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json @@ -0,0 +1,53 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 5, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json new file mode 100644 index 000000000..9ee50427c --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json @@ -0,0 +1,55 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "shared-id" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "shared-id", + "content": { + "toolCallID": "shared-id" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "shared-id" + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "toolCallID": "shared-id", + "content": { + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "shared-id" + } + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json index 7891a7e0e..e6e710b44 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json @@ -1,8 +1,50 @@ { "events": [ - {"eventType": "ToolCallRequested", "toolName": "check-network-telemetry"}, - {"eventType": "ToolCallRequested", "toolName": "dispatch-work-order"}, - {"eventType": "ApprovalRequested", "toolName": "dispatch-work-order"}, - {"eventType": "AgentRuntimeCompleted"} + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "AgentRuntimeCompleted" + } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json new file mode 100644 index 000000000..65ae7d334 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json @@ -0,0 +1,52 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 3, + "eventType": "ApprovalApproved", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 4, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + }, + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 5, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json new file mode 100644 index 000000000..243e339de --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json @@ -0,0 +1,86 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": {"harness": {"frameType": "ToolCallRequested"}} + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "toolCallID": "dispatch-call-1", + "content": {"harness": {"frameType": "ToolCallRequested"}} + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-dispatch", + "content": { + "approvalID": "approval-dispatch", + "targetTool": "dispatch-work-order", + "toolCallID": "dispatch-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-dispatch", + "content": {"approvalID": "approval-dispatch", "decision": "approve"} + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "toolCallID": "dispatch-call-1", + "content": { + "approvalID": "approval-dispatch", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-dispatch" + } + }, + { + "seq": 6, + "eventType": "ToolCallStarted", + "toolName": "escalate-incident", + "toolCallID": "escalate-call-1", + "content": {"harness": {"frameType": "ToolCallRequested"}} + }, + { + "seq": 7, + "eventType": "ApprovalRequested", + "toolName": "escalate-incident", + "toolCallID": "approval-escalate", + "content": { + "approvalID": "approval-escalate", + "targetTool": "escalate-incident", + "toolCallID": "escalate-call-1" + } + }, + { + "seq": 8, + "eventType": "ApprovalApproved", + "toolCallID": "approval-escalate", + "content": {"approvalID": "approval-escalate", "decision": "approve"} + }, + { + "seq": 9, + "eventType": "ToolCallStarted", + "toolName": "escalate-incident", + "toolCallID": "escalate-call-1", + "content": { + "approvalID": "approval-escalate", + "brokeredClass": "write", + "executionState": "started" + } + }, + { + "seq": 10, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json index 3b248f21b..e403af9ed 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json @@ -1,29 +1,62 @@ { "events": [ { - "id": "read-request", - "eventType": "ToolCallRequested", + "seq": 1, + "eventType": "ToolCallStarted", "toolName": "check-network-telemetry", - "content": {"type": "business_payload"} + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" }, { - "id": "write-request", - "eventType": "ToolCallRequested", - "toolName": "dispatch-work-order" + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" }, { - "id": "approval", + "seq": 3, "eventType": "ApprovalRequested", - "toolName": "dispatch-work-order" + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } }, { - "id": "write-started", + "seq": 5, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", - "content": {"idempotencyKey": "dispatch-1"} + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" }, { - "id": "done", + "seq": 6, "eventType": "AgentRuntimeCompleted" } ] diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 242a43ca5..5b61f5da4 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -12,8 +12,8 @@ Orka task events. By default it calls `orka task events --output json`. Use Expected evidence: - read brokered tool request for check-network-telemetry or get-active-incidents - write brokered tool request for dispatch-work-order or escalate-incident - - ApprovalRequested is present before write ToolCallStarted - - an idempotency key is present in write ToolCallStarted content + - matching ApprovalRequested and ApprovalApproved events precede write execution + - an idempotency key is present in the write execution ledger event - terminal TaskSucceeded/AgentRuntimeCompleted/TurnCompleted-style event exists This verifier does not approve tasks and never reads Foundry credentials. @@ -103,10 +103,6 @@ TERMINAL_TYPES = { "TurnCompleted", "TaskCompleted", } -READ_REQUEST_TYPES = {"ToolCallRequested", "ToolCallStarted"} -WRITE_EXEC_TYPES = {"ToolCallStarted"} - - def field(event, name): if not isinstance(event, dict): return None @@ -125,6 +121,22 @@ def field(event, name): return None +def content_field(event, name): + if not isinstance(event, dict): + return None + content = event.get("content") + if isinstance(content, dict): + return content.get(name) + if isinstance(content, str): + try: + decoded = json.loads(content) + except Exception: # noqa: BLE001 + return None + if isinstance(decoded, dict): + return decoded.get(name) + return None + + def event_type(event): if not isinstance(event, dict): return "" @@ -176,8 +188,39 @@ def idempotency_value(value): return "" -def contains_idempotency(event): - return bool(idempotency_value(event)) +def approval_id(event): + for key in ("approvalID", "approvalId"): + value = field(event, key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def tool_call_id(event): + for key in ("toolCallID", "toolCallId"): + value = field(event, key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def is_write_execution_start(event): + if event_type(event) != "ToolCallStarted": + return False + if is_harness_tool_request(event): + return False + return field(event, "executionState") == "started" and field(event, "brokeredClass") == "write" + + +def is_harness_tool_request(event): + if event_type(event) == "ToolCallRequested": + return True + harness_identity = field(event, "harness") + return ( + event_type(event) == "ToolCallStarted" + and isinstance(harness_identity, dict) + and harness_identity.get("frameType") == "ToolCallRequested" + ) ordered_events = [] @@ -190,33 +233,75 @@ for index, event in enumerate(events, start=1): ordered_events.append(event) events = ordered_events -read_events = [e for e in events if tool_name(e) in READ_TOOLS and event_type(e) in READ_REQUEST_TYPES] +read_events = [e for e in events if tool_name(e) in READ_TOOLS and is_harness_tool_request(e)] write_events = [e for e in events if tool_name(e) in WRITE_TOOLS] -approval_events = [e for e in events if event_type(e) == "ApprovalRequested"] -write_exec_events = [e for e in write_events if event_type(e) in WRITE_EXEC_TYPES] -write_start_events = [e for e in write_events if event_type(e) == "ToolCallStarted"] +write_request_events = [e for e in write_events if is_harness_tool_request(e)] +approval_request_events = [e for e in events if event_type(e) == "ApprovalRequested"] +approval_approved_events = [e for e in events if event_type(e) == "ApprovalApproved"] +approval_declined_events = [e for e in events if event_type(e) == "ApprovalDeclined"] +write_exec_events = [e for e in write_events if is_write_execution_start(e)] +write_start_events = write_exec_events terminal_events = [e for e in events if event_type(e) in TERMINAL_TYPES] idempotency_events = [e for e in write_exec_events if idempotency_value(e)] failures = [] if not read_events: failures.append("missing read brokered tool event for check-network-telemetry/get-active-incidents") -if not write_events: +if not write_request_events: failures.append("missing write brokered tool event for dispatch-work-order/escalate-incident") -if not approval_events: +if not approval_request_events: failures.append("missing ApprovalRequested event") +if not approval_approved_events: + failures.append("missing ApprovalApproved event") if not write_exec_events: failures.append("missing write ToolCallStarted event after approval") -if write_exec_events and approval_events: +if write_exec_events: for event in write_exec_events: write_tool = tool_name(event) write_order = seq(event) - matching_approvals = [ - approval for approval in approval_events - if tool_name(approval) == write_tool and seq(approval) < write_order + if not idempotency_value(event): + failures.append(f"write execution for {write_tool} is missing idempotency key evidence") + write_tool_call_id = tool_call_id(event) + if not write_tool_call_id: + failures.append(f"write execution for {write_tool} is missing toolCallID") + continue + matching_write_requests = [ + request for request in write_request_events + if tool_name(request) == write_tool + and tool_call_id(request) == write_tool_call_id + and seq(request) < write_order + ] + if not matching_write_requests: + failures.append(f"write execution for {write_tool} has no matching preceding mapped request") + continue + write_approval_id = approval_id(event) + if not write_approval_id: + failures.append(f"write execution for {write_tool} is missing approvalID") + continue + matching_approval_requests = [ + approval for approval in approval_request_events + if approval_id(approval) == write_approval_id + and tool_name(approval) == write_tool + and content_field(approval, "toolCallID") == write_tool_call_id + and seq(approval) < write_order + and any(seq(request) < seq(approval) for request in matching_write_requests) + ] + if not matching_approval_requests: + failures.append(f"write execution for {write_tool} has no matching preceding ApprovalRequested") + matching_approved = [ + approval for approval in approval_approved_events + if approval_id(approval) == write_approval_id + and seq(approval) < write_order + and any(seq(request) < seq(approval) for request in matching_approval_requests) + ] + if not matching_approved: + failures.append(f"write execution for {write_tool} has no matching preceding ApprovalApproved") + matching_declined = [ + approval for approval in approval_declined_events + if approval_id(approval) == write_approval_id and seq(approval) < write_order ] - if not matching_approvals: - failures.append(f"write execution for {write_tool} has no preceding approval") + if matching_declined: + failures.append(f"write execution for {write_tool} follows ApprovalDeclined") if not idempotency_events: failures.append("missing write ToolCallStarted idempotency key evidence") @@ -254,8 +339,9 @@ if failures: print("Fibey Foundry Responses verification passed:") print(f"- read events: {len(read_events)}") -print(f"- write events: {len(write_events)}") -print(f"- approvals: {len(approval_events)}") +print(f"- write requests: {len(write_request_events)}") +print(f"- approval requests: {len(approval_request_events)}") +print(f"- approval decisions: {len(approval_approved_events)}") print(f"- idempotency evidence events: {len(idempotency_events)}") print(f"- terminal events: {len(terminal_events)}") PY diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 3ea46d422..80a9d6437 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -55,6 +55,8 @@ Capabilities must reflect the **static schemas actually deployed in AgentKit**: - If it is `read`, the adapter advertises brokered read only. - Advertise `write` only after the hosted AgentKit deployment has a static write schema and passes write conformance. Orka will still gate the write with approval/idempotency, but the hosted model must not be told it can request writes unless that path is intentionally enabled. +AgentRuntime readiness deliberately deep-probes every advertised brokered class. Because this adapter never sends request-level `tools`, the hosted AgentKit deployment must statically expose the probe-only `conformance_read` and/or `conformance_write` schemas in addition to its real tools. These schemas take an empty object and must be safe to call: Orka's conformance client supplies the synthetic result and no production tool credential is sent to Foundry. Fake-server tests alone are not sufficient; if the hosted deployment cannot request the matching probe tool, leave that brokered class unadvertised and readiness will fail closed. + ## Protocol mapping Initial turn: @@ -110,7 +112,7 @@ docker build -t ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:lates ## Kubernetes smoke skeleton -`kubernetes.example.yaml` contains a credentials-free Deployment, Service, Secret placeholders, and matching `AgentRuntime` facade for a read-profile hosted Responses smoke. Replace the `REDACTED` values through your secret-management flow, set the hosted Responses endpoint or project/agent-name pair, and keep `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` narrowed to classes whose static AgentKit schemas passed conformance. +`kubernetes.example.yaml` contains a credentials-free Deployment, Service, Secret placeholders, and matching `AgentRuntime` facade for a read-profile hosted Responses smoke. Replace the `REDACTED` values through your secret-management flow, set the hosted Responses endpoint or project/agent-name pair, ensure the hosted agent statically exposes the probe-only `conformance_read` schema, and keep `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` narrowed to classes whose live AgentRuntime conformance passed. ```bash kubectl apply -f examples/harness/foundry-responses/kubernetes.example.yaml diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 8166f8608..4e689b49f 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -99,7 +99,9 @@ The following remain required before declaring the full hosted Foundry/Fibey pla complete: 1. Deploy an AgentKit prototype as a real Foundry hosted agent with static safe - brokered schemas and a configured brokered continuation proof. + brokered schemas, the probe-only `conformance_read`/`conformance_write` + schemas for every advertised class, and a configured brokered continuation + proof. 2. Deploy the Orka `foundry-responses` adapter with real Foundry auth and a real hosted `/responses` endpoint. 3. Verify `AgentRuntime` readiness for the read profile. diff --git a/examples/harness/foundry-responses/kubernetes.example.yaml b/examples/harness/foundry-responses/kubernetes.example.yaml index 70f095ee7..5b59ffeb8 100644 --- a/examples/harness/foundry-responses/kubernetes.example.yaml +++ b/examples/harness/foundry-responses/kubernetes.example.yaml @@ -68,7 +68,9 @@ spec: name: sample-foundry-responses-adapter-config key: continuation-proof optional: true - # Advertise only classes whose static AgentKit brokered schemas passed conformance. + # The hosted agent must statically expose probe-only conformance_read + # before read can pass AgentRuntime readiness. Advertise only classes + # whose live brokered conformance passed. - name: ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES value: read ports: diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index 8752bad58..c537f33df 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -275,9 +275,11 @@ for index, item in enumerate(items, start=1): item = item if isinstance(item, dict) else {} summary.append({ "index": index, - "type": item.get("type") or item.get("eventType"), - "status": item.get("status") or item.get("decision"), - "toolName": item.get("toolName") or item.get("tool"), + "id": item.get("id"), + "status": item.get("status"), + "targetTool": item.get("targetTool"), + "toolCallID": item.get("toolCallID"), + "decisionTime": item.get("decisionTime"), }) Path(sys.argv[2]).write_text(json.dumps({"approvalCount": len(items), "approvals": summary}, indent=2, sort_keys=True) + "\n") PY diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index da41fe524..55a7f4b17 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -26,6 +26,8 @@ Optional environment: ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN generated if absent for this run ORKA_FOUNDRY_RESPONSES_API_VERSION default: v1 ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES default: read + Every advertised class requires the hosted agent to statically expose the + matching probe-only conformance_read/conformance_write schema. ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF optional The script never prints secret values. Do not run with shell tracing (set -x). diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 679668d7d..b50305855 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -367,11 +367,12 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { s.turns[req.TurnID] = turn s.mu.Unlock() - ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.cfg.requestTimeout) + ctx, cancel := s.foundryRequestContext(r.Context(), req.Deadline) defer cancel() var response responsesResponse initialRequest := responsesRequest{Input: req.Input.Prompt} - if err := s.postResponses(ctx, req.RuntimeSessionID, initialRequest, &response); err != nil { + foundrySessionID, err := s.postResponses(ctx, req.RuntimeSessionID, initialRequest, &response) + if err != nil { s.mu.Lock() delete(s.turns, req.TurnID) s.mu.Unlock() @@ -379,7 +380,7 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { return } s.mu.Lock() - s.updateTurnSessionLocked(turn) + s.recordTurnSessionLocked(turn, foundrySessionID) s.mu.Unlock() s.handleResponsesResponse(turn, response) s.mu.Lock() @@ -518,7 +519,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteError(w, http.StatusBadRequest, err.Error()) return } - ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), s.cfg.requestTimeout) + ctx, cancel := s.foundryRequestContext(r.Context(), turn.request.Deadline) defer cancel() var response responsesResponse continuation := responsesRequest{ @@ -526,7 +527,8 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn AgentSessionID: foundrySessionID, Input: outputs, } - if err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response); err != nil { + updatedSessionID, err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response) + if err != nil { s.mu.Lock() s.appendFailedLocked( turn, @@ -539,13 +541,19 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn return } s.mu.Lock() + if turn.completed { + s.mu.Unlock() + harness.WriteError(w, http.StatusConflict, "turn completed while hosted continuation was in flight") + return + } for _, result := range resultsToSubmit { toolName := turn.pendingTools[result.ToolCallID] if toolName == "" { toolName = result.ToolCallID } - s.appendFrameLocked( + frame := s.newFrame( turn, + int64(len(turn.frames)+1), harness.FrameToolResultReceived, "brokered tool result received", func(f *harness.HarnessEventFrame) { @@ -555,12 +563,23 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn f.Error = result.Error }, ) + if !harnessFrameFitsSSE(frame) { + s.appendFailedLocked( + turn, + "brokered_tool_result_frame_too_large", + "brokered tool result exceeded the harness SSE frame limit", + ) + s.mu.Unlock() + harness.WriteError(w, http.StatusBadGateway, "brokered tool result exceeds harness SSE frame limit") + return + } + s.appendPreparedFrameLocked(turn, frame) delete(turn.pendingTools, result.ToolCallID) delete(turn.pendingSince, result.ToolCallID) delete(turn.bufferedResults, result.ToolCallID) delete(turn.bufferedPayloads, result.ToolCallID) } - s.updateTurnSessionLocked(turn) + s.recordTurnSessionLocked(turn, updatedSessionID) s.mu.Unlock() s.handleResponsesResponse(turn, response) harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) @@ -666,13 +685,12 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp return } } - now := time.Now().UTC() - for _, call := range calls { - turn.pendingTools[call.callID] = call.name - turn.pendingSince[call.callID] = now - s.schedulePendingToolTimeoutLocked(turn, call.callID) - s.appendFrameLocked( + frames := make([]harness.HarnessEventFrame, 0, len(calls)) + baseSeq := int64(len(turn.frames) + 1) + for index, call := range calls { + frame := s.newFrame( turn, + baseSeq+int64(index), harness.FrameToolCallRequested, "foundry hosted tool call requested", func(f *harness.HarnessEventFrame) { @@ -681,6 +699,22 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp f.Content = call.args }, ) + if !harnessFrameFitsSSE(frame) { + s.appendFailedLocked( + turn, + "foundry_tool_call_frame_too_large", + "hosted function call exceeded the harness SSE frame limit", + ) + return + } + frames = append(frames, frame) + } + now := time.Now().UTC() + for index, call := range calls { + turn.pendingTools[call.callID] = call.name + turn.pendingSince[call.callID] = now + s.schedulePendingToolTimeoutLocked(turn, call.callID) + s.appendPreparedFrameLocked(turn, frames[index]) } return } @@ -689,14 +723,24 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp s.appendFailedLocked(turn, "foundry_output_too_large", "foundry completion exceeded advertised output limit") return } - s.appendFrameLocked( + completedFrame := s.newFrame( turn, + int64(len(turn.frames)+1), harness.FrameTurnCompleted, "foundry hosted response completed", func(f *harness.HarnessEventFrame) { f.Completed = &harness.TurnCompleted{Result: result, FinalEventSeq: f.Seq} }, ) + if !harnessFrameFitsSSE(completedFrame) { + s.appendFailedLocked( + turn, + "foundry_output_frame_too_large", + "foundry completion exceeded the harness SSE frame limit", + ) + return + } + s.appendPreparedFrameLocked(turn, completedFrame) turn.completed = true s.scheduleTurnCleanupLocked(turn) } @@ -847,11 +891,37 @@ func (s *server) recordContinueResults( continue } toSubmit = append(toSubmit, turn.bufferedResults[id]) - turn.submittedPayloads[id] = turn.bufferedPayloads[id] } if len(toSubmit) == 0 { return nil, nil } + baseSeq := int64(len(turn.frames) + 1) + for index, result := range toSubmit { + toolName := firstNonBlank(turn.pendingTools[result.ToolCallID], result.ToolCallID) + frame := s.newFrame( + turn, + baseSeq+int64(index), + harness.FrameToolResultReceived, + "brokered tool result received", + func(f *harness.HarnessEventFrame) { + f.ToolName = toolName + f.ToolCallID = result.ToolCallID + f.Content = result.Output + f.Error = result.Error + }, + ) + if !harnessFrameFitsSSE(frame) { + s.appendFailedLocked( + turn, + "brokered_tool_result_frame_too_large", + "brokered tool result exceeded the harness SSE frame limit", + ) + return nil, fmt.Errorf("brokered tool result %q exceeds harness SSE frame limit", result.ToolCallID) + } + } + for _, result := range toSubmit { + turn.submittedPayloads[result.ToolCallID] = turn.bufferedPayloads[result.ToolCallID] + } return toSubmit, nil } @@ -945,13 +1015,13 @@ func (s *server) postResponses( runtimeSessionID harness.RuntimeSessionID, body responsesRequest, out *responsesResponse, -) error { +) (string, error) { if !exactlyOneFoundryAuth(s.cfg) { - return fmt.Errorf("exactly one Foundry auth mode is required") + return "", fmt.Errorf("exactly one Foundry auth mode is required") } endpoint, err := s.responsesEndpoint() if err != nil { - return err + return "", err } s.mu.Lock() session := s.runtimeSessions[runtimeSessionID] @@ -968,11 +1038,11 @@ func (s *server) postResponses( } payload, err := json.Marshal(body) if err != nil { - return err + return "", err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) if err != nil { - return err + return "", err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") @@ -991,7 +1061,7 @@ func (s *server) postResponses( } resp, err := s.client.Do(req) if err != nil { - return err + return "", err } defer resp.Body.Close() //nolint:errcheck sessionID = firstNonBlank( @@ -1001,7 +1071,7 @@ func (s *server) postResponses( ) if resp.StatusCode < 200 || resp.StatusCode >= 300 { data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - return fmt.Errorf( + return "", fmt.Errorf( "foundry hosted Responses request failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data)), @@ -1010,16 +1080,11 @@ func (s *server) postResponses( if out != nil { decoder := json.NewDecoder(io.LimitReader(resp.Body, maxFoundryBodyBytes)) if err := decoder.Decode(out); err != nil { - return fmt.Errorf("decode Foundry hosted Responses response: %w", err) + return "", fmt.Errorf("decode Foundry hosted Responses response: %w", err) } sessionID = firstNonBlank(out.AgentSessionID, sessionID) } - if sessionID != "" { - s.mu.Lock() - s.runtimeSessions[runtimeSessionID] = foundrySession{ID: sessionID, LastSeen: time.Now().UTC()} - s.mu.Unlock() - } - return nil + return sessionID, nil } func (s *server) responsesEndpoint() (string, error) { @@ -1129,6 +1194,21 @@ func exactlyOneFoundryAuth(cfg config) bool { return hasKey != hasBearer } +func (s *server) foundryRequestContext( + parent context.Context, + turnDeadline time.Time, +) (context.Context, context.CancelFunc) { + timeout := s.cfg.requestTimeout + if timeout <= 0 { + timeout = defaultRequestTimeout + } + deadline := time.Now().Add(timeout) + if !turnDeadline.IsZero() && turnDeadline.Before(deadline) { + deadline = turnDeadline + } + return context.WithDeadline(context.WithoutCancel(parent), deadline) +} + func (s *server) validateStartRequest(req harness.StartTurnRequest) error { if s.cfg.configError != "" { return errors.New(s.cfg.configError) @@ -1245,11 +1325,13 @@ func isFailureStatus(status string) bool { } } -func (s *server) updateTurnSessionLocked(turn *turnState) { - session := s.runtimeSessions[turn.request.RuntimeSessionID] - if session.ID != "" { - turn.foundrySessionID = session.ID +func (s *server) recordTurnSessionLocked(turn *turnState, sessionID string) { + sessionID = strings.TrimSpace(sessionID) + if sessionID == "" { + return } + turn.foundrySessionID = sessionID + s.runtimeSessions[turn.request.RuntimeSessionID] = foundrySession{ID: sessionID, LastSeen: time.Now().UTC()} } func (s *server) schedulePendingToolTimeoutLocked(turn *turnState, toolCallID string) { @@ -1283,8 +1365,9 @@ func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { if turn.completed { return } - s.appendFrameLocked( + failedFrame := s.newFrame( turn, + int64(len(turn.frames)+1), harness.FrameTurnFailed, "foundry hosted response failed", func(f *harness.HarnessEventFrame) { @@ -1292,6 +1375,21 @@ func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { f.Error = &harness.ErrorInfo{Code: reason, Message: msg} }, ) + if !harnessFrameFitsSSE(failedFrame) { + failedFrame = s.newFrame( + turn, + int64(len(turn.frames)+1), + harness.FrameTurnFailed, + "foundry hosted response failed", + func(f *harness.HarnessEventFrame) { + fallbackReason := "foundry_failure_frame_too_large" + message := "failure detail exceeded the harness SSE frame limit" + f.Failed = &harness.TurnFailed{Reason: fallbackReason, Message: message} + f.Error = &harness.ErrorInfo{Code: fallbackReason, Message: message} + }, + ) + } + s.appendPreparedFrameLocked(turn, failedFrame) turn.completed = true s.scheduleTurnCleanupLocked(turn) } @@ -1335,7 +1433,17 @@ func (s *server) appendFrameLocked( summary string, mutate func(*harness.HarnessEventFrame), ) { - seq := int64(len(turn.frames) + 1) + frame := s.newFrame(turn, int64(len(turn.frames)+1), typ, summary, mutate) + s.appendPreparedFrameLocked(turn, frame) +} + +func (s *server) newFrame( + turn *turnState, + seq int64, + typ harness.FrameType, + summary string, + mutate func(*harness.HarnessEventFrame), +) harness.HarnessEventFrame { frame := harness.HarnessEventFrame{ Version: harness.ProtocolVersion, Type: typ, @@ -1350,6 +1458,15 @@ func (s *server) appendFrameLocked( if mutate != nil { mutate(&frame) } + return frame +} + +func harnessFrameFitsSSE(frame harness.HarnessEventFrame) bool { + payload, err := json.Marshal(frame) + return err == nil && len("data: ")+len(payload) < harness.MaxSSEFrameBytes +} + +func (s *server) appendPreparedFrameLocked(turn *turnState, frame harness.HarnessEventFrame) { turn.frames = append(turn.frames, frame) if turn.frameUpdates != nil { close(turn.frameUpdates) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 0ec4bcded..d2330dcc1 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -168,6 +169,74 @@ func TestResponsesAdapterRuntimeSessionHeaderReuse(t *testing.T) { } } +func TestResponsesAdapterInterleavedResponsesRetainResponseSpecificSession(t *testing.T) { + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := ioReadAll(r.Body) + var request responsesRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + prompt, _ := request.Input.(string) + writeJSON(w, map[string]any{ + "id": "response-" + prompt, + "agent_session_id": "session-" + prompt, + "status": "completed", + "output": []any{map[string]any{ + "type": "message", + "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": "done"}}, + }}, + }) + })) + t.Cleanup(foundry.Close) + server := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + }, &http.Client{Timeout: time.Second}) + firstRequest := responsesStartTurnRequest("foundry-interleaved-session-a") + secondRequest := responsesStartTurnRequest("foundry-interleaved-session-b") + secondRequest.RuntimeSessionID = firstRequest.RuntimeSessionID + + var firstResponse responsesResponse + firstSession, err := server.postResponses( + context.Background(), + firstRequest.RuntimeSessionID, + responsesRequest{Input: "a"}, + &firstResponse, + ) + if err != nil { + t.Fatalf("first postResponses: %v", err) + } + var secondResponse responsesResponse + secondSession, err := server.postResponses( + context.Background(), + secondRequest.RuntimeSessionID, + responsesRequest{Input: "b"}, + &secondResponse, + ) + if err != nil { + t.Fatalf("second postResponses: %v", err) + } + + firstTurn := &turnState{request: firstRequest} + secondTurn := &turnState{request: secondRequest} + server.mu.Lock() + server.recordTurnSessionLocked(secondTurn, secondSession) + server.recordTurnSessionLocked(firstTurn, firstSession) + server.mu.Unlock() + if firstTurn.foundrySessionID != "session-a" { + t.Fatalf("first turn session = %q, want session-a", firstTurn.foundrySessionID) + } + if secondTurn.foundrySessionID != "session-b" { + t.Fatalf("second turn session = %q, want session-b", secondTurn.foundrySessionID) + } +} + func TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) adapter, server := newTestResponsesAdapterWithServer( @@ -297,6 +366,54 @@ func TestResponsesAdapterInitialPostSurvivesControlDisconnect(t *testing.T) { } } +func TestResponsesAdapterFoundryRequestContextDetachesAndUsesEarlierTurnDeadline(t *testing.T) { + server := newServer(config{requestTimeout: time.Second}, &http.Client{Timeout: time.Second}) + parent, cancelParent := context.WithCancel(context.Background()) + cancelParent() + turnDeadline := time.Now().Add(100 * time.Millisecond) + ctx, cancel := server.foundryRequestContext(parent, turnDeadline) + defer cancel() + if err := ctx.Err(); err != nil { + t.Fatalf("detached context inherited control cancellation: %v", err) + } + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("detached context has no deadline") + } + if delta := deadline.Sub(turnDeadline); delta < -10*time.Millisecond || delta > 10*time.Millisecond { + t.Fatalf("deadline = %v, want turn deadline %v", deadline, turnDeadline) + } + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("detached context error = %v, want deadline exceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("detached context did not stop at turn deadline") + } +} + +func TestResponsesAdapterInitialPostHonorsTurnDeadline(t *testing.T) { + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(900 * time.Millisecond) + writeJSON(w, finalResponsesMessage()) + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter := newTestResponsesAdapter(t, endpoint, nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-initial-deadline") + request.Deadline = time.Now().Add(150 * time.Millisecond) + + started := time.Now() + if _, err := client.StartTurn(context.Background(), request); err == nil { + t.Fatal("StartTurn past turn deadline succeeded, want failure") + } + if elapsed := time.Since(started); elapsed > 800*time.Millisecond { + t.Fatalf("StartTurn elapsed = %v, want turn deadline to beat request timeout", elapsed) + } +} + func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { unready := httptest.NewServer(newServer(config{ runtimeName: "foundry-responses-test", @@ -663,6 +780,42 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { } } +func TestResponsesAdapterContinuationHonorsOriginalTurnDeadline(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter := newTestResponsesAdapter( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-continuation-deadline") + request.Deadline = time.Now().Add(300 * time.Millisecond) + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamCurrentFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + if wait := time.Until(request.Deadline); wait > 0 { + time.Sleep(wait + 10*time.Millisecond) + } + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + if _, err := client.ContinueTurn(context.Background(), continueRequest); err == nil { + t.Fatal("ContinueTurn after original deadline succeeded, want failure") + } + if got := foundry.postCount.Load(); got != 1 { + t.Fatalf("hosted post count = %d, want no continuation after turn deadline", got) + } + frames = streamCurrentFrames(t, client, request.TurnID) + failed := findFrame(frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed == nil || failed.Failed.Reason != "foundry_continuation_unknown" { + t.Fatalf("failed frame = %#v, want foundry_continuation_unknown", failed) + } +} + func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -1401,6 +1554,124 @@ func TestResponsesLargeOutputFails(t *testing.T) { } } +func TestResponsesOutputThatExceedsSSEFrameLimitFails(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: responsesStartTurnRequest("foundry-large-frame-output"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-large-frame", + Status: "completed", + Output: []responsesOutput{{ + Type: "message", + Content: strings.Repeat("\"", maxFoundryOutputBytes), + }}, + }) + if hasFrameType(turn.frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, oversized SSE frame should not complete", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_output_frame_too_large" { + t.Fatalf("failed frame = %#v, want foundry_output_frame_too_large", failed) + } +} + +func TestResponsesOversizedToolCallFrameFailsBeforeRequestingTool(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + turn := &turnState{ + request: brokeredReadRequest("foundry-large-tool-call-frame"), + pendingTools: map[string]string{}, + pendingSince: map[string]time.Time{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + arguments, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) + if err != nil { + t.Fatalf("marshal arguments: %v", err) + } + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-large-tool-call", + Status: "completed", + Output: []responsesOutput{{ + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: arguments, + }}, + }) + if hasFrameType(turn.frames, harness.FrameToolCallRequested) { + t.Fatalf("frames = %#v, oversized tool call should not be requested", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_tool_call_frame_too_large" { + t.Fatalf("failed frame = %#v, want foundry_tool_call_frame_too_large", failed) + } +} + +func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) { + server := newServer(config{ + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + maxApprovalWait: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-large-tool-result-frame") + turn := &turnState{ + request: request, + responseID: "resp-1", + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedPayloads: map[string]string{}, + submittedPayloads: map[string]string{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + output, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) + if err != nil { + t.Fatalf("marshal output: %v", err) + } + result := toolResultForRequest(request, "call-1", true, output, nil) + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}); err == nil { + t.Fatal("recordContinueResults oversized output error = nil") + } + if hasFrameType(turn.frames, harness.FrameToolResultReceived) { + t.Fatalf("frames = %#v, oversized tool result should not be streamed", turn.frames) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "brokered_tool_result_frame_too_large" { + t.Fatalf("failed frame = %#v, want brokered_tool_result_frame_too_large", failed) + } +} + func TestResponsesInitialPlatformErrorDoesNotRetainTurn(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "platform_error"}) adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 5875bb49c..3195fbdda 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -99,6 +99,34 @@ expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json \ "duplicate write execution starts for dispatch-work-order" \ "duplicate-write" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json \ + "missing ApprovalApproved event" \ + "missing-approval-decision" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json \ + "follows ApprovalDeclined" \ + "declined-write" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json \ + "is missing approvalID" \ + "missing-approval-id" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json \ + "has no matching preceding ApprovalApproved" \ + "decision-before-request" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json \ + "missing write ToolCallStarted event after approval" \ + "overlapping-write-marker" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json \ + "has no matching preceding mapped request" \ + "mismatched-write-request" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json \ + "write execution for escalate-incident is missing idempotency key evidence" \ + "partial-idempotency" if [[ "$run_full" == "1" ]]; then run make test diff --git a/internal/harness/client.go b/internal/harness/client.go index e9c4698b3..b782ba0e0 100644 --- a/internal/harness/client.go +++ b/internal/harness/client.go @@ -27,7 +27,8 @@ type Client struct { authBearerValue string } -const maxHarnessSSEFrameBytes = 1 << 20 +// MaxSSEFrameBytes is the largest single SSE data line the harness client accepts. +const MaxSSEFrameBytes = 1 << 20 var errSSEDone = errors.New("harness SSE stream done") @@ -327,7 +328,7 @@ func (c *Client) resolve(rel string) *url.URL { func readSSEFrames(r io.Reader, emit func(HarnessEventFrame) error) error { scanner := bufio.NewScanner(r) - scanner.Buffer(make([]byte, 0, 64*1024), maxHarnessSSEFrameBytes) + scanner.Buffer(make([]byte, 0, 64*1024), MaxSSEFrameBytes) var data strings.Builder for scanner.Scan() { line := scanner.Text() diff --git a/website/docs/guides/bring-your-own-agent-runtime.md b/website/docs/guides/bring-your-own-agent-runtime.md index 1dcc570dc..8006e6259 100644 --- a/website/docs/guides/bring-your-own-agent-runtime.md +++ b/website/docs/guides/bring-your-own-agent-runtime.md @@ -91,6 +91,8 @@ For AgentKit agents deployed as Foundry hosted agents, use the `examples/harness Advertise only the brokered classes that the hosted AgentKit deployment is statically configured and conformance-tested to request: +Readiness deep-probes each advertised class. Since hosted Responses requests do not carry request-level tools, the AgentKit deployment must also statically expose the probe-only `conformance_read` and/or `conformance_write` schema for those classes. Leave a class unadvertised until that live probe succeeds; local fake-server conformance alone does not satisfy the readiness gate. + ```yaml apiVersion: core.orka.ai/v1alpha1 kind: AgentRuntime From a6b5d9d57a3a0ae7b1c42d6bfbaa8b802d7b9ba1 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 10:06:04 -0700 Subject: [PATCH 34/51] fix: address Foundry Responses review feedback Signed-off-by: Sertac Ozercan --- .../foundry-responses-events-tail-page.json | 65 +++++++++ ...undry-responses-events-truncated-page.json | 65 +++++++++ .../verify-foundry-responses.sh | 29 +++- examples/harness/foundry-responses/README.md | 2 +- .../harness/foundry-responses/VALIDATION.md | 6 +- .../foundry-responses/fetch_task_events.py | 129 ++++++++++++++++++ .../foundry-responses/live-evidence.sh | 39 +++++- examples/harness/foundry-responses/main.go | 34 ++++- .../harness/foundry-responses/main_test.go | 90 ++++++++++-- .../test_fetch_task_events.py | 76 +++++++++++ .../harness/foundry-responses/validate.sh | 9 ++ 11 files changed, 512 insertions(+), 32 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json create mode 100755 examples/harness/foundry-responses/fetch_task_events.py create mode 100644 examples/harness/foundry-responses/test_fetch_task_events.py diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json new file mode 100644 index 000000000..8149f3d4a --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json @@ -0,0 +1,65 @@ +{ + "events": [ + { + "seq": 101, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 102, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 103, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 104, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 105, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 106, + "eventType": "AgentRuntimeCompleted" + } + ], + "afterSeq": 100, + "latestSeq": 106 +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json new file mode 100644 index 000000000..01effcd54 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json @@ -0,0 +1,65 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + } + ], + "afterSeq": 0, + "latestSeq": 7 +} diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 5b61f5da4..60c63a232 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -6,8 +6,9 @@ usage() { Usage: examples/fibey-custom-agent-demo/verify-foundry-responses.sh [--task NAME] [--namespace NAME] [--json EVENTS.json] Checks the live Fibey Foundry hosted AgentKit Responses scenario evidence from -Orka task events. By default it calls `orka task events --output json`. Use ---json to verify a previously captured event payload without contacting Orka. +Orka task events. By default it paginates `orka task events --output json` +through `latestSeq`. Use --json to verify a previously captured event payload +without contacting Orka; payloads with `latestSeq` must be complete. Expected evidence: - read brokered tool request for check-network-telemetry or get-active-incidents @@ -68,7 +69,9 @@ trap cleanup EXIT if [[ -z "$json_file" ]]; then require_cmd orka json_tmp="$(mktemp)" - orka task events "$task" --namespace "$namespace" --output json >"$json_tmp" + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + event_fetcher="${script_dir}/../harness/foundry-responses/fetch_task_events.py" + python3 "$event_fetcher" --task "$task" --namespace "$namespace" --output "$json_tmp" json_file="$json_tmp" fi @@ -95,6 +98,26 @@ elif isinstance(payload, list): else: raise SystemExit("error: event JSON must be an object or list") +if isinstance(payload, dict) and payload.get("latestSeq") is not None: + try: + after_seq = int(payload.get("afterSeq", 0)) + latest_seq = int(payload["latestSeq"]) + sequences = [int(event.get("seq", 0)) for event in events if isinstance(event, dict)] + except (TypeError, ValueError) as exc: + raise SystemExit(f"error: invalid event sequence metadata: {exc}") from exc + if after_seq != 0: + raise SystemExit(f"error: event JSON is incomplete: afterSeq must be 0, got {after_seq}") + complete_sequence = len(sequences) == latest_seq and all( + seq == expected for expected, seq in enumerate(sequences, start=1) + ) + if not complete_sequence: + captured_seq = max(sequences, default=0) + raise SystemExit( + "error: event JSON is incomplete: " + f"captured sequences do not cover 1 through latestSeq {latest_seq} " + f"(highest captured sequence {captured_seq})" + ) + READ_TOOLS = {"check-network-telemetry", "get-active-incidents"} WRITE_TOOLS = {"dispatch-work-order", "escalate-incident"} TERMINAL_TYPES = { diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 80a9d6437..1e218a0ce 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -24,7 +24,7 @@ Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `exa | `ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN` | Bearer token Orka uses for mutating/streaming harness endpoints. | | `ORKA_FOUNDRY_RESPONSES_ENDPOINT` | Preferred full hosted Responses endpoint URL, including `/agents//endpoint/protocols/openai/responses`. The adapter appends `api-version` from `ORKA_FOUNDRY_RESPONSES_API_VERSION` when missing. | | `ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT` + `ORKA_FOUNDRY_RESPONSES_AGENT_NAME` | Optional alternative to build the hosted Responses endpoint from a project endpoint and agent name. | -| `ORKA_FOUNDRY_RESPONSES_API_VERSION` | API version query value, default `v1`. | +| `ORKA_FOUNDRY_RESPONSES_API_VERSION` | API-version query value, default `v1`, matching the current `AIProjectClient.get_openai_client(agent_name=...)` default query. Override it for deployments pinned to another preview version. | | `ORKA_FOUNDRY_RESPONSES_API_KEY` | Static API-key auth mode. Tests/demo only unless your deployment standard permits it. | | `ORKA_FOUNDRY_RESPONSES_AUTH_BEARER` | Static bearer auth mode. Tests/demo only unless supplied by a production token refresher sidecar. | | `ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE` | Reserved for future workload-identity token refresh support; currently not used. | diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 4e689b49f..b3256284f 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -70,14 +70,14 @@ uv run --extra dev pytest -q \ | Duplicate/replayed function calls fail closed | `TestResponsesRepeatedSubmittedFunctionCallFailsTurn` and `TestResponsesMixedRepeatedFunctionCallFailsTurn`. | | Duplicate identical `/continue` is idempotent and conflicting duplicates reject | `TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject`, `TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit`, and `TestResponsesAdapterAlreadySubmittedPendingResultIsNoop`. | | Continuation failures fail closed rather than duplicating hosted progress | `TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost`. | -| Pending approval/tool waits are bounded | `TestResponsesAdapterPendingToolTimesOutWithoutContinuation`, `TestResponsesAdapterPendingTimeoutSkipsSubmittedCall`, and `TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait`. | +| Pending approval/tool waits are bounded without understating total brokered duration | `TestResponsesAdapterPendingToolTimesOutWithoutContinuation`, `TestResponsesAdapterPendingTimeoutSkipsSubmittedCall`, and `TestResponsesAdapterBrokeredMaxTurnIsUnknown`. | | Restart/state loss fails safely without hosted continuation | `TestResponsesAdapterStateLossContinueFailsSafely`. | | Runtime session continuity | `TestResponsesAdapterRuntimeSessionHeaderReuse` and `TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup`. | | Hosted response status handling is fail-closed | `TestResponsesFailureStatusDoesNotCompleteWithPartialText`, `TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest`, `TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText`, and `TestResponsesMissingStatusDoesNotCompleteWithPartialText`. | -| Large hosted output and platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorDoesNotRetainTurn`. | +| Large hosted output and uncertain initial platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorRetainsFailedTurn`. | | No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | | Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | -| Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`. | +| Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` tests paginated event aggregation and runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`, including an explicitly truncated event page. | | Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | ## Live gates that cannot be satisfied by local fixtures diff --git a/examples/harness/foundry-responses/fetch_task_events.py b/examples/harness/foundry-responses/fetch_task_events.py new file mode 100755 index 000000000..8da0d3290 --- /dev/null +++ b/examples/harness/foundry-responses/fetch_task_events.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Fetch a complete Orka task event stream through the paginated CLI.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable + +PAGE_LIMIT = 1000 +MAX_PAGES = 10000 + +Runner = Callable[..., subprocess.CompletedProcess[str]] + + +def _int_field(payload: dict[str, Any], name: str) -> int: + value = payload.get(name, 0) + if isinstance(value, bool): + raise ValueError(f"{name} must be an integer") + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be an integer") from exc + if parsed < 0: + raise ValueError(f"{name} must be non-negative") + return parsed + + +def fetch_task_events( + task: str, + namespace: str, + *, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + after = 0 + all_events: list[dict[str, Any]] = [] + first_payload: dict[str, Any] | None = None + + for _ in range(MAX_PAGES): + command = [ + "orka", + "task", + "events", + task, + "--namespace", + namespace, + "--after", + str(after), + "--limit", + str(PAGE_LIMIT), + "--output", + "json", + ] + result = runner(command, capture_output=True, text=True, check=False) + if result.returncode != 0: + detail = result.stderr.strip() or result.stdout.strip() or "unknown error" + raise RuntimeError(f"orka task events failed: {detail}") + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ValueError(f"orka task events returned invalid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError("orka task events response must be a JSON object") + if first_payload is None: + first_payload = payload + + response_after = _int_field(payload, "afterSeq") + latest = _int_field(payload, "latestSeq") + if response_after != after: + raise ValueError( + f"orka task events returned afterSeq {response_after}, expected {after}" + ) + events = payload.get("events") + if not isinstance(events, list): + raise ValueError("orka task events response is missing an events array") + + page_events: list[dict[str, Any]] = [] + last_seq = after + expected_seq = after + 1 + for event in events: + if not isinstance(event, dict): + raise ValueError("orka task events returned a non-object event") + seq = _int_field(event, "seq") + if seq != expected_seq: + raise ValueError( + f"orka task events returned sequence {seq}, expected {expected_seq}" + ) + last_seq = seq + expected_seq += 1 + page_events.append(event) + all_events.extend(page_events) + + if last_seq >= latest: + base = dict(first_payload) + base["afterSeq"] = 0 + base["latestSeq"] = latest + base["events"] = all_events + return base + if not page_events: + raise RuntimeError( + f"orka task events stopped at sequence {after} before latestSeq {latest}" + ) + after = last_seq + + raise RuntimeError(f"orka task events exceeded {MAX_PAGES} pages") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task", required=True) + parser.add_argument("--namespace", required=True) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + try: + payload = fetch_task_events(args.task, args.namespace) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + except (OSError, RuntimeError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index c537f33df..ca144341b 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -7,8 +7,9 @@ Usage: examples/harness/foundry-responses/live-evidence.sh [--namespace NAME] [- Capture a credentials-safe evidence bundle for the live Foundry hosted Responses/Fibey gate after the task has run. The bundle stores Kubernetes -AgentRuntime metadata, Orka task events/approvals, verifier output, and a -summary-only adapter log scan. It intentionally does not store raw adapter logs. +AgentRuntime metadata, the complete paginated Orka task event stream, approvals, +verifier output, and a summary-only adapter log scan. It intentionally does not +store raw adapter logs. Defaults: --namespace default @@ -180,7 +181,11 @@ scan_saved_artifact "$agentruntime_json" "AgentRuntime summary" events_tmp="$(mktemp)" approvals_tmp="$(mktemp)" -orka task events "$task" --namespace "$namespace" --output json >"$events_tmp" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +python3 "${script_dir}/fetch_task_events.py" \ + --task "$task" \ + --namespace "$namespace" \ + --output "$events_tmp" orka task approvals "$task" --namespace "$namespace" --output json >"$approvals_tmp" python3 - "$events_tmp" "$events_json" <<'PY' import json @@ -243,6 +248,26 @@ def has_idempotency(value): return has_idempotency(decoded) return False +latest_seq = payload.get("latestSeq") if isinstance(payload, dict) else None +sequence_values = [ + int(event.get("seq", 0)) + for event in events + if isinstance(event, dict) and event.get("seq") is not None +] +captured_through_seq = max(sequence_values, default=0) +if latest_seq is not None: + after_seq = int(payload.get("afterSeq", 0)) + if after_seq != 0: + raise SystemExit(f"task event capture is incomplete: afterSeq must be 0, got {after_seq}") + complete_sequence = len(sequence_values) == int(latest_seq) and all( + seq == expected for expected, seq in enumerate(sequence_values, start=1) + ) + if not complete_sequence: + raise SystemExit( + "task event capture is incomplete: " + f"sequences do not cover 1 through latestSeq {latest_seq}" + ) + summary = [] for index, event in enumerate(events, start=1): event = event if isinstance(event, dict) else {} @@ -253,7 +278,12 @@ for index, event in enumerate(events, start=1): "hasIdempotencyEvidence": has_idempotency(event), "hasError": bool(event_field(event, "error", "errorCode")), }) -Path(sys.argv[2]).write_text(json.dumps({"eventCount": len(events), "events": summary}, indent=2, sort_keys=True) + "\n") +Path(sys.argv[2]).write_text(json.dumps({ + "eventCount": len(events), + "capturedThroughSeq": captured_through_seq, + "latestSeq": latest_seq, + "events": summary, +}, indent=2, sort_keys=True) + "\n") PY scan_saved_artifact "$events_json" "task events summary" python3 - "$approvals_tmp" "$approvals_json" <<'PY' @@ -285,7 +315,6 @@ Path(sys.argv[2]).write_text(json.dumps({"approvalCount": len(items), "approvals PY scan_saved_artifact "$approvals_json" "task approvals summary" -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" fibey_verifier="${script_dir}/../fibey-custom-agent-demo/verify-foundry-responses.sh" if [[ ! -x "$fibey_verifier" ]]; then fibey_verifier="${script_dir}/../../fibey-custom-agent-demo/verify-foundry-responses.sh" diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index b50305855..79ee16b0d 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -35,6 +35,7 @@ const ( maxFoundryOutputBytes = 1 << 20 maxFoundryBodyBytes = 4 << 20 readinessPath = "/v1/ready" + foundryInitialUnknown = "foundry_initial_unknown" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -288,7 +289,11 @@ func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { maxTurnSeconds := int(s.cfg.requestTimeout.Seconds()) if len(s.cfg.brokeredToolClasses) > 0 { modes = append(modes, harness.ToolExecutionModeBrokered) - maxTurnSeconds = int((s.cfg.requestTimeout + s.cfg.maxApprovalWait).Seconds()) + // A brokered turn can contain multiple hosted request/tool-result rounds, + // each with its own request timeout and approval wait. The harness contract + // has only one runtime-wide ceiling, so advertise unknown rather than an + // understated duration when brokered mode is available. + maxTurnSeconds = 0 } harness.WriteJSON(w, http.StatusOK, harness.CapabilitiesResponse{ Version: harness.ProtocolVersion, @@ -374,9 +379,23 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { foundrySessionID, err := s.postResponses(ctx, req.RuntimeSessionID, initialRequest, &response) if err != nil { s.mu.Lock() - delete(s.turns, req.TurnID) + turn.initializing = false + s.appendFailedLocked( + turn, + foundryInitialUnknown, + "initial hosted request failed after submission was attempted; "+ + "failing closed to avoid a duplicate hosted turn", + ) s.mu.Unlock() - harness.WriteError(w, http.StatusBadGateway, err.Error()) + log.Printf( + "Foundry hosted Responses initial request failed after submission for turn %q (error type %T)", + req.TurnID, + err, + ) + // Accept the turn so Orka consumes and persists the terminal failure frame. + // Returning a control-plane error here would cause the caller to retry a + // submission whose outcome is unknown at Foundry. + harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) return } s.mu.Lock() @@ -534,10 +553,15 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn turn, "foundry_continuation_unknown", "hosted continuation failed after submission was attempted; "+ - "failing closed to avoid duplicate continuation: "+err.Error(), + "failing closed to avoid duplicate continuation", ) s.mu.Unlock() - harness.WriteError(w, http.StatusBadGateway, err.Error()) + log.Printf( + "Foundry hosted Responses continuation failed after submission for turn %q (error type %T)", + req.TurnID, + err, + ) + harness.WriteError(w, http.StatusBadGateway, "hosted continuation failed after submission was attempted") return } s.mu.Lock() diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index d2330dcc1..6fb8126ad 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -69,8 +69,14 @@ func TestResponsesAdapterDoesNotFollowCredentialedRedirects(t *testing.T) { "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" adapter := newTestResponsesAdapter(t, endpoint, nil) client := newHarnessClient(t, adapter) - if _, err := client.StartTurn(context.Background(), responsesStartTurnRequest("foundry-redirect")); err == nil { - t.Fatalf("StartTurn followed redirect and succeeded, want rejection") + request := responsesStartTurnRequest("foundry-redirect") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn should accept terminal redirect failure: %v", err) + } + frames := streamCurrentFrames(t, client, request.TurnID) + if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || + failed.Failed.Reason != foundryInitialUnknown { + t.Fatalf("failed frame = %#v, want foundry_initial_unknown", failed) } if redirectTargetHit.Load() { t.Fatal("redirect target was called; credentialed Foundry request followed an unvalidated redirect") @@ -406,12 +412,17 @@ func TestResponsesAdapterInitialPostHonorsTurnDeadline(t *testing.T) { request.Deadline = time.Now().Add(150 * time.Millisecond) started := time.Now() - if _, err := client.StartTurn(context.Background(), request); err == nil { - t.Fatal("StartTurn past turn deadline succeeded, want failure") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn should accept terminal deadline failure: %v", err) } if elapsed := time.Since(started); elapsed > 800*time.Millisecond { t.Fatalf("StartTurn elapsed = %v, want turn deadline to beat request timeout", elapsed) } + frames := streamCurrentFrames(t, client, request.TurnID) + if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || + failed.Failed.Reason != foundryInitialUnknown { + t.Fatalf("failed frame = %#v, want foundry_initial_unknown", failed) + } } func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { @@ -963,9 +974,13 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t t.Fatalf("frames = %#v, want tool request", frames) } continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) - if _, err := client.ContinueTurn(context.Background(), continueRequest); err == nil { + _, continueErr := client.ContinueTurn(context.Background(), continueRequest) + if continueErr == nil { t.Fatalf("ContinueTurn succeeded, want hosted continuation failure") } + if strings.Contains(continueErr.Error(), foundry.URL) || strings.Contains(continueErr.Error(), "HTTP 500") { + t.Fatalf("ContinueTurn error leaked upstream detail: %v", continueErr) + } if foundry.postCount.Load() != 2 { t.Fatalf("hosted post count after failed continue = %d, want 2", foundry.postCount.Load()) } @@ -980,6 +995,10 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t if failed == nil || failed.Failed.Reason != "foundry_continuation_unknown" { t.Fatalf("failed frame = %#v, want fail-closed continuation failure", failed) } + if strings.Contains(failed.Failed.Message, foundry.URL) || + strings.Contains(failed.Failed.Message, "HTTP 500") { + t.Fatalf("failed frame leaked upstream detail: %#v", failed.Failed) + } } func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { @@ -1164,7 +1183,7 @@ func TestResponsesAdapterPendingTimeoutSkipsSubmittedCall(t *testing.T) { } } -func TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait(t *testing.T) { +func TestResponsesAdapterBrokeredMaxTurnIsUnknown(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) s := newServer(config{ runtimeName: "foundry-responses-test", @@ -1184,8 +1203,8 @@ func TestResponsesAdapterBrokeredMaxTurnIncludesApprovalWait(t *testing.T) { if err != nil { t.Fatalf("Capabilities: %v", err) } - if caps.MaxTurnSeconds < int((30*time.Minute + 2*time.Second).Seconds()) { - t.Fatalf("MaxTurnSeconds = %d, want approval wait included", caps.MaxTurnSeconds) + if caps.MaxTurnSeconds != 0 { + t.Fatalf("MaxTurnSeconds = %d, want unknown ceiling for brokered turns", caps.MaxTurnSeconds) } } @@ -1235,6 +1254,17 @@ func TestResponsesAdapterStateLossContinueFailsSafely(t *testing.T) { } } +func TestResponsesAPIVersionDefaultsToSDKValue(t *testing.T) { + t.Setenv(envAPIVersion, "") + if got := loadConfig().apiVersion; got != defaultAPIVersion { + t.Fatalf("loadConfig apiVersion = %q, want %q", got, defaultAPIVersion) + } + t.Setenv(envAPIVersion, "2025-11-15-preview") + if got := loadConfig().apiVersion; got != "2025-11-15-preview" { + t.Fatalf("loadConfig apiVersion override = %q", got) + } +} + func TestResponsesEndpointSafety(t *testing.T) { tests := []struct { name string @@ -1242,7 +1272,12 @@ func TestResponsesEndpointSafety(t *testing.T) { want bool }{ { - name: "https responses", + name: "https responses without version", + endpoint: "https://example.openai.azure.com/agents/a/endpoint/protocols/openai/responses", + want: true, + }, + { + name: "https responses with explicit version", endpoint: "https://example.openai.azure.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", want: true, }, @@ -1672,19 +1707,44 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) } } -func TestResponsesInitialPlatformErrorDoesNotRetainTurn(t *testing.T) { +func TestResponsesInitialPlatformErrorRetainsFailedTurn(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "platform_error"}) adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) client := newHarnessClient(t, adapter) request := responsesStartTurnRequest("foundry-platform-error") - if _, err := client.StartTurn(context.Background(), request); err == nil { - t.Fatal("StartTurn succeeded, want platform error") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn should accept the terminal failed turn: %v", err) } server.mu.Lock() - defer server.mu.Unlock() - if _, exists := server.turns[request.TurnID]; exists { - t.Fatalf("turn %q retained after failed initial hosted response", request.TurnID) + turn := server.turns[request.TurnID] + if turn == nil { + server.mu.Unlock() + t.Fatalf("turn %q was discarded after an uncertain initial hosted response", request.TurnID) + } + if turn.initializing || !turn.completed { + server.mu.Unlock() + t.Fatalf("turn state = initializing:%v completed:%v, want terminal failed turn", turn.initializing, turn.completed) + } + failed := findFrame(turn.frames, harness.FrameTurnFailed) + server.mu.Unlock() + if failed == nil || failed.Failed.Reason != foundryInitialUnknown { + t.Fatalf("failed frame = %#v, want foundry_initial_unknown", failed) + } + if strings.Contains(failed.Failed.Message, foundry.URL) || + strings.Contains(failed.Failed.Message, "unknown scenario") { + t.Fatalf("failed frame leaked upstream detail: %#v", failed.Failed) + } + frames := streamCurrentFrames(t, client, request.TurnID) + if streamed := findFrame(frames, harness.FrameTurnFailed); streamed == nil || + streamed.Failed.Reason != foundryInitialUnknown { + t.Fatalf("streamed failed frame = %#v, want foundry_initial_unknown", streamed) + } + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("retrying retained TurnID: %v", err) + } + if foundry.postCount.Load() != 1 { + t.Fatalf("hosted post count after retry = %d, want 1", foundry.postCount.Load()) } } diff --git a/examples/harness/foundry-responses/test_fetch_task_events.py b/examples/harness/foundry-responses/test_fetch_task_events.py new file mode 100644 index 000000000..7234485f4 --- /dev/null +++ b/examples/harness/foundry-responses/test_fetch_task_events.py @@ -0,0 +1,76 @@ +import importlib.util +import json +import subprocess +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).with_name("fetch_task_events.py") +SPEC = importlib.util.spec_from_file_location("fetch_task_events", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class FakeRunner: + def __init__(self, pages): + self.pages = pages + self.commands = [] + + def __call__(self, command, **kwargs): + self.commands.append((command, kwargs)) + after = int(command[command.index("--after") + 1]) + payload = self.pages[after] + return subprocess.CompletedProcess(command, 0, json.dumps(payload), "") + + +class FetchTaskEventsTests(unittest.TestCase): + def test_fetches_until_latest_sequence(self): + runner = FakeRunner( + { + 0: { + "namespace": "ns", + "streamType": "task", + "streamID": "task-a", + "afterSeq": 0, + "latestSeq": 3, + "events": [{"seq": 1}, {"seq": 2}], + }, + 2: { + "namespace": "ns", + "streamType": "task", + "streamID": "task-a", + "afterSeq": 2, + "latestSeq": 3, + "events": [{"seq": 3}], + }, + } + ) + + payload = MODULE.fetch_task_events("task-a", "ns", runner=runner) + + self.assertEqual([event["seq"] for event in payload["events"]], [1, 2, 3]) + self.assertEqual(payload["latestSeq"], 3) + self.assertEqual(payload["afterSeq"], 0) + self.assertEqual(len(runner.commands), 2) + self.assertIn("1000", runner.commands[0][0]) + + def test_accepts_empty_stream(self): + runner = FakeRunner({0: {"afterSeq": 0, "latestSeq": 0, "events": []}}) + payload = MODULE.fetch_task_events("task-a", "ns", runner=runner) + self.assertEqual(payload["events"], []) + + def test_rejects_empty_truncated_page(self): + runner = FakeRunner({0: {"afterSeq": 0, "latestSeq": 2, "events": []}}) + with self.assertRaisesRegex(RuntimeError, "stopped at sequence 0"): + MODULE.fetch_task_events("task-a", "ns", runner=runner) + + def test_rejects_sequence_gaps(self): + runner = FakeRunner( + {0: {"afterSeq": 0, "latestSeq": 2, "events": [{"seq": 2}]}} + ) + with self.assertRaisesRegex(ValueError, "sequence 2, expected 1"): + MODULE.fetch_task_events("task-a", "ns", runner=runner) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 3195fbdda..b166a0097 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -59,6 +59,7 @@ run go test \ ./internal/harness \ ./internal/harness/conformance run go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)' +run python3 -m unittest examples/harness/foundry-responses/test_fetch_task_events.py while IFS= read -r -d '' script; do run bash -n "$script" done < <(find examples -type f -name '*.sh' -print0 | sort -z) @@ -127,6 +128,14 @@ expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json \ "write execution for escalate-incident is missing idempotency key evidence" \ "partial-idempotency" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json \ + "event JSON is incomplete" \ + "truncated-event-page" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json \ + "afterSeq must be 0" \ + "tail-event-page" if [[ "$run_full" == "1" ]]; then run make test From 11baaa08f63a69e33bb9df366f0e8b77265f7c86 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 11:09:35 -0700 Subject: [PATCH 35/51] fix: harden Foundry turn admission Signed-off-by: Sertac Ozercan --- .../harness/foundry-responses/VALIDATION.md | 1 + .../harness/foundry-responses/live-smoke.sh | 12 +- examples/harness/foundry-responses/main.go | 55 +++++- .../harness/foundry-responses/main_test.go | 167 ++++++++++++++++++ .../harness/foundry-responses/validate.sh | 18 ++ 5 files changed, 248 insertions(+), 5 deletions(-) diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index b3256284f..1239b9811 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -77,6 +77,7 @@ uv run --extra dev pytest -q \ | Large hosted output and uncertain initial platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorRetainsFailedTurn`. | | No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | | Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | +| Turn admission, cancel identity, and endpoint safety | Focused adapter tests cover bounded consumed-turn tombstones, malformed/mismatched cancel rejection before mutation, and empty-hostname endpoint rejection; `validate.sh` also checks the live-smoke shell preflight. | | Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` tests paginated event aggregation and runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`, including an explicitly truncated event page. | | Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 55a7f4b17..13e64c99c 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -93,6 +93,15 @@ require_cmd() { command -v "$1" >/dev/null 2>&1 || fail "$1 is required" } +authority_has_hostname() { + local authority="$1" + if [[ "$authority" == \[* ]]; then + [[ "$authority" =~ ^\[[^]]+\](:[0-9]+)?$ ]] + return + fi + [[ -n "${authority%%:*}" ]] +} + is_https_or_loopback_http() { local value="$1" local rest authority @@ -103,7 +112,8 @@ is_https_or_loopback_http() { authority="${rest%%/*}" authority="${authority%%\?*}" authority="${authority%%#*}" - [[ -n "$authority" && "$authority" != *@* && "$authority" != *[[:space:]]* ]] + [[ -n "$authority" && "$authority" != *@* && "$authority" != *[[:space:]]* ]] || return 1 + authority_has_hostname "$authority" return fi [[ "$value" == http://* ]] || return 1 diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 79ee16b0d..2204f8c7d 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -34,6 +34,7 @@ const ( defaultIdleTimeout = 60 * time.Second maxFoundryOutputBytes = 1 << 20 maxFoundryBodyBytes = 4 << 20 + maxConsumedTurnIDs = 1024 readinessPath = "/v1/ready" foundryInitialUnknown = "foundry_initial_unknown" @@ -78,6 +79,8 @@ type server struct { mu sync.Mutex turns map[harness.HarnessTurnID]*turnState + consumedTurns map[harness.HarnessTurnID]struct{} + consumedOrder []harness.HarnessTurnID runtimeSessions map[harness.RuntimeSessionID]foundrySession } @@ -220,6 +223,7 @@ func newServer(cfg config, client *http.Client) *server { cfg: cfg, client: &clientCopy, turns: map[harness.HarnessTurnID]*turnState{}, + consumedTurns: map[harness.HarnessTurnID]struct{}{}, runtimeSessions: map[harness.RuntimeSessionID]foundrySession{}, } } @@ -358,6 +362,11 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteJSON(w, http.StatusAccepted, response) return } + if _, consumed := s.consumedTurns[req.TurnID]; consumed { + s.mu.Unlock() + harness.WriteError(w, http.StatusConflict, "turn already completed") + return + } turn := &turnState{ request: req, initializing: true, @@ -614,6 +623,19 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt harness.WriteError(w, http.StatusMethodNotAllowed, "method not allowed") return } + var req harness.CancelTurnRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + harness.WriteError(w, http.StatusBadRequest, "invalid JSON request") + return + } + if err := req.Validate(); err != nil { + harness.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if !sameCancelIdentity(turn.request, req) { + harness.WriteError(w, http.StatusBadRequest, "cancel request does not match started turn") + return + } turn.continueMu.Lock() defer turn.continueMu.Unlock() s.mu.Lock() @@ -622,7 +644,6 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt turn.completed = true s.scheduleTurnCleanupLocked(turn) } - req := turn.request s.mu.Unlock() harness.WriteJSON( w, @@ -1157,7 +1178,8 @@ func responsesEndpointWithVersion(raw, apiVersion string) (string, error) { func responsesEndpointIsSafe(raw string) bool { trimmed := strings.TrimSpace(raw) parsed, err := url.Parse(trimmed) - if err != nil || parsed.Scheme == "" || parsed.Host == "" || strings.TrimSpace(parsed.Path) == "" { + if err != nil || parsed.Scheme == "" || parsed.Host == "" || strings.TrimSpace(parsed.Hostname()) == "" || + strings.TrimSpace(parsed.Path) == "" { return false } if parsed.User != nil || parsed.ForceQuery || parsed.Fragment != "" || strings.Contains(trimmed, "#") { @@ -1195,7 +1217,7 @@ func responsesEndpointIsSafe(raw string) bool { func projectEndpointIsSafe(raw string) bool { trimmed := strings.TrimSpace(raw) parsed, err := url.Parse(trimmed) - if err != nil || parsed.Scheme == "" || parsed.Host == "" { + if err != nil || parsed.Scheme == "" || parsed.Host == "" || strings.TrimSpace(parsed.Hostname()) == "" { return false } if parsed.User != nil || parsed.ForceQuery || parsed.RawQuery != "" || parsed.Fragment != "" || @@ -1427,7 +1449,10 @@ func (s *server) scheduleTurnCleanupLocked(turn *turnState) { time.AfterFunc(retention, func() { s.mu.Lock() defer s.mu.Unlock() - delete(s.turns, turnID) + if current := s.turns[turnID]; current == turn { + delete(s.turns, turnID) + s.markTurnConsumedLocked(turnID) + } activeSessions := s.activeRuntimeSessionsLocked() cutoff := time.Now().UTC().Add(-retention) for sessionID, session := range s.runtimeSessions { @@ -1441,6 +1466,19 @@ func (s *server) scheduleTurnCleanupLocked(turn *turnState) { }) } +func (s *server) markTurnConsumedLocked(turnID harness.HarnessTurnID) { + if _, consumed := s.consumedTurns[turnID]; consumed { + return + } + s.consumedTurns[turnID] = struct{}{} + s.consumedOrder = append(s.consumedOrder, turnID) + for len(s.consumedOrder) > maxConsumedTurnIDs { + oldest := s.consumedOrder[0] + s.consumedOrder = s.consumedOrder[1:] + delete(s.consumedTurns, oldest) + } +} + func (s *server) activeRuntimeSessionsLocked() map[harness.RuntimeSessionID]bool { active := map[harness.RuntimeSessionID]bool{} for _, turn := range s.turns { @@ -1515,6 +1553,15 @@ func sameStartTurnRequest(existing, retry harness.StartTurnRequest) bool { return reflect.DeepEqual(existing, retry) } +func sameCancelIdentity(start harness.StartTurnRequest, cancel harness.CancelTurnRequest) bool { + return start.Namespace == cancel.Namespace && + start.TaskName == cancel.TaskName && + start.SessionName == cancel.SessionName && + start.RuntimeSessionID == cancel.RuntimeSessionID && + start.TurnID == cancel.TurnID && + start.CorrelationID == cancel.CorrelationID +} + func sameContinueIdentity(start harness.StartTurnRequest, cont harness.ContinueTurnRequest) bool { return start.Namespace == cont.Namespace && start.TaskName == cont.TaskName && diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 6fb8126ad..e2cec85c6 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -1265,6 +1266,133 @@ func TestResponsesAPIVersionDefaultsToSDKValue(t *testing.T) { } } +func TestResponsesAdapterRejectsConsumedTurnAfterCleanup(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "platform_error"}) + server := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.endpoint(), + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: 50 * time.Millisecond, + maxApprovalWait: time.Minute, + }, &http.Client{Timeout: time.Second}) + adapter := httptest.NewServer(server.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-consumed-turn") + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + server.mu.Lock() + _, retainedBeforeCleanup := server.turns[request.TurnID] + _, consumedBeforeCleanup := server.consumedTurns[request.TurnID] + server.mu.Unlock() + if !retainedBeforeCleanup || consumedBeforeCleanup { + t.Fatalf( + "pre-cleanup state retained=%v consumed=%v, want retained state to own admission", + retainedBeforeCleanup, + consumedBeforeCleanup, + ) + } + deadline := time.Now().Add(time.Second) + for { + server.mu.Lock() + _, retained := server.turns[request.TurnID] + _, consumed := server.consumedTurns[request.TurnID] + server.mu.Unlock() + if !retained && consumed { + break + } + if time.Now().After(deadline) { + t.Fatal("terminal turn was not evicted into the consumed-turn tombstone") + } + time.Sleep(5 * time.Millisecond) + } + if _, err := client.StartTurn(context.Background(), request); err == nil || + !strings.Contains(err.Error(), "turn already completed") { + t.Fatalf("retry after cleanup error = %v, want consumed-turn conflict", err) + } + if got := foundry.postCount.Load(); got != 1 { + t.Fatalf("hosted post count after consumed retry = %d, want 1", got) + } +} + +func TestResponsesConsumedTurnTombstonesAreBounded(t *testing.T) { + server := newServer(config{}, nil) + server.mu.Lock() + for i := 0; i <= maxConsumedTurnIDs; i++ { + server.markTurnConsumedLocked(harness.HarnessTurnID(fmt.Sprintf("turn-%d", i))) + } + _, oldestRetained := server.consumedTurns["turn-0"] + _, newestRetained := server.consumedTurns[harness.HarnessTurnID(fmt.Sprintf("turn-%d", maxConsumedTurnIDs))] + count := len(server.consumedTurns) + server.mu.Unlock() + if count != maxConsumedTurnIDs || oldestRetained || !newestRetained { + t.Fatalf( + "consumed tombstone count=%d oldest=%v newest=%v, want bounded FIFO", + count, + oldestRetained, + newestRetained, + ) + } +} + +func TestResponsesAdapterValidatesCancelIdentityBeforeMutation(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) + adapter, server := newTestResponsesAdapterWithServer( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-cancel-identity") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + + cancelPath, err := harness.CancelTurnPath(request.TurnID) + if err != nil { + t.Fatalf("CancelTurnPath: %v", err) + } + malformed, err := http.NewRequest(http.MethodPost, adapter.URL+cancelPath, strings.NewReader("{")) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + malformed.Header.Set("Authorization", "Bearer adapter-auth-value") + response, err := http.DefaultClient.Do(malformed) + if err != nil { + t.Fatalf("malformed cancel request: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed cancel status = %d, want 400", response.StatusCode) + } + + mismatched := cancelRequestForStart(request) + mismatched.CorrelationID = "other-correlation" + if _, err := client.CancelTurn(context.Background(), mismatched); err == nil || + !strings.Contains(err.Error(), "does not match started turn") { + t.Fatalf("mismatched cancel error = %v, want identity rejection", err) + } + server.mu.Lock() + turn := server.turns[request.TurnID] + completedAfterReject := turn == nil || turn.completed + server.mu.Unlock() + if completedAfterReject { + t.Fatal("malformed or mismatched cancel mutated the active turn") + } + + if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { + t.Fatalf("valid CancelTurn: %v", err) + } + frames := streamCurrentFrames(t, client, request.TurnID) + if !hasFrameType(frames, harness.FrameTurnCancelled) { + t.Fatalf("frames = %#v, want TurnCancelled", frames) + } +} + func TestResponsesEndpointSafety(t *testing.T) { tests := []struct { name string @@ -1286,6 +1414,11 @@ func TestResponsesEndpointSafety(t *testing.T) { endpoint: "http://127.0.0.1:8080/agents/a/endpoint/protocols/openai/responses?api-version=v1", want: true, }, + { + name: "empty hostname", + endpoint: "https://:443/agents/a/endpoint/protocols/openai/responses?api-version=v1", + want: false, + }, { name: "http non-loopback", endpoint: "http://example.com/agents/a/endpoint/protocols/openai/responses?api-version=v1", @@ -1317,6 +1450,27 @@ func TestResponsesEndpointSafety(t *testing.T) { } } +func TestProjectEndpointSafety(t *testing.T) { + tests := []struct { + name string + endpoint string + want bool + }{ + {name: "https", endpoint: "https://example.services.ai.azure.com", want: true}, + {name: "empty hostname", endpoint: "https://:443", want: false}, + {name: "query", endpoint: "https://example.services.ai.azure.com?unsafe=x", want: false}, + {name: "loopback", endpoint: "http://127.0.0.1:8080", want: true}, + {name: "non-loopback http", endpoint: "http://example.com", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := projectEndpointIsSafe(tt.endpoint); got != tt.want { + t.Fatalf("projectEndpointIsSafe(%q) = %v, want %v", tt.endpoint, got, tt.want) + } + }) + } +} + func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { server := newServer( config{ @@ -2195,6 +2349,19 @@ func responsesStartTurnRequest(name string) harness.StartTurnRequest { } } +func cancelRequestForStart(start harness.StartTurnRequest) harness.CancelTurnRequest { + return harness.CancelTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: start.Namespace, + TaskName: start.TaskName, + SessionName: start.SessionName, + RuntimeSessionID: start.RuntimeSessionID, + TurnID: start.TurnID, + CorrelationID: start.CorrelationID, + Reason: "test cancellation", + } +} + func brokeredReadRequest(name string) harness.StartTurnRequest { request := responsesStartTurnRequest(name) request.ToolExecutionMode = harness.ToolExecutionModeBrokered diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index b166a0097..fbfa37f40 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -90,6 +90,24 @@ expect_verifier_failure() { rm -f "$out_file" "$err_file" } +live_smoke_err="$(mktemp)" +set +e +ORKA_FOUNDRY_RESPONSES_ENDPOINT="https://:443/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT="" \ + ORKA_FOUNDRY_RESPONSES_AGENT_NAME="" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + examples/harness/foundry-responses/live-smoke.sh >/dev/null 2>"$live_smoke_err" +live_smoke_code=$? +set -e +if [[ "$live_smoke_code" == "0" ]] || ! grep -q "safe /responses URL" "$live_smoke_err"; then + cat "$live_smoke_err" >&2 + rm -f "$live_smoke_err" + echo "expected empty-hostname live smoke preflight to fail" >&2 + exit 1 +fi +rm -f "$live_smoke_err" + run examples/fibey-custom-agent-demo/verify-foundry-responses.sh \ --json examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json expect_verifier_failure \ From 431bf50c7fcd223d5abdbe52c61f6e2023ab3312 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 12:10:17 -0700 Subject: [PATCH 36/51] fix: tighten Foundry adapter lifecycle Signed-off-by: Sertac Ozercan --- .dockerignore | 2 + examples/harness/foundry-responses/README.md | 1 - .../harness/foundry-responses/VALIDATION.md | 4 +- examples/harness/foundry-responses/main.go | 89 +---- .../harness/foundry-responses/main_test.go | 314 +++++++++--------- .../harness/foundry-responses/validate.sh | 5 + 6 files changed, 178 insertions(+), 237 deletions(-) diff --git a/.dockerignore b/.dockerignore index c8ebabb7e..af6607012 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,6 +20,8 @@ !examples/harness/ !examples/harness/echo/ !examples/harness/echo/*.go +!examples/harness/foundry-responses/ +!examples/harness/foundry-responses/*.go !*.go **/*_test.go diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 1e218a0ce..6b259df75 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -32,7 +32,6 @@ Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `exa | `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` | Comma-separated static classes the hosted AgentKit deployment has been configured and conformance-tested to request, e.g. `read` or `read,write`. Empty means observed-only. | | `ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT` | Per-request timeout for hosted Responses calls, default `20s`. | | `ORKA_FOUNDRY_RESPONSES_STATE_RETENTION` | How long terminal in-memory turn/session state is retained, default `10m`. | -| `ORKA_FOUNDRY_RESPONSES_MAX_APPROVAL_WAIT` | Maximum time a pending brokered call may wait before a late continuation fails safely, default `30m`. | Exactly one Foundry auth mode (`API_KEY` or `AUTH_BEARER`) must be set. diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 1239b9811..be9255c9f 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -70,14 +70,14 @@ uv run --extra dev pytest -q \ | Duplicate/replayed function calls fail closed | `TestResponsesRepeatedSubmittedFunctionCallFailsTurn` and `TestResponsesMixedRepeatedFunctionCallFailsTurn`. | | Duplicate identical `/continue` is idempotent and conflicting duplicates reject | `TestResponsesAdapterDuplicateContinueIsIdempotentAndConflictsReject`, `TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit`, and `TestResponsesAdapterAlreadySubmittedPendingResultIsNoop`. | | Continuation failures fail closed rather than duplicating hosted progress | `TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost`. | -| Pending approval/tool waits are bounded without understating total brokered duration | `TestResponsesAdapterPendingToolTimesOutWithoutContinuation`, `TestResponsesAdapterPendingTimeoutSkipsSubmittedCall`, and `TestResponsesAdapterBrokeredMaxTurnIsUnknown`. | +| Brokered approval waits remain governed by Orka | The adapter does not expire pending brokered calls after Orka may have executed an approved tool, and `TestResponsesAdapterBrokeredMaxTurnIsUnknown` verifies that it advertises no misleading fixed turn ceiling. | | Restart/state loss fails safely without hosted continuation | `TestResponsesAdapterStateLossContinueFailsSafely`. | | Runtime session continuity | `TestResponsesAdapterRuntimeSessionHeaderReuse` and `TestResponsesAdapterContinuationUsesTurnSessionAfterSessionMapCleanup`. | | Hosted response status handling is fail-closed | `TestResponsesFailureStatusDoesNotCompleteWithPartialText`, `TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest`, `TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText`, and `TestResponsesMissingStatusDoesNotCompleteWithPartialText`. | | Large hosted output and uncertain initial platform failures are safe | `TestResponsesLargeOutputFails` and `TestResponsesInitialPlatformErrorRetainsFailedTurn`. | | No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | | Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | -| Turn admission, cancel identity, and endpoint safety | Focused adapter tests cover bounded consumed-turn tombstones, malformed/mismatched cancel rejection before mutation, and empty-hostname endpoint rejection; `validate.sh` also checks the live-smoke shell preflight. | +| Turn admission, credential retention, cancellation, and endpoint safety | Focused adapter tests cover bounded consumed-turn tombstones, discarding unused resolved turn environment values, cancel/initial-response races, malformed/mismatched cancel rejection before mutation, and empty-hostname endpoint rejection; `validate.sh` also checks Docker-context inclusion and the live-smoke shell preflight. | | Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` tests paginated event aggregation and runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`, including an explicitly truncated event page. | | Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 2204f8c7d..68bd7c907 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -28,7 +28,6 @@ const ( defaultAPIVersion = "v1" defaultRequestTimeout = 20 * time.Second defaultStateRetention = 10 * time.Minute - defaultMaxApprovalWait = 30 * time.Minute defaultReadHeaderTimeout = 5 * time.Second defaultReadTimeout = 30 * time.Second defaultIdleTimeout = 60 * time.Second @@ -49,7 +48,6 @@ const ( envAPIVersion = "ORKA_FOUNDRY_RESPONSES_API_VERSION" envRequestTimeout = "ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT" envStateRetention = "ORKA_FOUNDRY_RESPONSES_STATE_RETENTION" - envMaxApprovalWait = "ORKA_FOUNDRY_RESPONSES_MAX_APPROVAL_WAIT" envContinuationProof = "ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF" envBrokeredToolClasses = "ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES" envAudience = "ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE" @@ -67,7 +65,6 @@ type config struct { apiVersion string requestTimeout time.Duration stateRetention time.Duration - maxApprovalWait time.Duration continuationProof string brokeredToolClasses []harness.BrokeredToolClass configError string @@ -95,7 +92,6 @@ type turnState struct { responseID string foundrySessionID string pendingTools map[string]string - pendingSince map[string]time.Time bufferedResults map[string]harness.ToolCallResult bufferedPayloads map[string]string submittedPayloads map[string]string @@ -198,7 +194,6 @@ func loadConfig() config { apiVersion: firstNonBlank(os.Getenv(envAPIVersion), defaultAPIVersion), requestTimeout: parseDurationEnv(envRequestTimeout, defaultRequestTimeout), stateRetention: parseDurationEnv(envStateRetention, defaultStateRetention), - maxApprovalWait: parseDurationEnv(envMaxApprovalWait, defaultMaxApprovalWait), continuationProof: strings.TrimSpace(os.Getenv(envContinuationProof)), brokeredToolClasses: classes, } @@ -340,6 +335,9 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteError(w, http.StatusBadRequest, err.Error()) return } + // Resolved turn environment values can contain credentials. This adapter does + // not consume them, so discard them before duplicate comparison or retention. + req.Input.Env = nil eventsPath, err := harness.EventStreamPath(req.TurnID) if err != nil { harness.WriteError(w, http.StatusBadRequest, err.Error()) @@ -371,13 +369,12 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { request: req, initializing: true, pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, frameUpdates: make(chan struct{}), } - s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") s.turns[req.TurnID] = turn s.mu.Unlock() @@ -408,10 +405,10 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { return } s.mu.Lock() - s.recordTurnSessionLocked(turn, foundrySessionID) - s.mu.Unlock() - s.handleResponsesResponse(turn, response) - s.mu.Lock() + if !turn.completed { + s.recordTurnSessionLocked(turn, foundrySessionID) + s.handleResponsesResponseLocked(turn, response) + } turn.initializing = false s.mu.Unlock() harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) @@ -608,13 +605,12 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn } s.appendPreparedFrameLocked(turn, frame) delete(turn.pendingTools, result.ToolCallID) - delete(turn.pendingSince, result.ToolCallID) delete(turn.bufferedResults, result.ToolCallID) delete(turn.bufferedPayloads, result.ToolCallID) } s.recordTurnSessionLocked(turn, updatedSessionID) + s.handleResponsesResponseLocked(turn, response) s.mu.Unlock() - s.handleResponsesResponse(turn, response) harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) } @@ -640,7 +636,7 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt defer turn.continueMu.Unlock() s.mu.Lock() if !turn.completed { - s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled", nil) + s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled") turn.completed = true s.scheduleTurnCleanupLocked(turn) } @@ -661,6 +657,10 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt func (s *server) handleResponsesResponse(turn *turnState, response responsesResponse) { s.mu.Lock() defer s.mu.Unlock() + s.handleResponsesResponseLocked(turn, response) +} + +func (s *server) handleResponsesResponseLocked(turn *turnState, response responsesResponse) { if turn.completed { return } @@ -754,11 +754,8 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp } frames = append(frames, frame) } - now := time.Now().UTC() for index, call := range calls { turn.pendingTools[call.callID] = call.name - turn.pendingSince[call.callID] = now - s.schedulePendingToolTimeoutLocked(turn, call.callID) s.appendPreparedFrameLocked(turn, frames[index]) } return @@ -855,7 +852,6 @@ func (s *server) recordContinueResults( if len(turn.pendingTools) == 0 { return nil, fmt.Errorf("no tool calls are pending for this turn") } - now := time.Now().UTC() newResults := map[string]harness.ToolCallResult{} newPayloads := map[string]string{} for _, result := range results { @@ -872,27 +868,6 @@ func (s *server) recordContinueResults( if _, pending := turn.pendingTools[result.ToolCallID]; !pending { return nil, fmt.Errorf("tool result %q is not pending for this turn", result.ToolCallID) } - if pendingAt := turn.pendingSince[result.ToolCallID]; !pendingAt.IsZero() && s.cfg.maxApprovalWait > 0 && - now.Sub(pendingAt) > s.cfg.maxApprovalWait { - turn.completed = true - s.appendFrameLocked( - turn, - harness.FrameTurnFailed, - "approval wait exceeded", - func(f *harness.HarnessEventFrame) { - f.Failed = &harness.TurnFailed{ - Reason: "approval_wait_exceeded", - Message: "maximum brokered tool wait exceeded", - } - f.Error = &harness.ErrorInfo{ - Code: "approval_wait_exceeded", - Message: "maximum brokered tool wait exceeded", - } - }, - ) - s.scheduleTurnCleanupLocked(turn) - return nil, fmt.Errorf("maximum brokered tool wait exceeded") - } if buffered, exists := turn.bufferedPayloads[result.ToolCallID]; exists { if buffered == payload { continue @@ -1380,33 +1355,6 @@ func (s *server) recordTurnSessionLocked(turn *turnState, sessionID string) { s.runtimeSessions[turn.request.RuntimeSessionID] = foundrySession{ID: sessionID, LastSeen: time.Now().UTC()} } -func (s *server) schedulePendingToolTimeoutLocked(turn *turnState, toolCallID string) { - wait := s.cfg.maxApprovalWait - if wait <= 0 { - return - } - turnID := turn.request.TurnID - time.AfterFunc(wait, func() { - s.mu.Lock() - defer s.mu.Unlock() - current := s.turns[turnID] - if current != turn || turn.completed { - return - } - if _, pending := turn.pendingTools[toolCallID]; !pending { - return - } - if _, submitted := turn.submittedPayloads[toolCallID]; submitted { - return - } - s.appendFailedLocked( - turn, - "approval_wait_exceeded", - "maximum brokered tool wait exceeded", - ) - }) -} - func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { if turn.completed { return @@ -1489,13 +1437,8 @@ func (s *server) activeRuntimeSessionsLocked() map[harness.RuntimeSessionID]bool return active } -func (s *server) appendFrameLocked( - turn *turnState, - typ harness.FrameType, - summary string, - mutate func(*harness.HarnessEventFrame), -) { - frame := s.newFrame(turn, int64(len(turn.frames)+1), typ, summary, mutate) +func (s *server) appendFrameLocked(turn *turnState, typ harness.FrameType, summary string) { + frame := s.newFrame(turn, int64(len(turn.frames)+1), typ, summary, nil) s.appendPreparedFrameLocked(turn, frame) } diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index e2cec85c6..c4e92baab 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -314,6 +314,106 @@ func TestResponsesAdapterDuplicateStartDuringInitializationRejected(t *testing.T } } +func TestResponsesAdapterDiscardsUnusedTurnEnvironment(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-env-redaction") + request.Input.Env = []harness.TurnEnvVar{{Name: "SENSITIVE_VALUE", Value: "do-not-retain"}} + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + server.mu.Lock() + turn := server.turns[request.TurnID] + var retainedRequest harness.StartTurnRequest + if turn != nil { + retainedRequest = turn.request + } + server.mu.Unlock() + if turn == nil { + t.Fatal("turn was not retained for duplicate handling") + } + if len(retainedRequest.Input.Env) != 0 { + t.Fatalf("retained turn env = %#v, want discarded credentials", retainedRequest.Input.Env) + } + encoded, err := json.Marshal(retainedRequest) + if err != nil { + t.Fatalf("marshal retained request: %v", err) + } + if bytes.Contains(encoded, []byte("do-not-retain")) { + t.Fatalf("retained request contains discarded environment value: %s", encoded) + } + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("duplicate StartTurn with original environment: %v", err) + } +} + +func TestResponsesAdapterCancelDuringInitialPostDoesNotRetainSession(t *testing.T) { + received := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseFoundry := func() { releaseOnce.Do(func() { close(release) }) } + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + select { + case <-received: + default: + close(received) + } + <-release + w.Header().Set("x-agent-session-id", "cancelled-session") + writeJSON(w, finalResponsesMessage()) + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter, server := newTestResponsesAdapterWithServer(t, endpoint, nil) + t.Cleanup(releaseFoundry) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-cancel-initial") + + startErr := make(chan error, 1) + go func() { + _, err := client.StartTurn(context.Background(), request) + startErr <- err + }() + select { + case <-received: + case <-time.After(time.Second): + t.Fatal("initial Foundry request did not arrive") + } + if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { + t.Fatalf("CancelTurn: %v", err) + } + releaseFoundry() + select { + case err := <-startErr: + if err != nil { + t.Fatalf("StartTurn after cancellation: %v", err) + } + case <-time.After(time.Second): + t.Fatal("StartTurn did not finish after releasing Foundry response") + } + + server.mu.Lock() + turn := server.turns[request.TurnID] + _, sessionRetained := server.runtimeSessions[request.RuntimeSessionID] + foundrySessionID := "" + if turn != nil { + foundrySessionID = turn.foundrySessionID + } + server.mu.Unlock() + if turn == nil || !turn.completed || !hasFrameType(turn.frames, harness.FrameTurnCancelled) { + t.Fatalf("turn = %#v, want retained terminal cancellation", turn) + } + if sessionRetained || foundrySessionID != "" { + t.Fatalf( + "cancelled initial post retained session map=%v turnSession=%q", + sessionRetained, + foundrySessionID, + ) + } +} + func TestResponsesAdapterInitialPostSurvivesControlDisconnect(t *testing.T) { received := make(chan struct{}) release := make(chan struct{}) @@ -757,7 +857,6 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, continuationProof: "proof-for-test", brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) @@ -836,7 +935,6 @@ func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing. foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-partial-reject") @@ -845,7 +943,6 @@ func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing. turn := &turnState{ request: request, pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, @@ -866,7 +963,6 @@ func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-already-submitted") @@ -878,7 +974,6 @@ func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { turn := &turnState{ request: request, pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, bufferedResults: map[string]harness.ToolCallResult{"call-1": result}, bufferedPayloads: map[string]string{"call-1": payload}, submittedPayloads: map[string]string{"call-1": payload}, @@ -1010,19 +1105,17 @@ func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-repeated-call") turn := &turnState{ request: request, pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{"call-1": `{"approved":true}`}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-repeat", Status: "completed", @@ -1047,19 +1140,17 @@ func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-mixed-repeated-call") turn := &turnState{ request: request, pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-repeat", Status: "completed", @@ -1087,53 +1178,12 @@ func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { } } -func TestResponsesAdapterPendingToolTimesOutWithoutContinuation(t *testing.T) { - server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: 10 * time.Millisecond, - brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, - }, &http.Client{Timeout: time.Second}) - turn := &turnState{ - request: brokeredReadRequest("foundry-timeout"), - pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, - } - server.turns[turn.request.TurnID] = turn - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) - server.handleResponsesResponse(turn, responsesResponse{ - ID: "resp-1", - Status: "completed", - Output: []responsesOutput{{ - Type: "function_call", - CallID: "call-1", - Name: "support-ticket-lookup", - Arguments: json.RawMessage(`{"incident":"inc-1"}`), - }}, - }) - time.Sleep(50 * time.Millisecond) - server.mu.Lock() - defer server.mu.Unlock() - failed := findFrame(turn.frames, harness.FrameTurnFailed) - if failed == nil || failed.Failed.Reason != "approval_wait_exceeded" { - t.Fatalf("failed frame = %#v, want approval_wait_exceeded", failed) - } -} - func TestResponsesAdapterAlreadySubmittedPendingResultIsNoop(t *testing.T) { - server := newServer(config{maxApprovalWait: time.Minute}, &http.Client{Timeout: time.Second}) + server := newServer(config{}, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-submitted-noop") turn := &turnState{ request: request, pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, @@ -1154,36 +1204,6 @@ func TestResponsesAdapterAlreadySubmittedPendingResultIsNoop(t *testing.T) { } } -func TestResponsesAdapterPendingTimeoutSkipsSubmittedCall(t *testing.T) { - server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: 10 * time.Millisecond, - brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, - }, &http.Client{Timeout: time.Second}) - turn := &turnState{ - request: brokeredReadRequest("foundry-timeout-submitted"), - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{"call-1": `{"approved":true}`}, - } - server.turns[turn.request.TurnID] = turn - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) - server.schedulePendingToolTimeoutLocked(turn, "call-1") - time.Sleep(50 * time.Millisecond) - server.mu.Lock() - defer server.mu.Unlock() - if failed := findFrame(turn.frames, harness.FrameTurnFailed); failed != nil { - t.Fatalf("failed frame = %#v, submitted call should not time out", failed) - } -} - func TestResponsesAdapterBrokeredMaxTurnIsUnknown(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) s := newServer(config{ @@ -1193,7 +1213,6 @@ func TestResponsesAdapterBrokeredMaxTurnIsUnknown(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: 2 * time.Second, stateRetention: time.Minute, - maxApprovalWait: 30 * time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) @@ -1269,13 +1288,12 @@ func TestResponsesAPIVersionDefaultsToSDKValue(t *testing.T) { func TestResponsesAdapterRejectsConsumedTurnAfterCleanup(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "platform_error"}) server := newServer(config{ - runtimeName: "foundry-responses-test", - adapterBearer: "adapter-auth-value", - endpoint: foundry.endpoint(), - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: 50 * time.Millisecond, - maxApprovalWait: time.Minute, + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.endpoint(), + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: 50 * time.Millisecond, }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(server.handler()) t.Cleanup(adapter.Close) @@ -1480,7 +1498,6 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}, @@ -1489,12 +1506,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { turn := &turnState{ request: request, pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") var functionCall responsesResponse decodeFixtureInto(t, "testdata/golden/02_function_call_response.json", &functionCall) server.handleResponsesResponse(turn, functionCall) @@ -1507,12 +1523,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { finalTurn := &turnState{ request: responsesStartTurnRequest("foundry-final"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started") var finalMessage responsesResponse decodeFixtureInto(t, "testdata/golden/06_final_message_response.json", &finalMessage) server.handleResponsesResponse(finalTurn, finalMessage) @@ -1524,12 +1539,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { multipleTurn := &turnState{ request: request, pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(multipleTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(multipleTurn, harness.FrameTurnStarted, "foundry hosted response started") var multiple responsesResponse decodeFixtureInto(t, "testdata/golden/10_multiple_calls_response.json", &multiple) server.handleResponsesResponse(multipleTurn, multiple) @@ -1548,7 +1562,6 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}, @@ -1564,12 +1577,11 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { turn := &turnState{ request: request, pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") var functionCall responsesResponse decodeFixtureInto( t, @@ -1608,12 +1620,11 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { finalTurn := &turnState{ request: responsesStartTurnRequest("agentkit-final-fixture"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started") var finalMessage responsesResponse decodeFixtureInto(t, "testdata/agentkit-foundry-brokered/final_message_response.json", &finalMessage) server.handleResponsesResponse(finalTurn, finalMessage) @@ -1709,23 +1720,21 @@ func TestResponsesPreservesNumericJSONTokens(t *testing.T) { func TestResponsesLargeOutputFails(t *testing.T) { server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: time.Minute, + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: responsesStartTurnRequest("foundry-large-output"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-large", Status: "completed", @@ -1745,23 +1754,21 @@ func TestResponsesLargeOutputFails(t *testing.T) { func TestResponsesOutputThatExceedsSSEFrameLimitFails(t *testing.T) { server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: time.Minute, + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: responsesStartTurnRequest("foundry-large-frame-output"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-large-frame", Status: "completed", @@ -1787,18 +1794,16 @@ func TestResponsesOversizedToolCallFrameFailsBeforeRequestingTool(t *testing.T) foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: brokeredReadRequest("foundry-large-tool-call-frame"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") arguments, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) if err != nil { t.Fatalf("marshal arguments: %v", err) @@ -1830,7 +1835,6 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-large-tool-result-frame") @@ -1838,12 +1842,11 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) request: request, responseID: "resp-1", pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - pendingSince: map[string]time.Time{"call-1": time.Now().UTC()}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") output, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) if err != nil { t.Fatalf("marshal output: %v", err) @@ -1905,23 +1908,21 @@ func TestResponsesInitialPlatformErrorRetainsFailedTurn(t *testing.T) { //nolint:dupl // Mirrors failure-status regression with a distinct non-terminal status. func TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText(t *testing.T) { server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: time.Minute, + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: responsesStartTurnRequest("foundry-in-progress"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-in-progress", Status: "in_progress", @@ -1939,23 +1940,21 @@ func TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText(t *testing.T) //nolint:dupl // Mirrors non-terminal-status regression with a distinct failed status. func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: time.Minute, + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: responsesStartTurnRequest("foundry-failed-status"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-failed", Status: "failed", @@ -1981,18 +1980,16 @@ func TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest(t *testing foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: brokeredReadRequest("foundry-failed-function-call"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-failed", Status: "incomplete", @@ -2021,23 +2018,21 @@ func TestSanitizeEndpointDoesNotReturnRawMalformedURL(t *testing.T) { func TestResponsesMissingStatusDoesNotCompleteWithPartialText(t *testing.T) { server := newServer(config{ - runtimeName: "test", - adapterBearer: "adapter-auth-value", - endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", - foundryAuth: "foundry-auth-value", - requestTimeout: time.Second, - stateRetention: time.Minute, - maxApprovalWait: time.Minute, + runtimeName: "test", + adapterBearer: "adapter-auth-value", + endpoint: "http://127.0.0.1/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: responsesStartTurnRequest("foundry-missing-status"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-missing-status", Output: []responsesOutput{{Type: "message", Content: "partial text"}}, @@ -2059,18 +2054,16 @@ func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ request: brokeredReadRequest("foundry-missing-id"), pendingTools: map[string]string{}, - pendingSince: map[string]time.Time{}, bufferedResults: map[string]harness.ToolCallResult{}, bufferedPayloads: map[string]string{}, submittedPayloads: map[string]string{}, } - server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started", nil) + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ Status: "completed", Output: []responsesOutput{{ @@ -2312,7 +2305,6 @@ func newTestResponsesAdapterWithServer( apiVersion: "v1", requestTimeout: time.Second, stateRetention: time.Minute, - maxApprovalWait: time.Minute, brokeredToolClasses: append([]harness.BrokeredToolClass(nil), classes...), }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index fbfa37f40..6a79655b5 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -51,6 +51,11 @@ run() { cd "$repo_root" +grep -qx '!examples/harness/foundry-responses/\*.go' .dockerignore || { + echo "Foundry Responses Go sources are not included in the Docker build context" >&2 + exit 1 +} + run go test ./examples/harness/foundry-responses -count=1 run go test \ ./examples/harness/foundry \ From d6ae6ead8ee95b9d0263c4686c7525aae24dbd81 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 13:03:23 -0700 Subject: [PATCH 37/51] fix: bound Foundry result retention Signed-off-by: Sertac Ozercan --- ...ponses-events-terminal-write-terminal.json | 67 ++++ ...responses-events-write-after-terminal.json | 63 ++++ .../verify-foundry-responses.sh | 5 + .../harness/foundry-responses/VALIDATION.md | 2 +- examples/harness/foundry-responses/main.go | 218 +++++++------ .../harness/foundry-responses/main_test.go | 292 ++++++++++++------ .../harness/foundry-responses/validate.sh | 8 + 7 files changed, 459 insertions(+), 196 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json new file mode 100644 index 000000000..6a9a1f993 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json @@ -0,0 +1,67 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 6, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 7, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json new file mode 100644 index 000000000..59c341aa0 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json @@ -0,0 +1,63 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 6, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 60c63a232..49825e677 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -353,6 +353,11 @@ for write_tool, keys in idempotency_by_tool.items(): failures.append(f"multiple write idempotency keys for {write_tool}") if not terminal_events: failures.append("missing terminal completion event") +elif write_exec_events: + latest_write_seq = max(seq(event) for event in write_exec_events) + earliest_terminal_seq = min(seq(event) for event in terminal_events) + if earliest_terminal_seq <= latest_write_seq: + failures.append("terminal completion event does not follow all write executions") if failures: print("Fibey Foundry Responses verification failed:", file=sys.stderr) diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index be9255c9f..788b46f75 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -78,7 +78,7 @@ uv run --extra dev pytest -q \ | No secrets or endpoint URLs in golden fixtures | `TestResponsesGoldenFixturesDoNotContainEndpointsOrSecrets`. | | Existing Orka broker approval/idempotency/write ledger behavior | `go test ./internal/controller -run 'Test.*(AgentRuntime|Harness|Brokered|Runtime)'`, especially brokered write approval, decline, replay, and unresolved-ledger tests in `internal/controller/harness_wrapper_test.go`. | | Turn admission, credential retention, cancellation, and endpoint safety | Focused adapter tests cover bounded consumed-turn tombstones, discarding unused resolved turn environment values, cancel/initial-response races, malformed/mismatched cancel rejection before mutation, and empty-hostname endpoint rejection; `validate.sh` also checks Docker-context inclusion and the live-smoke shell preflight. | -| Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` tests paginated event aggregation and runs `verify-foundry-responses.sh` against pass/fail fixtures under `examples/fibey-custom-agent-demo/testdata/`, including an explicitly truncated event page. | +| Fibey live evidence verifier behavior | `examples/harness/foundry-responses/validate.sh` tests paginated event aggregation and runs `verify-foundry-responses.sh` against pass/fail fixtures, including truncated pages and write execution after a terminal event. | | Kubernetes smoke skeleton is credentials-free | `examples/harness/foundry-responses/kubernetes.example.yaml` uses `REDACTED` placeholders and a read-only advertised class by default. | ## Live gates that cannot be satisfied by local fixtures diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 68bd7c907..5d6957241 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "crypto/sha256" "crypto/subtle" "encoding/json" "errors" @@ -24,18 +25,19 @@ import ( ) const ( - defaultAddr = ":8090" - defaultAPIVersion = "v1" - defaultRequestTimeout = 20 * time.Second - defaultStateRetention = 10 * time.Minute - defaultReadHeaderTimeout = 5 * time.Second - defaultReadTimeout = 30 * time.Second - defaultIdleTimeout = 60 * time.Second - maxFoundryOutputBytes = 1 << 20 - maxFoundryBodyBytes = 4 << 20 - maxConsumedTurnIDs = 1024 - readinessPath = "/v1/ready" - foundryInitialUnknown = "foundry_initial_unknown" + defaultAddr = ":8090" + defaultAPIVersion = "v1" + defaultRequestTimeout = 20 * time.Second + defaultStateRetention = 10 * time.Minute + defaultReadHeaderTimeout = 5 * time.Second + defaultReadTimeout = 30 * time.Second + defaultIdleTimeout = 60 * time.Second + maxFoundryOutputBytes = 1 << 20 + maxFoundryBodyBytes = 4 << 20 + maxConsumedTurnIDs = 1024 + maxFrameSequence int64 = 1<<63 - 1 + readinessPath = "/v1/ready" + foundryInitialUnknown = "foundry_initial_unknown" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -81,24 +83,26 @@ type server struct { runtimeSessions map[harness.RuntimeSessionID]foundrySession } +type toolResultDigest [sha256.Size]byte + type foundrySession struct { ID string LastSeen time.Time } type turnState struct { - request harness.StartTurnRequest - initializing bool - responseID string - foundrySessionID string - pendingTools map[string]string - bufferedResults map[string]harness.ToolCallResult - bufferedPayloads map[string]string - submittedPayloads map[string]string - frames []harness.HarnessEventFrame - completed bool - frameUpdates chan struct{} - continueMu sync.Mutex + request harness.StartTurnRequest + initializing bool + responseID string + foundrySessionID string + pendingTools map[string]string + bufferedResults map[string]harness.ToolCallResult + bufferedDigests map[string]toolResultDigest + submittedDigests map[string]toolResultDigest + frames []harness.HarnessEventFrame + completed bool + frameUpdates chan struct{} + continueMu sync.Mutex } type responsesRequest struct { @@ -152,6 +156,8 @@ type pendingFunctionCall struct { args json.RawMessage } +var sseSizeProbeTime = time.Date(2000, time.January, 1, 0, 0, 0, 123456789, time.UTC) + const responsesEndpointRequirement = "foundry hosted Responses endpoint must use https " + "(http allowed only for loopback), target /responses, and must not include credentials, " + "fragments, or query parameters other than api-version" @@ -366,13 +372,13 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { return } turn := &turnState{ - request: req, - initializing: true, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, - frameUpdates: make(chan struct{}), + request: req, + initializing: true, + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + frameUpdates: make(chan struct{}), } s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") s.turns[req.TurnID] = turn @@ -577,22 +583,8 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn return } for _, result := range resultsToSubmit { - toolName := turn.pendingTools[result.ToolCallID] - if toolName == "" { - toolName = result.ToolCallID - } - frame := s.newFrame( - turn, - int64(len(turn.frames)+1), - harness.FrameToolResultReceived, - "brokered tool result received", - func(f *harness.HarnessEventFrame) { - f.ToolName = toolName - f.ToolCallID = result.ToolCallID - f.Content = result.Output - f.Error = result.Error - }, - ) + toolName := firstNonBlank(turn.pendingTools[result.ToolCallID], result.ToolCallID) + frame := s.newToolResultFrame(turn, int64(len(turn.frames)+1), toolName, result) if !harnessFrameFitsSSE(frame) { s.appendFailedLocked( turn, @@ -606,7 +598,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn s.appendPreparedFrameLocked(turn, frame) delete(turn.pendingTools, result.ToolCallID) delete(turn.bufferedResults, result.ToolCallID) - delete(turn.bufferedPayloads, result.ToolCallID) + delete(turn.bufferedDigests, result.ToolCallID) } s.recordTurnSessionLocked(turn, updatedSessionID) s.handleResponsesResponseLocked(turn, response) @@ -636,6 +628,7 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt defer turn.continueMu.Unlock() s.mu.Lock() if !turn.completed { + s.clearBufferedToolResultsLocked(turn) s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled") turn.completed = true s.scheduleTurnCleanupLocked(turn) @@ -713,7 +706,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons return } seenInResponse[call.callID] = struct{}{} - if _, submitted := turn.submittedPayloads[call.callID]; submitted { + if _, submitted := turn.submittedDigests[call.callID]; submitted { s.appendFailedLocked( turn, "foundry_repeated_function_call", @@ -783,6 +776,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons return } s.appendPreparedFrameLocked(turn, completedFrame) + s.clearBufferedToolResultsLocked(turn) turn.completed = true s.scheduleTurnCleanupLocked(turn) } @@ -853,14 +847,15 @@ func (s *server) recordContinueResults( return nil, fmt.Errorf("no tool calls are pending for this turn") } newResults := map[string]harness.ToolCallResult{} - newPayloads := map[string]string{} + newDigests := map[string]toolResultDigest{} for _, result := range results { payload, err := canonicalToolResultOutput(result) if err != nil { return nil, err } - if submitted, done := turn.submittedPayloads[result.ToolCallID]; done { - if submitted == payload { + digest := digestToolResultPayload(payload) + if submitted, done := turn.submittedDigests[result.ToolCallID]; done { + if submitted == digest { continue } return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) @@ -868,28 +863,54 @@ func (s *server) recordContinueResults( if _, pending := turn.pendingTools[result.ToolCallID]; !pending { return nil, fmt.Errorf("tool result %q is not pending for this turn", result.ToolCallID) } - if buffered, exists := turn.bufferedPayloads[result.ToolCallID]; exists { - if buffered == payload { + if buffered, exists := turn.bufferedDigests[result.ToolCallID]; exists { + if buffered == digest { continue } return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) } - if buffered, exists := newPayloads[result.ToolCallID]; exists { - if buffered == payload { + if buffered, exists := newDigests[result.ToolCallID]; exists { + if buffered == digest { continue } return nil, fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) } newResults[result.ToolCallID] = result - newPayloads[result.ToolCallID] = payload + newDigests[result.ToolCallID] = digest } + + unsubmittedIDs := make([]string, 0, len(turn.pendingTools)) + for id := range turn.pendingTools { + if _, submitted := turn.submittedDigests[id]; !submitted { + unsubmittedIDs = append(unsubmittedIDs, id) + } + } + sort.Strings(unsubmittedIDs) + baseSeq := int64(len(turn.frames) + 1) + for index, id := range unsubmittedIDs { + result, isNew := newResults[id] + if !isNew { + continue + } + toolName := firstNonBlank(turn.pendingTools[id], id) + frame := s.newToolResultFrame(turn, baseSeq+int64(index), toolName, result) + if !toolResultFrameFitsSSE(frame) { + s.appendFailedLocked( + turn, + "brokered_tool_result_frame_too_large", + "brokered tool result exceeded the harness SSE frame limit", + ) + return nil, fmt.Errorf("brokered tool result %q exceeds harness SSE frame limit", id) + } + } + for id, result := range newResults { turn.bufferedResults[id] = result - turn.bufferedPayloads[id] = newPayloads[id] + turn.bufferedDigests[id] = newDigests[id] } readyCount := 0 for id := range turn.pendingTools { - if _, submitted := turn.submittedPayloads[id]; submitted { + if _, submitted := turn.submittedDigests[id]; submitted { readyCount++ continue } @@ -900,47 +921,15 @@ func (s *server) recordContinueResults( if readyCount < len(turn.pendingTools) { return nil, nil } - ids := make([]string, 0, len(turn.pendingTools)) - for id := range turn.pendingTools { - ids = append(ids, id) - } - sort.Strings(ids) - toSubmit := make([]harness.ToolCallResult, 0, len(ids)) - for _, id := range ids { - if _, submitted := turn.submittedPayloads[id]; submitted { - continue - } + toSubmit := make([]harness.ToolCallResult, 0, len(unsubmittedIDs)) + for _, id := range unsubmittedIDs { toSubmit = append(toSubmit, turn.bufferedResults[id]) } if len(toSubmit) == 0 { return nil, nil } - baseSeq := int64(len(turn.frames) + 1) - for index, result := range toSubmit { - toolName := firstNonBlank(turn.pendingTools[result.ToolCallID], result.ToolCallID) - frame := s.newFrame( - turn, - baseSeq+int64(index), - harness.FrameToolResultReceived, - "brokered tool result received", - func(f *harness.HarnessEventFrame) { - f.ToolName = toolName - f.ToolCallID = result.ToolCallID - f.Content = result.Output - f.Error = result.Error - }, - ) - if !harnessFrameFitsSSE(frame) { - s.appendFailedLocked( - turn, - "brokered_tool_result_frame_too_large", - "brokered tool result exceeded the harness SSE frame limit", - ) - return nil, fmt.Errorf("brokered tool result %q exceeds harness SSE frame limit", result.ToolCallID) - } - } for _, result := range toSubmit { - turn.submittedPayloads[result.ToolCallID] = turn.bufferedPayloads[result.ToolCallID] + turn.submittedDigests[result.ToolCallID] = turn.bufferedDigests[result.ToolCallID] } return toSubmit, nil } @@ -956,11 +945,11 @@ func (s *server) ensureTerminalContinueIsDuplicate(turn *turnState, results []ha if err != nil { return err } - submitted, done := turn.submittedPayloads[result.ToolCallID] + submitted, done := turn.submittedDigests[result.ToolCallID] if !done { return fmt.Errorf("terminal turn cannot accept new tool result %q", result.ToolCallID) } - if submitted != payload { + if submitted != digestToolResultPayload(payload) { return fmt.Errorf("conflicting duplicate result for tool call %q", result.ToolCallID) } } @@ -1333,6 +1322,35 @@ func outputItemText(item responsesOutput) string { return "" } +func digestToolResultPayload(payload string) toolResultDigest { + return sha256.Sum256([]byte(payload)) +} + +func (s *server) newToolResultFrame( + turn *turnState, + seq int64, + toolName string, + result harness.ToolCallResult, +) harness.HarnessEventFrame { + return s.newFrame( + turn, + seq, + harness.FrameToolResultReceived, + "brokered tool result received", + func(f *harness.HarnessEventFrame) { + f.ToolName = toolName + f.ToolCallID = result.ToolCallID + f.Content = result.Output + f.Error = result.Error + }, + ) +} + +func (s *server) clearBufferedToolResultsLocked(turn *turnState) { + clear(turn.bufferedResults) + clear(turn.bufferedDigests) +} + func isCompletionStatus(status string) bool { return strings.EqualFold(strings.TrimSpace(status), "completed") } @@ -1359,6 +1377,7 @@ func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { if turn.completed { return } + s.clearBufferedToolResultsLocked(turn) failedFrame := s.newFrame( turn, int64(len(turn.frames)+1), @@ -1466,6 +1485,13 @@ func (s *server) newFrame( return frame } +func toolResultFrameFitsSSE(frame harness.HarnessEventFrame) bool { + // RFC3339Nano omits trailing zeros. Probe with full nanosecond precision so + // the preflight is conservative for the exact sequence the frame will use. + frame.CreatedAt = sseSizeProbeTime + return harnessFrameFitsSSE(frame) +} + func harnessFrameFitsSSE(frame harness.HarnessEventFrame) bool { payload, err := json.Marshal(frame) return err == nil && len("data: ")+len(payload) < harness.MaxSSEFrameBytes diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index c4e92baab..92b8ccd34 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -941,17 +941,17 @@ func TestResponsesAdapterRejectedContinueDoesNotBufferPartialResults(t *testing. valid := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) unknown := toolResultForRequest(request, "call-missing", true, json.RawMessage(`{"success":true}`), nil) turn := &turnState{ - request: request, - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{valid, unknown}); err == nil { t.Fatalf("recordContinueResults succeeded, want unknown tool result error") } - if len(turn.bufferedResults) != 0 || len(turn.bufferedPayloads) != 0 { - t.Fatalf("buffered state = %#v/%#v, want no partial buffering", turn.bufferedResults, turn.bufferedPayloads) + if len(turn.bufferedResults) != 0 || len(turn.bufferedDigests) != 0 { + t.Fatalf("buffered state = %#v/%#v, want no partial buffering", turn.bufferedResults, turn.bufferedDigests) } } @@ -972,11 +972,11 @@ func TestResponsesAdapterAlreadySubmittedContinueDoesNotResubmit(t *testing.T) { t.Fatalf("canonicalToolResultOutput: %v", err) } turn := &turnState{ - request: request, - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - bufferedResults: map[string]harness.ToolCallResult{"call-1": result}, - bufferedPayloads: map[string]string{"call-1": payload}, - submittedPayloads: map[string]string{"call-1": payload}, + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{"call-1": result}, + bufferedDigests: map[string]toolResultDigest{"call-1": digestToolResultPayload(payload)}, + submittedDigests: map[string]toolResultDigest{"call-1": digestToolResultPayload(payload)}, } toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}) if err != nil { @@ -1109,11 +1109,11 @@ func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-repeated-call") turn := &turnState{ - request: request, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{"call-1": `{"approved":true}`}, + request: request, + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{"call-1": digestToolResultPayload(`{"approved":true}`)}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1144,11 +1144,11 @@ func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-mixed-repeated-call") turn := &turnState{ - request: request, - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1182,18 +1182,18 @@ func TestResponsesAdapterAlreadySubmittedPendingResultIsNoop(t *testing.T) { server := newServer(config{}, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-submitted-noop") turn := &turnState{ - request: request, - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } result := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) payload, err := canonicalToolResultOutput(result) if err != nil { t.Fatalf("canonicalToolResultOutput: %v", err) } - turn.submittedPayloads["call-1"] = payload + turn.submittedDigests["call-1"] = digestToolResultPayload(payload) toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}) if err != nil { @@ -1504,11 +1504,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { ) request := brokeredReadRequest("foundry-brokered") turn := &turnState{ - request: request, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") var functionCall responsesResponse @@ -1521,11 +1521,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { assertJSONFileEqual(t, "testdata/golden/03_tool_call_requested_frame.json", scrubFrameForGolden(*requested)) finalTurn := &turnState{ - request: responsesStartTurnRequest("foundry-final"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-final"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started") var finalMessage responsesResponse @@ -1537,11 +1537,11 @@ func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { } multipleTurn := &turnState{ - request: request, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(multipleTurn, harness.FrameTurnStarted, "foundry hosted response started") var multiple responsesResponse @@ -1575,11 +1575,11 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { Parameters: json.RawMessage(`{"type":"object","properties":{"probe":{"type":"boolean"}}}`), }} turn := &turnState{ - request: request, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") var functionCall responsesResponse @@ -1618,11 +1618,11 @@ func TestResponsesConsumesAgentKitBrokeredFixtures(t *testing.T) { }) finalTurn := &turnState{ - request: responsesStartTurnRequest("agentkit-final-fixture"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("agentkit-final-fixture"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(finalTurn, harness.FrameTurnStarted, "foundry hosted response started") var finalMessage responsesResponse @@ -1728,11 +1728,11 @@ func TestResponsesLargeOutputFails(t *testing.T) { stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: responsesStartTurnRequest("foundry-large-output"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-large-output"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1762,11 +1762,11 @@ func TestResponsesOutputThatExceedsSSEFrameLimitFails(t *testing.T) { stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: responsesStartTurnRequest("foundry-large-frame-output"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-large-frame-output"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1797,11 +1797,11 @@ func TestResponsesOversizedToolCallFrameFailsBeforeRequestingTool(t *testing.T) brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: brokeredReadRequest("foundry-large-tool-call-frame"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: brokeredReadRequest("foundry-large-tool-call-frame"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") arguments, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) @@ -1827,6 +1827,94 @@ func TestResponsesOversizedToolCallFrameFailsBeforeRequestingTool(t *testing.T) } } +func TestResponsesToolResultPreflightUsesExactEventualSequence(t *testing.T) { + server := newServer(config{}, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-tool-result-boundary") + turn := &turnState{ + request: request, + responseID: "resp-1", + pendingTools: map[string]string{ + "call-1": "support-ticket-lookup", + "call-2": "support-ticket-lookup", + }, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + for range 8 { + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + } + emptyResult := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"payload":""}`), nil) + seqNineFrame := server.newToolResultFrame(turn, 9, "support-ticket-lookup", emptyResult) + seqNineFrame.CreatedAt = sseSizeProbeTime + seqTenFrame := server.newToolResultFrame(turn, 10, "support-ticket-lookup", emptyResult) + seqTenFrame.CreatedAt = sseSizeProbeTime + seqNineJSON, err := json.Marshal(seqNineFrame) + if err != nil { + t.Fatalf("marshal sequence-nine frame: %v", err) + } + seqTenJSON, err := json.Marshal(seqTenFrame) + if err != nil { + t.Fatalf("marshal sequence-ten frame: %v", err) + } + if len(seqTenJSON) <= len(seqNineJSON) { + t.Fatalf("frame sizes seq10=%d seq9=%d, want decimal-boundary overhead", len(seqTenJSON), len(seqNineJSON)) + } + payloadSize := harness.MaxSSEFrameBytes - len("data: ") - len(seqNineJSON) - 1 + if payloadSize <= 0 { + t.Fatalf("payload boundary = %d, want positive", payloadSize) + } + output := json.RawMessage(`{"payload":"` + strings.Repeat("x", payloadSize) + `"}`) + result := toolResultForRequest(request, "call-1", true, output, nil) + if !toolResultFrameFitsSSE(server.newToolResultFrame(turn, 9, "support-ticket-lookup", result)) { + t.Fatal("eventual sequence-nine frame should fit") + } + if toolResultFrameFitsSSE(server.newToolResultFrame(turn, 10, "support-ticket-lookup", result)) { + t.Fatal("sequence-ten probe unexpectedly fits boundary frame") + } + toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}) + if err != nil { + t.Fatalf("recordContinueResults rejected exact eventual sequence: %v", err) + } + if len(toSubmit) != 0 || len(turn.bufferedResults) != 1 { + t.Fatalf("toSubmit=%#v buffered=%#v, want one accepted partial result", toSubmit, turn.bufferedResults) + } +} + +func TestResponsesOversizedBatchValidatesAllResultsBeforeFailure(t *testing.T) { + server := newServer(config{}, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-oversized-batch") + turn := &turnState{ + request: request, + pendingTools: map[string]string{ + "call-1": "support-ticket-lookup", + }, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + oversizedOutput, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) + if err != nil { + t.Fatalf("marshal oversized output: %v", err) + } + oversized := toolResultForRequest(request, "call-1", true, oversizedOutput, nil) + unknown := toolResultForRequest(request, "call-missing", true, json.RawMessage(`{"success":true}`), nil) + + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{oversized, unknown}); err == nil || + !strings.Contains(err.Error(), "not pending") { + t.Fatalf("recordContinueResults error = %v, want structural rejection", err) + } + if turn.completed || len(turn.bufferedResults) != 0 || len(turn.bufferedDigests) != 0 { + t.Fatalf( + "invalid batch mutated turn: completed=%v buffers=%#v/%#v", + turn.completed, + turn.bufferedResults, + turn.bufferedDigests, + ) + } +} + func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -1839,12 +1927,12 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) }, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-large-tool-result-frame") turn := &turnState{ - request: request, - responseID: "resp-1", - pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: request, + responseID: "resp-1", + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") output, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) @@ -1862,6 +1950,12 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) if failed == nil || failed.Failed.Reason != "brokered_tool_result_frame_too_large" { t.Fatalf("failed frame = %#v, want brokered_tool_result_frame_too_large", failed) } + if len(turn.bufferedResults) != 0 || len(turn.bufferedDigests) != 0 { + t.Fatalf("oversized result retained buffered state: %#v/%#v", turn.bufferedResults, turn.bufferedDigests) + } + if len(turn.submittedDigests) != 0 { + t.Fatalf("oversized result retained submitted digests: %#v", turn.submittedDigests) + } } func TestResponsesInitialPlatformErrorRetainsFailedTurn(t *testing.T) { @@ -1916,11 +2010,11 @@ func TestResponsesNonTerminalStatusDoesNotCompleteWithPartialText(t *testing.T) stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: responsesStartTurnRequest("foundry-in-progress"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-in-progress"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1948,11 +2042,11 @@ func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: responsesStartTurnRequest("foundry-failed-status"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-failed-status"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -1983,11 +2077,11 @@ func TestResponsesFailureStatusWithFunctionCallFailsBeforeToolRequest(t *testing brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: brokeredReadRequest("foundry-failed-function-call"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: brokeredReadRequest("foundry-failed-function-call"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -2026,11 +2120,11 @@ func TestResponsesMissingStatusDoesNotCompleteWithPartialText(t *testing.T) { stateRetention: time.Minute, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: responsesStartTurnRequest("foundry-missing-status"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: responsesStartTurnRequest("foundry-missing-status"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ @@ -2057,11 +2151,11 @@ func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) turn := &turnState{ - request: brokeredReadRequest("foundry-missing-id"), - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedPayloads: map[string]string{}, - submittedPayloads: map[string]string{}, + request: brokeredReadRequest("foundry-missing-id"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 6a79655b5..b56ac68c6 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -159,6 +159,14 @@ expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json \ "afterSeq must be 0" \ "tail-event-page" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json \ + "terminal completion event does not follow all write executions" \ + "write-after-terminal" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json \ + "terminal completion event does not follow all write executions" \ + "terminal-write-terminal" if [[ "$run_full" == "1" ]]; then run make test From acd8395474f70d958ee8f53af97b8177aa2450ef Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 13:42:49 -0700 Subject: [PATCH 38/51] fix: align Foundry smoke lifecycle Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 7 ++- .../harness/foundry-responses/VALIDATION.md | 3 +- .../foundry-responses/kubernetes.example.yaml | 3 +- .../harness/foundry-responses/live-smoke.sh | 45 ++++++++++----- examples/harness/foundry-responses/main.go | 41 +++++++------- .../harness/foundry-responses/main_test.go | 36 ++++++++++++ .../harness/foundry-responses/validate.sh | 55 +++++++++++++++++++ 7 files changed, 150 insertions(+), 40 deletions(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 6b259df75..965aef036 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -105,13 +105,13 @@ The adapter captures Foundry session identifiers from the hosted response body ( ## Local build ```bash -docker build -t ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest -f examples/harness/foundry-responses/Dockerfile . +docker build -t foundry-responses-harness-adapter:local -f examples/harness/foundry-responses/Dockerfile . ``` ## Kubernetes smoke skeleton -`kubernetes.example.yaml` contains a credentials-free Deployment, Service, Secret placeholders, and matching `AgentRuntime` facade for a read-profile hosted Responses smoke. Replace the `REDACTED` values through your secret-management flow, set the hosted Responses endpoint or project/agent-name pair, ensure the hosted agent statically exposes the probe-only `conformance_read` schema, and keep `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` narrowed to classes whose live AgentRuntime conformance passed. +`kubernetes.example.yaml` contains a credentials-free Deployment, Service, Secret placeholders, and matching `AgentRuntime` facade for a read-profile hosted Responses smoke. It references the local image tag built above; load that image into your local cluster or replace it with an explicitly published image. Replace the `REDACTED` values through your secret-management flow, set the hosted Responses endpoint or project/agent-name pair, ensure the hosted agent statically exposes the probe-only `conformance_read` schema, and keep `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` narrowed to classes whose live AgentRuntime conformance passed. ```bash kubectl apply -f examples/harness/foundry-responses/kubernetes.example.yaml @@ -136,7 +136,8 @@ For the live Foundry hosted AgentKit smoke gate, first run the credentials-safe ```bash examples/harness/foundry-responses/live-smoke.sh -examples/harness/foundry-responses/live-smoke.sh --apply --wait +ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE= \ + examples/harness/foundry-responses/live-smoke.sh --apply --wait # After the live task completes, capture redacted evidence. examples/harness/foundry-responses/live-evidence.sh \ diff --git a/examples/harness/foundry-responses/VALIDATION.md b/examples/harness/foundry-responses/VALIDATION.md index 788b46f75..2a16e9d59 100644 --- a/examples/harness/foundry-responses/VALIDATION.md +++ b/examples/harness/foundry-responses/VALIDATION.md @@ -87,7 +87,8 @@ Use the credentials-safe live smoke helper as the first live preflight/deploy st ```bash examples/harness/foundry-responses/live-smoke.sh -examples/harness/foundry-responses/live-smoke.sh --apply --wait +ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE= \ + examples/harness/foundry-responses/live-smoke.sh --apply --wait ``` The helper validates required environment without printing secret values and can diff --git a/examples/harness/foundry-responses/kubernetes.example.yaml b/examples/harness/foundry-responses/kubernetes.example.yaml index 5b59ffeb8..1eca993b5 100644 --- a/examples/harness/foundry-responses/kubernetes.example.yaml +++ b/examples/harness/foundry-responses/kubernetes.example.yaml @@ -39,7 +39,8 @@ spec: spec: containers: - name: adapter - image: ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest + # Build and load/publish examples/harness/foundry-responses/Dockerfile first. + image: foundry-responses-harness-adapter:local imagePullPolicy: IfNotPresent env: - name: ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 13e64c99c..761ac20f8 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -22,12 +22,13 @@ Required environment for preflight/apply: Optional environment: ORKA_FOUNDRY_RESPONSES_NAMESPACE default: foundry-responses-smoke ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME default: sample-foundry-responses-runtime - ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE default: ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest + ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE required with --apply; build/publish this PR's Dockerfile first ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN generated if absent for this run ORKA_FOUNDRY_RESPONSES_API_VERSION default: v1 ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES default: read - Every advertised class requires the hosted agent to statically expose the - matching probe-only conformance_read/conformance_write schema. + Set explicitly to an empty string for observed-only mode. Every advertised + class requires the hosted agent to statically expose the matching probe-only + conformance_read/conformance_write schema. ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF optional The script never prints secret values. Do not run with shell tracing (set -x). @@ -72,9 +73,9 @@ fi runtime_name="${ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME:-sample-foundry-responses-runtime}" service_url="http://${runtime_name}.${namespace}.svc.cluster.local:8080" -adapter_image="${ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE:-ghcr.io/orka-agents/orka/foundry-responses-harness-adapter:latest}" +adapter_image="${ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE:-}" api_version="${ORKA_FOUNDRY_RESPONSES_API_VERSION:-v1}" -brokered_classes="${ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES:-read}" +brokered_classes="${ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES-read}" endpoint="${ORKA_FOUNDRY_RESPONSES_ENDPOINT:-}" project_endpoint="${ORKA_FOUNDRY_RESPONSES_PROJECT_ENDPOINT:-}" agent_name="${ORKA_FOUNDRY_RESPONSES_AGENT_NAME:-}" @@ -201,12 +202,18 @@ preflight() { fail "set one Foundry auth value: ORKA_FOUNDRY_RESPONSES_API_KEY or ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" fi - IFS=',' read -r -a classes <<<"$brokered_classes" - for class in "${classes[@]}"; do - class="${class//[[:space:]]/}" - [[ "$class" == "read" || "$class" == "write" ]] || fail "unsupported brokered class '$class' (expected read/write)" - done + if [[ -n "${brokered_classes//[[:space:],]/}" ]]; then + IFS=',' read -r -a classes <<<"$brokered_classes" + for class in "${classes[@]}"; do + class="${class//[[:space:]]/}" + [[ -z "$class" ]] && continue + [[ "$class" == "read" || "$class" == "write" ]] || fail "unsupported brokered class '$class' (expected read/write)" + done + fi + if [[ "$apply" == "1" && -z "$adapter_image" ]]; then + fail "ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE is required with --apply" + fi if [[ "$apply" == "1" || "$wait_ready" == "1" ]]; then require_cmd kubectl fi @@ -281,6 +288,16 @@ emit_runtime_yaml() { auth_env_name="ORKA_FOUNDRY_RESPONSES_AUTH_BEARER" auth_key_name="foundry-bearer" fi + local tool_modes_yaml=" - observed" + local brokered_classes_yaml="" + local supports_continuation="false" + local normalized_classes + normalized_classes="$(printf '%s' "$brokered_classes" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;/^$/d')" + if [[ -n "$normalized_classes" ]]; then + tool_modes_yaml+=$'\n - brokered' + brokered_classes_yaml=$(printf ' brokeredToolClasses:\n%s' "$(printf '%s\n' "$normalized_classes" | sed 's/^/ - /')") + supports_continuation="true" + fi cat < 0 { if !responseIDPresent { @@ -693,7 +693,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_response_id_missing", "hosted response returned a function_call without an id needed for continuation", ) - return + return false } seenInResponse := map[string]struct{}{} for _, call := range calls { @@ -703,7 +703,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated a function call id", ) - return + return false } seenInResponse[call.callID] = struct{}{} if _, submitted := turn.submittedDigests[call.callID]; submitted { @@ -712,7 +712,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated an already-submitted function call", ) - return + return false } if _, pending := turn.pendingTools[call.callID]; pending { s.appendFailedLocked( @@ -720,7 +720,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated an already-pending function call", ) - return + return false } } frames := make([]harness.HarnessEventFrame, 0, len(calls)) @@ -743,7 +743,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_tool_call_frame_too_large", "hosted function call exceeded the harness SSE frame limit", ) - return + return false } frames = append(frames, frame) } @@ -751,12 +751,12 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons turn.pendingTools[call.callID] = call.name s.appendPreparedFrameLocked(turn, frames[index]) } - return + return true } result := responsesMessageText(response.Output) if len([]byte(result)) > maxFoundryOutputBytes { s.appendFailedLocked(turn, "foundry_output_too_large", "foundry completion exceeded advertised output limit") - return + return false } completedFrame := s.newFrame( turn, @@ -773,12 +773,13 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_output_frame_too_large", "foundry completion exceeded the harness SSE frame limit", ) - return + return false } s.appendPreparedFrameLocked(turn, completedFrame) s.clearBufferedToolResultsLocked(turn) turn.completed = true s.scheduleTurnCleanupLocked(turn) + return true } func (s *server) extractFunctionCalls( diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 92b8ccd34..90f7b2589 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -349,6 +349,36 @@ func TestResponsesAdapterDiscardsUnusedTurnEnvironment(t *testing.T) { } } +func TestResponsesAdapterRejectedResponseDoesNotRetainSession(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "failed_with_session"}) + adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + first := responsesStartTurnRequest("foundry-rejected-session-first") + + if _, err := client.StartTurn(context.Background(), first); err != nil { + t.Fatalf("first StartTurn: %v", err) + } + frames := streamCurrentFrames(t, client, first.TurnID) + if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || failed.Failed.Reason != "foundry_failed" { + t.Fatalf("failed frame = %#v, want foundry_failed", failed) + } + server.mu.Lock() + _, retained := server.runtimeSessions[first.RuntimeSessionID] + server.mu.Unlock() + if retained { + t.Fatal("rejected hosted response retained runtime session") + } + + second := responsesStartTurnRequest("foundry-rejected-session-second") + second.RuntimeSessionID = first.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), second); err != nil { + t.Fatalf("second StartTurn: %v", err) + } + if got := requestMap(t, foundry.requestBody(1))["agent_session_id"]; got != nil { + t.Fatalf("second request agent_session_id = %#v, want no rejected session reuse", got) + } +} + func TestResponsesAdapterCancelDuringInitialPostDoesNotRetainSession(t *testing.T) { received := make(chan struct{}) release := make(chan struct{}) @@ -2286,6 +2316,12 @@ func newFakeResponses(t *testing.T, cfg fakeResponsesConfig) *fakeResponses { writeJSON(w, finalResponsesMessage()) case "function_call": writeJSON(w, functionCallResponse(f.cfg.toolName)) + case "failed_with_session": + writeJSON(w, map[string]any{ + "id": "resp-failed", + "agent_session_id": fakeSessionID, + "status": "failed", + }) case "malformed_arguments": writeJSON( w, diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index b56ac68c6..fd2dbb5c1 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -113,6 +113,61 @@ if [[ "$live_smoke_code" == "0" ]] || ! grep -q "safe /responses URL" "$live_smo fi rm -f "$live_smoke_err" +missing_image_err="$(mktemp)" +set +e +ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE="" \ + examples/harness/foundry-responses/live-smoke.sh --apply >/dev/null 2>"$missing_image_err" +missing_image_code=$? +set -e +if [[ "$missing_image_code" == "0" ]] || ! grep -q "ADAPTER_IMAGE is required" "$missing_image_err"; then + cat "$missing_image_err" >&2 + rm -f "$missing_image_err" + echo "expected live smoke apply without an explicit image to fail" >&2 + exit 1 +fi +rm -f "$missing_image_err" + +smoke_tmp="$(mktemp -d)" +smoke_capture="$smoke_tmp/rendered.yaml" +cat >"$smoke_tmp/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +if [[ " $* " == *" apply "* ]]; then + cat >>"$CAPTURE_FILE" + printf '\n---\n' >>"$CAPTURE_FILE" + exit 0 +fi +if [[ "${1:-}" == "get" && "${2:-}" == "namespace" ]]; then + exit 0 +fi +if [[ " $* " == *" get deployment/"* ]]; then + exit 1 +fi +exit 0 +SH +chmod +x "$smoke_tmp/kubectl" +PATH="$smoke_tmp:$PATH" \ + CAPTURE_FILE="$smoke_capture" \ + ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + ORKA_FOUNDRY_RESPONSES_ADAPTER_BEARER_TOKEN="adapter-placeholder" \ + ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE="example.invalid/foundry-adapter:test" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES="" \ + examples/harness/foundry-responses/live-smoke.sh --apply >/dev/null 2>"$smoke_tmp/stderr" +if grep -q '^ - brokered$' "$smoke_capture" || \ + grep -q '^ brokeredToolClasses:' "$smoke_capture" || \ + ! grep -q '^ supportsContinuation: false$' "$smoke_capture"; then + cat "$smoke_capture" >&2 + rm -rf "$smoke_tmp" + echo "observed-only live smoke rendered incompatible brokered capabilities" >&2 + exit 1 +fi +rm -rf "$smoke_tmp" + run examples/fibey-custom-agent-demo/verify-foundry-responses.sh \ --json examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json expect_verifier_failure \ From 60dbd16b0993eb6fe3b5ddd838ac8987d60de21a Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 14:31:34 -0700 Subject: [PATCH 39/51] fix: enforce Foundry terminal ordering Signed-off-by: Sertac Ozercan --- ...ponses-events-decision-before-request.json | 4 + ...undry-responses-events-declined-write.json | 4 + ...ndry-responses-events-duplicate-write.json | 4 + ...onses-events-mismatched-write-request.json | 4 + ...nses-events-missing-approval-decision.json | 4 + ...-responses-events-missing-approval-id.json | 4 + ...y-responses-events-missing-write-exec.json | 4 + ...onses-events-overlapping-write-marker.json | 4 + ...-responses-events-partial-idempotency.json | 32 +++++- .../foundry-responses-events-pass.json | 4 + ...nts-success-before-runtime-completion.json | 67 +++++++++++ .../foundry-responses-events-tail-page.json | 6 +- .../foundry-responses-events-task-failed.json | 67 +++++++++++ ...ponses-events-terminal-write-terminal.json | 4 + ...undry-responses-events-truncated-page.json | 6 +- ...responses-events-write-after-terminal.json | 4 + .../verify-foundry-responses.sh | 13 +++ examples/harness/foundry-responses/main.go | 18 +++ .../harness/foundry-responses/main_test.go | 107 +++++++++++++++++- .../harness/foundry-responses/validate.sh | 8 ++ 20 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-success-before-runtime-completion.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-task-failed.json diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json index 4e2309522..2be354064 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-decision-before-request.json @@ -57,6 +57,10 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json index 9cfc14fad..cbd68639d 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-declined-write.json @@ -57,6 +57,10 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json index 319af3c23..e3b643210 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-duplicate-write.json @@ -69,6 +69,10 @@ { "seq": 7, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 8, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json index c248385b4..a7df08693 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-mismatched-write-request.json @@ -57,6 +57,10 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json index 6eb246fd8..f10bd3a4f 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-decision.json @@ -48,6 +48,10 @@ { "seq": 5, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 6, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json index 9ee50427c..09b8d0bc3 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-approval-id.json @@ -50,6 +50,10 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json index e6e710b44..b7def3b42 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-missing-write-exec.json @@ -45,6 +45,10 @@ { "seq": 5, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 6, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json index 65ae7d334..497d5114e 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-overlapping-write-marker.json @@ -47,6 +47,10 @@ { "seq": 5, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 6, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json index 243e339de..bc431667b 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json @@ -5,14 +5,22 @@ "eventType": "ToolCallStarted", "toolName": "check-network-telemetry", "toolCallID": "read-call-1", - "content": {"harness": {"frameType": "ToolCallRequested"}} + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + } }, { "seq": 2, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", "toolCallID": "dispatch-call-1", - "content": {"harness": {"frameType": "ToolCallRequested"}} + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + } }, { "seq": 3, @@ -29,7 +37,10 @@ "seq": 4, "eventType": "ApprovalApproved", "toolCallID": "approval-dispatch", - "content": {"approvalID": "approval-dispatch", "decision": "approve"} + "content": { + "approvalID": "approval-dispatch", + "decision": "approve" + } }, { "seq": 5, @@ -48,7 +59,11 @@ "eventType": "ToolCallStarted", "toolName": "escalate-incident", "toolCallID": "escalate-call-1", - "content": {"harness": {"frameType": "ToolCallRequested"}} + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + } }, { "seq": 7, @@ -65,7 +80,10 @@ "seq": 8, "eventType": "ApprovalApproved", "toolCallID": "approval-escalate", - "content": {"approvalID": "approval-escalate", "decision": "approve"} + "content": { + "approvalID": "approval-escalate", + "decision": "approve" + } }, { "seq": 9, @@ -81,6 +99,10 @@ { "seq": 10, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 11, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json index e403af9ed..7231ad15f 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json @@ -58,6 +58,10 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-success-before-runtime-completion.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-success-before-runtime-completion.json new file mode 100644 index 000000000..fecf9adbf --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-success-before-runtime-completion.json @@ -0,0 +1,67 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "TaskSucceeded" + }, + { + "seq": 7, + "eventType": "AgentRuntimeCompleted" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json index 8149f3d4a..3b3515f3e 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-tail-page.json @@ -58,8 +58,12 @@ { "seq": 106, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 107, + "eventType": "TaskSucceeded" } ], "afterSeq": 100, - "latestSeq": 106 + "latestSeq": 107 } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-task-failed.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-task-failed.json new file mode 100644 index 000000000..7dbf70433 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-task-failed.json @@ -0,0 +1,67 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskFailed" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json index 6a9a1f993..82ef64da9 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json @@ -62,6 +62,10 @@ { "seq": 7, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 8, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json index 01effcd54..5bc553c05 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json @@ -58,8 +58,12 @@ { "seq": 6, "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ], "afterSeq": 0, - "latestSeq": 7 + "latestSeq": 8 } diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json index 59c341aa0..fe491ddea 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-write-after-terminal.json @@ -58,6 +58,10 @@ "executionIdempotencyKey": "approval-1" }, "toolCallID": "write-call-1" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" } ] } diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 49825e677..1f7fca1a2 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -126,6 +126,7 @@ TERMINAL_TYPES = { "TurnCompleted", "TaskCompleted", } +TASK_TERMINAL_TYPES = {"TaskSucceeded", "TaskFailed", "TaskCancelled"} def field(event, name): if not isinstance(event, dict): return None @@ -265,6 +266,7 @@ approval_declined_events = [e for e in events if event_type(e) == "ApprovalDecli write_exec_events = [e for e in write_events if is_write_execution_start(e)] write_start_events = write_exec_events terminal_events = [e for e in events if event_type(e) in TERMINAL_TYPES] +task_terminal_events = [e for e in events if event_type(e) in TASK_TERMINAL_TYPES] idempotency_events = [e for e in write_exec_events if idempotency_value(e)] failures = [] @@ -358,6 +360,17 @@ elif write_exec_events: earliest_terminal_seq = min(seq(event) for event in terminal_events) if earliest_terminal_seq <= latest_write_seq: failures.append("terminal completion event does not follow all write executions") +if not task_terminal_events: + failures.append("missing final TaskSucceeded lifecycle event") +else: + final_task_terminal = max(task_terminal_events, key=seq) + final_task_type = event_type(final_task_terminal) + if final_task_type != "TaskSucceeded": + failures.append(f"final Task lifecycle outcome is {final_task_type}, want TaskSucceeded") + final_event = max(events, key=seq) if events else None + final_event_type = event_type(final_event) if final_event else "" + if final_event_type != "TaskSucceeded": + failures.append(f"final execution event is {final_event_type}, want TaskSucceeded") if failures: print("Fibey Foundry Responses verification failed:", file=sys.stderr) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index f43582bad..9419408c0 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log" + "maps" "net/http" "net/url" "os" @@ -371,6 +372,11 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteError(w, http.StatusConflict, "turn already completed") return } + if s.activeTurnCountLocked() >= 1 { + s.mu.Unlock() + harness.WriteError(w, http.StatusConflict, "maximum concurrent turns reached") + return + } turn := &turnState{ request: req, initializing: true, @@ -896,6 +902,8 @@ func (s *server) recordContinueResults( toolName := firstNonBlank(turn.pendingTools[id], id) frame := s.newToolResultFrame(turn, baseSeq+int64(index), toolName, result) if !toolResultFrameFitsSSE(frame) { + maps.Copy(turn.submittedDigests, turn.bufferedDigests) + maps.Copy(turn.submittedDigests, newDigests) s.appendFailedLocked( turn, "brokered_tool_result_frame_too_large", @@ -1447,6 +1455,16 @@ func (s *server) markTurnConsumedLocked(turnID harness.HarnessTurnID) { } } +func (s *server) activeTurnCountLocked() int { + active := 0 + for _, turn := range s.turns { + if turn != nil && (turn.initializing || !turn.completed) { + active++ + } + } + return active +} + func (s *server) activeRuntimeSessionsLocked() map[harness.RuntimeSessionID]bool { active := map[harness.RuntimeSessionID]bool{} for _, turn := range s.turns { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 90f7b2589..9531c2e00 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -314,6 +314,58 @@ func TestResponsesAdapterDuplicateStartDuringInitializationRejected(t *testing.T } } +func TestResponsesAdapterEnforcesAdvertisedConcurrentTurnLimit(t *testing.T) { + received := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseFoundry := func() { releaseOnce.Do(func() { close(release) }) } + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + select { + case <-received: + default: + close(received) + } + <-release + writeJSON(w, finalResponsesMessage()) + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter := newTestResponsesAdapter(t, endpoint, nil) + t.Cleanup(releaseFoundry) + client := newHarnessClient(t, adapter) + first := responsesStartTurnRequest("foundry-concurrent-first") + second := responsesStartTurnRequest("foundry-concurrent-second") + second.RuntimeSessionID = first.RuntimeSessionID + + firstErr := make(chan error, 1) + go func() { + _, err := client.StartTurn(context.Background(), first) + firstErr <- err + }() + select { + case <-received: + case <-time.After(time.Second): + t.Fatal("first Foundry request did not arrive") + } + if _, err := client.StartTurn(context.Background(), second); err == nil || + !strings.Contains(err.Error(), "maximum concurrent turns reached") { + t.Fatalf("second concurrent StartTurn error = %v, want admission rejection", err) + } + + releaseFoundry() + select { + case err := <-firstErr: + if err != nil { + t.Fatalf("first StartTurn: %v", err) + } + case <-time.After(time.Second): + t.Fatal("first StartTurn did not finish") + } + if _, err := client.StartTurn(context.Background(), second); err != nil { + t.Fatalf("second StartTurn after first completed: %v", err) + } +} + func TestResponsesAdapterDiscardsUnusedTurnEnvironment(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) @@ -414,6 +466,12 @@ func TestResponsesAdapterCancelDuringInitialPostDoesNotRetainSession(t *testing. if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { t.Fatalf("CancelTurn: %v", err) } + second := responsesStartTurnRequest("foundry-after-cancel-initial") + second.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), second); err == nil || + !strings.Contains(err.Error(), "maximum concurrent turns reached") { + t.Fatalf("StartTurn during cancelled in-flight request error = %v, want admission rejection", err) + } releaseFoundry() select { case err := <-startErr: @@ -1945,6 +2003,48 @@ func TestResponsesOversizedBatchValidatesAllResultsBeforeFailure(t *testing.T) { } } +func TestResponsesOversizedResultTombstonesPreviouslyBufferedResults(t *testing.T) { + server := newServer(config{}, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-partial-then-oversized") + turn := &turnState{ + request: request, + pendingTools: map[string]string{ + "call-1": "support-ticket-lookup", + "call-2": "support-ticket-lookup", + }, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + first := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{first}) + if err != nil || len(toSubmit) != 0 { + t.Fatalf("buffer first result = %#v, %v", toSubmit, err) + } + oversizedOutput, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) + if err != nil { + t.Fatalf("marshal oversized output: %v", err) + } + second := toolResultForRequest(request, "call-2", true, oversizedOutput, nil) + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{second}); err == nil { + t.Fatal("oversized second result error = nil") + } + if !turn.completed || len(turn.bufferedResults) != 0 || len(turn.bufferedDigests) != 0 { + t.Fatalf( + "terminal oversized state = completed:%v buffers:%#v/%#v", + turn.completed, + turn.bufferedResults, + turn.bufferedDigests, + ) + } + for _, result := range []harness.ToolCallResult{first, second} { + if err := server.ensureTerminalContinueIsDuplicate(turn, []harness.ToolCallResult{result}); err != nil { + t.Fatalf("terminal duplicate %s rejected: %v", result.ToolCallID, err) + } + } +} + func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) { server := newServer(config{ runtimeName: "test", @@ -1983,8 +2083,11 @@ func TestResponsesOversizedToolResultFrameFailsBeforeContinuation(t *testing.T) if len(turn.bufferedResults) != 0 || len(turn.bufferedDigests) != 0 { t.Fatalf("oversized result retained buffered state: %#v/%#v", turn.bufferedResults, turn.bufferedDigests) } - if len(turn.submittedDigests) != 0 { - t.Fatalf("oversized result retained submitted digests: %#v", turn.submittedDigests) + if len(turn.submittedDigests) != 1 { + t.Fatalf("oversized result submitted digests = %#v, want duplicate tombstone", turn.submittedDigests) + } + if err := server.ensureTerminalContinueIsDuplicate(turn, []harness.ToolCallResult{result}); err != nil { + t.Fatalf("identical oversized result retry was not accepted as duplicate: %v", err) } } diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index fd2dbb5c1..982d29f23 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -222,6 +222,14 @@ expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-terminal-write-terminal.json \ "terminal completion event does not follow all write executions" \ "terminal-write-terminal" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-task-failed.json \ + "final Task lifecycle outcome is TaskFailed" \ + "task-failed-after-runtime-completion" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-success-before-runtime-completion.json \ + "final execution event is AgentRuntimeCompleted" \ + "task-success-before-runtime-completion" if [[ "$run_full" == "1" ]]; then run make test From 144ea6dc56b5db8aa90c5b16f0ae2f4e5d4a9b18 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 15:00:30 -0700 Subject: [PATCH 40/51] fix: serialize Foundry continuation cancellation Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 138 ++++++++++++------ .../harness/foundry-responses/main_test.go | 121 ++++++++++++++- 2 files changed, 213 insertions(+), 46 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 9419408c0..d377721b8 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -86,24 +86,33 @@ type server struct { type toolResultDigest [sha256.Size]byte +type responseDisposition int + +const ( + responseRejected responseDisposition = iota + responsePending + responseCompleted +) + type foundrySession struct { ID string LastSeen time.Time } type turnState struct { - request harness.StartTurnRequest - initializing bool - responseID string - foundrySessionID string - pendingTools map[string]string - bufferedResults map[string]harness.ToolCallResult - bufferedDigests map[string]toolResultDigest - submittedDigests map[string]toolResultDigest - frames []harness.HarnessEventFrame - completed bool - frameUpdates chan struct{} - continueMu sync.Mutex + request harness.StartTurnRequest + initializing bool + responseID string + foundrySessionID string + pendingTools map[string]string + bufferedResults map[string]harness.ToolCallResult + bufferedDigests map[string]toolResultDigest + submittedDigests map[string]toolResultDigest + frames []harness.HarnessEventFrame + completed bool + continuationInFlight bool + frameUpdates chan struct{} + continueMu sync.Mutex } type responsesRequest struct { @@ -417,8 +426,14 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { return } s.mu.Lock() - if !turn.completed && s.handleResponsesResponseLocked(turn, response) { - s.recordTurnSessionLocked(turn, foundrySessionID) + if !turn.completed { + disposition := s.handleResponsesResponseLocked(turn, response) + if disposition != responseRejected { + s.setTurnSessionLocked(turn, foundrySessionID) + } + if disposition == responseCompleted { + s.publishRuntimeSessionLocked(turn) + } } turn.initializing = false s.mu.Unlock() @@ -556,7 +571,21 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn return } ctx, cancel := s.foundryRequestContext(r.Context(), turn.request.Deadline) - defer cancel() + s.mu.Lock() + if turn.completed { + s.mu.Unlock() + cancel() + harness.WriteError(w, http.StatusConflict, "turn completed before hosted continuation submission") + return + } + turn.continuationInFlight = true + s.mu.Unlock() + defer func() { + cancel() + s.mu.Lock() + turn.continuationInFlight = false + s.mu.Unlock() + }() var response responsesResponse continuation := responsesRequest{ PreviousResponseID: previousResponseID, @@ -566,13 +595,20 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn updatedSessionID, err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response) if err != nil { s.mu.Lock() - s.appendFailedLocked( - turn, - "foundry_continuation_unknown", - "hosted continuation failed after submission was attempted; "+ - "failing closed to avoid duplicate continuation", - ) + completed := turn.completed + if !completed { + s.appendFailedLocked( + turn, + "foundry_continuation_unknown", + "hosted continuation failed after submission was attempted; "+ + "failing closed to avoid duplicate continuation", + ) + } s.mu.Unlock() + if completed { + harness.WriteError(w, http.StatusConflict, "turn completed while hosted continuation was in flight") + return + } log.Printf( "Foundry hosted Responses continuation failed after submission for turn %q (error type %T)", req.TurnID, @@ -605,8 +641,12 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn delete(turn.bufferedResults, result.ToolCallID) delete(turn.bufferedDigests, result.ToolCallID) } - if s.handleResponsesResponseLocked(turn, response) { - s.recordTurnSessionLocked(turn, updatedSessionID) + disposition := s.handleResponsesResponseLocked(turn, response) + if disposition != responseRejected { + s.setTurnSessionLocked(turn, updatedSessionID) + } + if disposition == responseCompleted { + s.publishRuntimeSessionLocked(turn) } s.mu.Unlock() harness.WriteJSON(w, http.StatusAccepted, continueResponse(req, "continue accepted")) @@ -630,8 +670,6 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt harness.WriteError(w, http.StatusBadRequest, "cancel request does not match started turn") return } - turn.continueMu.Lock() - defer turn.continueMu.Unlock() s.mu.Lock() if !turn.completed { s.clearBufferedToolResultsLocked(turn) @@ -659,9 +697,9 @@ func (s *server) handleResponsesResponse(turn *turnState, response responsesResp _ = s.handleResponsesResponseLocked(turn, response) } -func (s *server) handleResponsesResponseLocked(turn *turnState, response responsesResponse) bool { +func (s *server) handleResponsesResponseLocked(turn *turnState, response responsesResponse) responseDisposition { if turn.completed { - return false + return responseRejected } responseIDPresent := strings.TrimSpace(response.ID) != "" if responseIDPresent { @@ -673,24 +711,24 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_response_error", firstNonBlank(response.Error.Message, response.Error.Code, "Foundry hosted Responses returned an error"), ) - return false + return responseRejected } if isFailureStatus(response.Status) { s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) - return false + return responseRejected } if strings.TrimSpace(response.Status) == "" { s.appendFailedLocked(turn, "foundry_status_missing", "Foundry hosted Responses status is missing") - return false + return responseRejected } if !isCompletionStatus(response.Status) { s.appendFailedLocked(turn, "foundry_"+response.Status, "Foundry hosted Responses status "+response.Status) - return false + return responseRejected } calls, err := s.extractFunctionCalls(turn.request, response.Output) if err != nil { s.appendFailedLocked(turn, "foundry_function_call_invalid", err.Error()) - return false + return responseRejected } if len(calls) > 0 { if !responseIDPresent { @@ -699,7 +737,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_response_id_missing", "hosted response returned a function_call without an id needed for continuation", ) - return false + return responseRejected } seenInResponse := map[string]struct{}{} for _, call := range calls { @@ -709,7 +747,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated a function call id", ) - return false + return responseRejected } seenInResponse[call.callID] = struct{}{} if _, submitted := turn.submittedDigests[call.callID]; submitted { @@ -718,7 +756,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated an already-submitted function call", ) - return false + return responseRejected } if _, pending := turn.pendingTools[call.callID]; pending { s.appendFailedLocked( @@ -726,7 +764,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_repeated_function_call", "hosted response repeated an already-pending function call", ) - return false + return responseRejected } } frames := make([]harness.HarnessEventFrame, 0, len(calls)) @@ -749,7 +787,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_tool_call_frame_too_large", "hosted function call exceeded the harness SSE frame limit", ) - return false + return responseRejected } frames = append(frames, frame) } @@ -757,12 +795,12 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons turn.pendingTools[call.callID] = call.name s.appendPreparedFrameLocked(turn, frames[index]) } - return true + return responsePending } result := responsesMessageText(response.Output) if len([]byte(result)) > maxFoundryOutputBytes { s.appendFailedLocked(turn, "foundry_output_too_large", "foundry completion exceeded advertised output limit") - return false + return responseRejected } completedFrame := s.newFrame( turn, @@ -779,13 +817,13 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons "foundry_output_frame_too_large", "foundry completion exceeded the harness SSE frame limit", ) - return false + return responseRejected } s.appendPreparedFrameLocked(turn, completedFrame) s.clearBufferedToolResultsLocked(turn) turn.completed = true s.scheduleTurnCleanupLocked(turn) - return true + return responseCompleted } func (s *server) extractFunctionCalls( @@ -850,6 +888,9 @@ func (s *server) recordContinueResults( ) ([]harness.ToolCallResult, error) { s.mu.Lock() defer s.mu.Unlock() + if turn.completed { + return nil, fmt.Errorf("turn is already terminal") + } if len(turn.pendingTools) == 0 { return nil, fmt.Errorf("no tool calls are pending for this turn") } @@ -1373,13 +1414,22 @@ func isFailureStatus(status string) bool { } } -func (s *server) recordTurnSessionLocked(turn *turnState, sessionID string) { +func (s *server) setTurnSessionLocked(turn *turnState, sessionID string) { sessionID = strings.TrimSpace(sessionID) if sessionID == "" { return } turn.foundrySessionID = sessionID - s.runtimeSessions[turn.request.RuntimeSessionID] = foundrySession{ID: sessionID, LastSeen: time.Now().UTC()} +} + +func (s *server) publishRuntimeSessionLocked(turn *turnState) { + if strings.TrimSpace(turn.foundrySessionID) == "" { + return + } + s.runtimeSessions[turn.request.RuntimeSessionID] = foundrySession{ + ID: turn.foundrySessionID, + LastSeen: time.Now().UTC(), + } } func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { @@ -1458,7 +1508,7 @@ func (s *server) markTurnConsumedLocked(turnID harness.HarnessTurnID) { func (s *server) activeTurnCountLocked() int { active := 0 for _, turn := range s.turns { - if turn != nil && (turn.initializing || !turn.completed) { + if turn != nil && (turn.initializing || turn.continuationInFlight || !turn.completed) { active++ } } diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 9531c2e00..7a81f971f 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -233,8 +233,8 @@ func TestResponsesAdapterInterleavedResponsesRetainResponseSpecificSession(t *te firstTurn := &turnState{request: firstRequest} secondTurn := &turnState{request: secondRequest} server.mu.Lock() - server.recordTurnSessionLocked(secondTurn, secondSession) - server.recordTurnSessionLocked(firstTurn, firstSession) + server.setTurnSessionLocked(secondTurn, secondSession) + server.setTurnSessionLocked(firstTurn, firstSession) server.mu.Unlock() if firstTurn.foundrySessionID != "session-a" { t.Fatalf("first turn session = %q, want session-a", firstTurn.foundrySessionID) @@ -979,6 +979,102 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { } } +func TestResponsesAdapterCancelDuringHostedContinuationWins(t *testing.T) { + continuationReceived := make(chan struct{}) + releaseContinuation := make(chan struct{}) + var releaseOnce sync.Once + releaseFoundry := func() { releaseOnce.Do(func() { close(releaseContinuation) }) } + foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioReadAll(r.Body) + if err != nil { + http.Error(w, "read body", http.StatusBadRequest) + return + } + var decoded map[string]any + if err := json.Unmarshal(body, &decoded); err != nil { + http.Error(w, "decode body", http.StatusBadRequest) + return + } + if _, continuing := decoded["previous_response_id"]; !continuing { + w.Header().Set("x-agent-session-id", fakeSessionID) + writeJSON(w, functionCallResponse("support-ticket-lookup")) + return + } + select { + case <-continuationReceived: + default: + close(continuationReceived) + } + <-releaseContinuation + w.Header().Set("x-agent-session-id", "session-after-cancel") + writeJSON(w, finalResponsesMessage()) + })) + t.Cleanup(foundry.Close) + endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter, server := newTestResponsesAdapterWithServer( + t, + endpoint, + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + t.Cleanup(releaseFoundry) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-cancel-continuation") + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamCurrentFrames(t, client, request.TurnID) + requested := findFrame(frames, harness.FrameToolCallRequested) + if requested == nil { + t.Fatalf("frames = %#v, want tool request", frames) + } + server.mu.Lock() + _, publishedBeforeCompletion := server.runtimeSessions[request.RuntimeSessionID] + server.mu.Unlock() + if publishedBeforeCompletion { + t.Fatal("pending function call published runtime session before completion") + } + + continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) + continueErr := make(chan error, 1) + go func() { + _, err := client.ContinueTurn(context.Background(), continueRequest) + continueErr <- err + }() + select { + case <-continuationReceived: + case <-time.After(time.Second): + t.Fatal("hosted continuation did not arrive") + } + if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { + t.Fatalf("CancelTurn during continuation: %v", err) + } + second := responsesStartTurnRequest("foundry-after-cancel-continuation") + second.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), second); err == nil || + !strings.Contains(err.Error(), "maximum concurrent turns reached") { + t.Fatalf("StartTurn during cancelled continuation error = %v, want admission rejection", err) + } + releaseFoundry() + select { + case err := <-continueErr: + if err == nil || !strings.Contains(err.Error(), "turn completed while hosted continuation was in flight") { + t.Fatalf("ContinueTurn after cancellation error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("ContinueTurn did not finish after releasing hosted response") + } + frames = streamCurrentFrames(t, client, request.TurnID) + if !hasFrameType(frames, harness.FrameTurnCancelled) || hasFrameType(frames, harness.FrameTurnCompleted) { + t.Fatalf("frames = %#v, want cancellation to win continuation race", frames) + } + server.mu.Lock() + _, publishedAfterCancel := server.runtimeSessions[request.RuntimeSessionID] + server.mu.Unlock() + if publishedAfterCancel { + t.Fatal("cancelled continuation published runtime session") + } +} + func TestResponsesAdapterContinuationHonorsOriginalTurnDeadline(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "function_call", toolName: "support-ticket-lookup"}) adapter := newTestResponsesAdapter( @@ -1266,6 +1362,27 @@ func TestResponsesMixedRepeatedFunctionCallFailsTurn(t *testing.T) { } } +func TestResponsesRecordContinueResultsRejectsTerminalTurn(t *testing.T) { + server := newServer(config{}, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-terminal-record") + turn := &turnState{ + request: request, + pendingTools: map[string]string{"call-1": "support-ticket-lookup"}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + completed: true, + } + result := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{result}); err == nil || + !strings.Contains(err.Error(), "already terminal") { + t.Fatalf("recordContinueResults terminal error = %v", err) + } + if len(turn.submittedDigests) != 0 || len(turn.bufferedResults) != 0 { + t.Fatalf("terminal result mutation = %#v/%#v", turn.submittedDigests, turn.bufferedResults) + } +} + func TestResponsesAdapterAlreadySubmittedPendingResultIsNoop(t *testing.T) { server := newServer(config{}, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-submitted-noop") From 79cabe4b5bd23f911fad3eef60faeae3f1b4b1c3 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 15:37:32 -0700 Subject: [PATCH 41/51] fix: make Foundry terminal expiry non-retryable Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 47 +++++++++++----- .../harness/foundry-responses/main_test.go | 53 +++++++++++++++++++ internal/controller/harness_wrapper.go | 15 +++++- internal/controller/harness_wrapper_test.go | 25 +++++++-- 4 files changed, 120 insertions(+), 20 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index d377721b8..53479cf7e 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -26,19 +26,19 @@ import ( ) const ( - defaultAddr = ":8090" - defaultAPIVersion = "v1" - defaultRequestTimeout = 20 * time.Second - defaultStateRetention = 10 * time.Minute - defaultReadHeaderTimeout = 5 * time.Second - defaultReadTimeout = 30 * time.Second - defaultIdleTimeout = 60 * time.Second - maxFoundryOutputBytes = 1 << 20 - maxFoundryBodyBytes = 4 << 20 - maxConsumedTurnIDs = 1024 - maxFrameSequence int64 = 1<<63 - 1 - readinessPath = "/v1/ready" - foundryInitialUnknown = "foundry_initial_unknown" + defaultAddr = ":8090" + defaultAPIVersion = "v1" + defaultRequestTimeout = 20 * time.Second + defaultStateRetention = 10 * time.Minute + defaultReadHeaderTimeout = 5 * time.Second + defaultReadTimeout = 30 * time.Second + defaultIdleTimeout = 60 * time.Second + maxFoundryOutputBytes = 1 << 20 + maxFoundryBodyBytes = 4 << 20 + maxConsumedTurnIDs = 1024 + maxBrokeredToolCalls = 32 + readinessPath = "/v1/ready" + foundryInitialUnknown = "foundry_initial_unknown" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -105,6 +105,7 @@ type turnState struct { responseID string foundrySessionID string pendingTools map[string]string + requestedToolCalls int bufferedResults map[string]harness.ToolCallResult bufferedDigests map[string]toolResultDigest submittedDigests map[string]toolResultDigest @@ -175,11 +176,15 @@ const responsesEndpointRequirement = "foundry hosted Responses endpoint must use func main() { cfg := loadConfig() s := newServer(cfg, &http.Client{Timeout: cfg.requestTimeout}) + logEndpoint, err := s.responsesEndpoint() + if err != nil { + logEndpoint = cfg.endpoint + } log.Printf( "Foundry hosted Responses AgentRuntime adapter listening on %s runtime=%s endpoint=%s", cfg.addr, cfg.runtimeName, - sanitizeEndpoint(cfg.endpoint), + sanitizeEndpoint(logEndpoint), ) if err := newAdapterHTTPServer(cfg.addr, s.handler()).ListenAndServe(); err != nil { log.Fatal(err) @@ -451,8 +456,13 @@ func (s *server) turn(w http.ResponseWriter, r *http.Request) { } s.mu.Lock() turn := s.turns[turnID] + _, consumed := s.consumedTurns[turnID] s.mu.Unlock() if turn == nil { + if consumed { + harness.WriteError(w, http.StatusGone, "terminal turn expired from runtime retention") + return + } harness.WriteError(w, http.StatusNotFound, "turn not found") return } @@ -731,6 +741,14 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons return responseRejected } if len(calls) > 0 { + if len(calls) > maxBrokeredToolCalls-turn.requestedToolCalls { + s.appendFailedLocked( + turn, + "foundry_tool_call_limit_exceeded", + "hosted response exceeded the brokered tool-call limit", + ) + return responseRejected + } if !responseIDPresent { s.appendFailedLocked( turn, @@ -791,6 +809,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons } frames = append(frames, frame) } + turn.requestedToolCalls += len(calls) for index, call := range calls { turn.pendingTools[call.callID] = call.name s.appendPreparedFrameLocked(turn, frames[index]) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 7a81f971f..8f237123b 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -1533,6 +1533,11 @@ func TestResponsesAdapterRejectsConsumedTurnAfterCleanup(t *testing.T) { } time.Sleep(5 * time.Millisecond) } + if err := client.StreamFrames(context.Background(), request.TurnID, 0, func(harness.HarnessEventFrame) error { + return nil + }); err == nil || !strings.Contains(err.Error(), "410") { + t.Fatalf("expired terminal stream error = %v, want non-retryable 410", err) + } if _, err := client.StartTurn(context.Background(), request); err == nil || !strings.Contains(err.Error(), "turn already completed") { t.Fatalf("retry after cleanup error = %v, want consumed-turn conflict", err) @@ -1616,6 +1621,23 @@ func TestResponsesAdapterValidatesCancelIdentityBeforeMutation(t *testing.T) { } } +func TestResponsesEndpointComposesProjectAndAgent(t *testing.T) { + server := newServer(config{ + projectEndpoint: "https://example.services.ai.azure.com/api/projects/project-a/", + agentName: "agent/name", + apiVersion: "v1", + }, nil) + endpoint, err := server.responsesEndpoint() + if err != nil { + t.Fatalf("responsesEndpoint: %v", err) + } + want := "https://example.services.ai.azure.com/api/projects/project-a/agents/" + + "agent%2Fname/endpoint/protocols/openai/responses?api-version=v1" + if endpoint != want { + t.Fatalf("responsesEndpoint = %q, want %q", endpoint, want) + } +} + func TestResponsesEndpointSafety(t *testing.T) { tests := []struct { name string @@ -1694,6 +1716,37 @@ func TestProjectEndpointSafety(t *testing.T) { } } +func TestResponsesRejectsOversizedHostedFunctionCallBatch(t *testing.T) { + server := newServer(config{ + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, nil) + turn := &turnState{ + request: brokeredReadRequest("foundry-tool-call-cap"), + pendingTools: map[string]string{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + output := make([]responsesOutput, 0, maxBrokeredToolCalls+1) + for i := 0; i <= maxBrokeredToolCalls; i++ { + output = append(output, responsesOutput{ + Type: "function_call", + CallID: fmt.Sprintf("call-%d", i), + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }) + } + server.handleResponsesResponse(turn, responsesResponse{ID: "resp-cap", Status: "completed", Output: output}) + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed.Reason != "foundry_tool_call_limit_exceeded" { + t.Fatalf("failed frame = %#v, want foundry_tool_call_limit_exceeded", failed) + } + if hasFrameType(turn.frames, harness.FrameToolCallRequested) || turn.requestedToolCalls != 0 { + t.Fatalf("oversized hosted batch was partially accepted: frames=%#v count=%d", turn.frames, turn.requestedToolCalls) + } +} + func TestResponsesParserConsumesGoldenFixtures(t *testing.T) { server := newServer( config{ diff --git a/internal/controller/harness_wrapper.go b/internal/controller/harness_wrapper.go index e4e064510..cf6e1fbfe 100644 --- a/internal/controller/harness_wrapper.go +++ b/internal/controller/harness_wrapper.go @@ -1452,6 +1452,19 @@ func (r *TaskReconciler) clearHarnessWrapperTurnState(ctx context.Context, task } func harnessWrapperStreamErrorIsMissingTurn(err error) bool { + if err == nil { + return false + } + message := err.Error() + for _, marker := range []string{"(404)", "turn not found"} { + if strings.Contains(message, marker) { + return true + } + } + return false +} + +func harnessWrapperCancelErrorIsMissingTurn(err error) bool { if err == nil { return false } @@ -1513,7 +1526,7 @@ func (r *TaskReconciler) cancelHarnessWrapperTurn(ctx context.Context, task *cor CorrelationID: strings.TrimSpace(task.Annotations[harnessWrapperCorrelationIDAnno]), Reason: reason, }) - if err != nil && harnessWrapperStreamErrorIsMissingTurn(err) { + if err != nil && harnessWrapperCancelErrorIsMissingTurn(err) { return nil } return err diff --git a/internal/controller/harness_wrapper_test.go b/internal/controller/harness_wrapper_test.go index e2e00c7ed..82a432b16 100644 --- a/internal/controller/harness_wrapper_test.go +++ b/internal/controller/harness_wrapper_test.go @@ -2574,18 +2574,33 @@ func TestHarnessWrapperCapabilitiesReadErrorRetryable(t *testing.T) { } func TestHarnessWrapperStreamMissingTurnErrorClassification(t *testing.T) { - for _, message := range []string{"stream_frames failed (404): turn not found", "stream_frames failed (410): gone"} { - if !harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("%s", message)) { - t.Fatalf("%q should be classified as missing turn", message) + if !harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("stream_frames failed (404): turn not found")) { + t.Fatal("404 turn-not-found stream error should be classified as retryable missing turn") + } + for _, message := range []string{ + "stream_frames failed (410): terminal turn expired from runtime retention", + "stream_frames failed (401): unauthorized", + } { + if harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("%s", message)) { + t.Fatalf("%q should not be classified as retryable missing turn", message) } } - if harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("stream_frames failed (401): unauthorized")) { - t.Fatal("unauthorized stream error should not be classified as missing turn") +} + +func TestHarnessWrapperCancelMissingTurnErrorClassification(t *testing.T) { + for _, message := range []string{ + "cancel_turn failed (404): turn not found", + "cancel_turn failed (410): terminal turn expired from runtime retention", + } { + if !harnessWrapperCancelErrorIsMissingTurn(fmt.Errorf("%s", message)) { + t.Fatalf("%q should be ignored during cancellation", message) + } } } func TestHarnessWrapperStreamTerminalErrorClassification(t *testing.T) { for _, message := range []string{ + "stream_frames failed (410): terminal turn expired from runtime retention", "harness frame identity does not match running turn", "invalid harness frame: turn completed payload is required", "stream_frames failed: decode harness frame: invalid character", From 07dad1f01251bff33119a2a5874b8d17a47cd385 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 16:43:06 -0700 Subject: [PATCH 42/51] fix: harden Foundry response evidence Signed-off-by: Sertac Ozercan --- .../foundry-responses/live-evidence.sh | 202 ++++++++++-------- .../harness/foundry-responses/live-smoke.sh | 2 +- examples/harness/foundry-responses/main.go | 13 +- .../harness/foundry-responses/main_test.go | 59 +++++ .../harness/foundry-responses/validate.sh | 17 ++ 5 files changed, 205 insertions(+), 88 deletions(-) diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index ca144341b..a0b1f13bb 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -193,99 +193,130 @@ import sys from pathlib import Path payload = json.loads(Path(sys.argv[1]).read_text()) -if isinstance(payload, dict): - if isinstance(payload.get("events"), list): - events = payload["events"] - elif isinstance(payload.get("items"), list): - events = payload["items"] - else: - events = [payload] -elif isinstance(payload, list): - events = payload -else: - events = [] - - -def content_dict(event): - content = event.get("content") if isinstance(event, dict) else None - if isinstance(content, dict): - return content - if isinstance(content, str): - try: - decoded = json.loads(content) - except Exception: # noqa: BLE001 - best-effort evidence summarizer - return {} - return decoded if isinstance(decoded, dict) else {} - return {} - - -def event_field(event, *names): - if not isinstance(event, dict): - return None - content = content_dict(event) - for name in names: - value = event.get(name) - if value not in (None, ""): - return value - value = content.get(name) - if value not in (None, ""): - return value - return None - - -def has_idempotency(value): - if isinstance(value, dict): - if any(k in value for k in ("idempotencyKey", "Idempotency-Key")): - return True - return any(has_idempotency(v) for v in value.values()) - if isinstance(value, list): - return any(has_idempotency(v) for v in value) - if isinstance(value, str): - try: - decoded = json.loads(value) - except Exception: # noqa: BLE001 - return False - return has_idempotency(decoded) - return False - -latest_seq = payload.get("latestSeq") if isinstance(payload, dict) else None +if not isinstance(payload, dict) or not isinstance(payload.get("events"), list): + raise SystemExit("task event capture must be a paginated event object") +events = payload["events"] +latest_seq = int(payload.get("latestSeq", 0)) +after_seq = int(payload.get("afterSeq", 0)) sequence_values = [ int(event.get("seq", 0)) for event in events if isinstance(event, dict) and event.get("seq") is not None ] -captured_through_seq = max(sequence_values, default=0) -if latest_seq is not None: - after_seq = int(payload.get("afterSeq", 0)) - if after_seq != 0: - raise SystemExit(f"task event capture is incomplete: afterSeq must be 0, got {after_seq}") - complete_sequence = len(sequence_values) == int(latest_seq) and all( - seq == expected for expected, seq in enumerate(sequence_values, start=1) +if after_seq != 0: + raise SystemExit(f"task event capture is incomplete: afterSeq must be 0, got {after_seq}") +complete_sequence = len(sequence_values) == latest_seq and all( + seq == expected for expected, seq in enumerate(sequence_values, start=1) +) +if not complete_sequence: + raise SystemExit( + "task event capture is incomplete: " + f"sequences do not cover 1 through latestSeq {latest_seq}" ) - if not complete_sequence: - raise SystemExit( - "task event capture is incomplete: " - f"sequences do not cover 1 through latestSeq {latest_seq}" - ) -summary = [] -for index, event in enumerate(events, start=1): - event = event if isinstance(event, dict) else {} - summary.append({ - "index": index, - "type": event.get("type") or event.get("eventType"), - "toolName": event_field(event, "toolName", "tool", "name"), - "hasIdempotencyEvidence": has_idempotency(event), - "hasError": bool(event_field(event, "error", "errorCode")), - }) -Path(sys.argv[2]).write_text(json.dumps({ - "eventCount": len(events), - "capturedThroughSeq": captured_through_seq, +safe_top_level = { + "seq", + "type", + "eventType", + "severity", + "toolName", + "tool", + "name", + "toolCallID", + "toolCallId", + "approvalID", + "approvalId", + "targetTool", + "brokeredClass", + "executionState", + "idempotencyKey", + "executionIdempotencyKey", + "Idempotency-Key", + "createdAt", +} +safe_scalar_content_keys = { + "approvalID", + "approvalId", + "targetTool", + "toolCallID", + "toolCallId", + "toolName", + "tool", + "name", + "brokeredClass", + "executionState", + "decision", +} + + +def scalar(value): + return value if isinstance(value, (str, int, float, bool)) and not isinstance(value, type(None)) else None + + +def error_marker_is_set(value): + return value not in (None, "", False, 0, {}, []) + + +def has_direct_error(value): + if not isinstance(value, dict): + return False + return error_marker_is_set(value.get("error")) or error_marker_is_set(value.get("errorCode")) + + +def safe_content(value): + if not isinstance(value, dict): + return {} + safe = {} + for key in safe_scalar_content_keys: + found = scalar(value.get(key)) + if found not in (None, ""): + safe[key] = found + for key in ("executionIdempotencyKey", "idempotencyKey", "Idempotency-Key"): + idempotency = scalar(value.get(key)) + if isinstance(idempotency, str) and idempotency.strip(): + safe["idempotencyKey"] = idempotency.strip() + break + harness = value.get("harness") + if isinstance(harness, dict): + frame_type = scalar(harness.get("frameType")) + if isinstance(frame_type, str) and frame_type: + safe["harness"] = {"frameType": frame_type} + return safe + + +safe_events = [] +for event in events: + if not isinstance(event, dict): + raise SystemExit("task event capture contains a non-object event") + safe_event = {} + for key in safe_top_level: + if key in event: + value = scalar(event[key]) + if value not in (None, ""): + safe_event[key] = value + content = event.get("content") + if isinstance(content, str): + try: + content = json.loads(content) + except Exception: # noqa: BLE001 - unsafe content is omitted, not stored + content = {} + redacted_content = safe_content(content) + if redacted_content: + safe_event["content"] = redacted_content + safe_event["hasError"] = has_direct_error(event) or has_direct_error(content) + safe_events.append(safe_event) + +safe_payload = { + "namespace": scalar(payload.get("namespace")), + "streamType": scalar(payload.get("streamType")), + "streamID": scalar(payload.get("streamID")), + "afterSeq": 0, "latestSeq": latest_seq, - "events": summary, -}, indent=2, sort_keys=True) + "\n") + "events": safe_events, +} +Path(sys.argv[2]).write_text(json.dumps(safe_payload, indent=2, sort_keys=True) + "\n") PY -scan_saved_artifact "$events_json" "task events summary" +scan_saved_artifact "$events_json" "redacted task events" python3 - "$approvals_tmp" "$approvals_json" <<'PY' import json import sys @@ -319,7 +350,8 @@ fibey_verifier="${script_dir}/../fibey-custom-agent-demo/verify-foundry-response if [[ ! -x "$fibey_verifier" ]]; then fibey_verifier="${script_dir}/../../fibey-custom-agent-demo/verify-foundry-responses.sh" fi -"$fibey_verifier" --json "$events_tmp" >"$verifier_out" +"$fibey_verifier" --json "$events_tmp" >/dev/null +"$fibey_verifier" --json "$events_json" >"$verifier_out" scan_saved_artifact "$verifier_out" "Fibey verifier output" pods_tmp="$(mktemp)" diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 761ac20f8..9657ce8f0 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -205,7 +205,7 @@ preflight() { if [[ -n "${brokered_classes//[[:space:],]/}" ]]; then IFS=',' read -r -a classes <<<"$brokered_classes" for class in "${classes[@]}"; do - class="${class//[[:space:]]/}" + class="$(printf '%s' "$class" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" [[ -z "$class" ]] && continue [[ "$class" == "read" || "$class" == "write" ]] || fail "unsupported brokered class '$class' (expected read/write)" done diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 53479cf7e..a1eb6a209 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -1155,9 +1155,15 @@ func (s *server) postResponses( strings.TrimSpace(string(data)), ) } + responseBody, err := io.ReadAll(io.LimitReader(resp.Body, maxFoundryBodyBytes+1)) + if err != nil { + return "", fmt.Errorf("read Foundry hosted Responses response: %w", err) + } + if len(responseBody) > maxFoundryBodyBytes { + return "", fmt.Errorf("foundry hosted Responses response exceeded %d bytes", maxFoundryBodyBytes) + } if out != nil { - decoder := json.NewDecoder(io.LimitReader(resp.Body, maxFoundryBodyBytes)) - if err := decoder.Decode(out); err != nil { + if err := json.Unmarshal(responseBody, out); err != nil { return "", fmt.Errorf("decode Foundry hosted Responses response: %w", err) } sessionID = firstNonBlank(out.AgentSessionID, sessionID) @@ -1379,6 +1385,9 @@ func outputItemText(item responsesOutput) string { if text, ok := m["text"].(string); ok && strings.TrimSpace(text) != "" { parts = append(parts, strings.TrimSpace(text)) } + if refusal, ok := m["refusal"].(string); ok && strings.TrimSpace(refusal) != "" { + parts = append(parts, strings.TrimSpace(refusal)) + } if textMap, ok := m["text"].(map[string]any); ok { if value, ok := textMap["value"].(string); ok && strings.TrimSpace(value) != "" { parts = append(parts, strings.TrimSpace(value)) diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 8f237123b..ee063a8da 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -1479,6 +1479,65 @@ func TestResponsesAdapterStateLossContinueFailsSafely(t *testing.T) { } } +func TestPostResponsesRejectsOversizedAndTrailingBodies(t *testing.T) { + valid, err := json.Marshal(finalResponsesMessage()) + if err != nil { + t.Fatalf("marshal valid response: %v", err) + } + tests := []struct { + name string + body []byte + wantErr string + }{ + { + name: "oversized", + body: append(append([]byte(nil), valid...), bytes.Repeat([]byte(" "), maxFoundryBodyBytes+1-len(valid))...), + wantErr: "exceeded", + }, + { + name: "trailing data", + body: append(append([]byte(nil), valid...), []byte("trailing")...), + wantErr: "decode", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(tt.body) + })) + t.Cleanup(upstream.Close) + server := newServer(config{ + endpoint: upstream.URL + "/agents/test/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + }, &http.Client{Timeout: time.Second}) + var response responsesResponse + _, err := server.postResponses( + context.Background(), + "runtime-session", + responsesRequest{Input: "test"}, + &response, + ) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("postResponses error = %v, want %q", err, tt.wantErr) + } + }) + } +} + +func TestResponsesMessageTextPreservesRefusal(t *testing.T) { + got := responsesMessageText([]responsesOutput{{ + Type: "message", + Content: []any{map[string]any{ + "type": "refusal", + "refusal": "I cannot perform that request.", + }}, + }}) + if got != "I cannot perform that request." { + t.Fatalf("responsesMessageText refusal = %q", got) + } +} + func TestResponsesAPIVersionDefaultsToSDKValue(t *testing.T) { t.Setenv(envAPIVersion, "") if got := loadConfig().apiVersion; got != defaultAPIVersion { diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 982d29f23..16a361e04 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -130,6 +130,23 @@ if [[ "$missing_image_code" == "0" ]] || ! grep -q "ADAPTER_IMAGE is required" " fi rm -f "$missing_image_err" +invalid_class_err="$(mktemp)" +set +e +ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES="r ead" \ + examples/harness/foundry-responses/live-smoke.sh >/dev/null 2>"$invalid_class_err" +invalid_class_code=$? +set -e +if [[ "$invalid_class_code" == "0" ]] || ! grep -q "unsupported brokered class 'r ead'" "$invalid_class_err"; then + cat "$invalid_class_err" >&2 + rm -f "$invalid_class_err" + echo "expected internally spaced brokered class to fail preflight" >&2 + exit 1 +fi +rm -f "$invalid_class_err" + smoke_tmp="$(mktemp -d)" smoke_capture="$smoke_tmp/rendered.yaml" cat >"$smoke_tmp/kubectl" <<'SH' From 1615b9c22d229e8abbc5a306997af8129a680ba7 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 16:57:56 -0700 Subject: [PATCH 43/51] fix: fail closed on lost Foundry turns Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 39 +++++++++---- .../harness/foundry-responses/main_test.go | 57 ++++++++++++++++++- internal/controller/harness_wrapper.go | 2 +- internal/controller/harness_wrapper_test.go | 10 ++++ 4 files changed, 95 insertions(+), 13 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index a1eb6a209..97d8bb839 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -105,6 +105,7 @@ type turnState struct { responseID string foundrySessionID string pendingTools map[string]string + suppressedToolCalls map[string]struct{} requestedToolCalls int bufferedResults map[string]harness.ToolCallResult bufferedDigests map[string]toolResultDigest @@ -392,13 +393,14 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { return } turn := &turnState{ - request: req, - initializing: true, - pendingTools: map[string]string{}, - bufferedResults: map[string]harness.ToolCallResult{}, - bufferedDigests: map[string]toolResultDigest{}, - submittedDigests: map[string]toolResultDigest{}, - frameUpdates: make(chan struct{}), + request: req, + initializing: true, + pendingTools: map[string]string{}, + suppressedToolCalls: map[string]struct{}{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + frameUpdates: make(chan struct{}), } s.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") s.turns[req.TurnID] = turn @@ -459,11 +461,11 @@ func (s *server) turn(w http.ResponseWriter, r *http.Request) { _, consumed := s.consumedTurns[turnID] s.mu.Unlock() if turn == nil { + message := "turn state unavailable after runtime restart" if consumed { - harness.WriteError(w, http.StatusGone, "terminal turn expired from runtime retention") - return + message = "terminal turn expired from runtime retention" } - harness.WriteError(w, http.StatusNotFound, "turn not found") + harness.WriteError(w, http.StatusGone, message) return } switch resource { @@ -497,6 +499,7 @@ func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turn for { s.mu.Lock() frames := append([]harness.HarnessEventFrame(nil), turn.frames...) + suppressed := maps.Clone(turn.suppressedToolCalls) completed := turn.completed updates := turn.frameUpdates if updates == nil { @@ -508,6 +511,11 @@ func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turn if frame.Seq <= nextSeq { continue } + if frame.Type == harness.FrameToolCallRequested { + if _, skip := suppressed[frame.ToolCallID]; skip { + continue + } + } if err := harness.WriteSSEFrame(w, frame); err != nil { return } @@ -682,6 +690,7 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt } s.mu.Lock() if !turn.completed { + s.suppressPendingToolCallsLocked(turn) s.clearBufferedToolResultsLocked(turn) s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled") turn.completed = true @@ -1424,6 +1433,15 @@ func (s *server) newToolResultFrame( ) } +func (s *server) suppressPendingToolCallsLocked(turn *turnState) { + if turn.suppressedToolCalls == nil { + turn.suppressedToolCalls = map[string]struct{}{} + } + for toolCallID := range turn.pendingTools { + turn.suppressedToolCalls[toolCallID] = struct{}{} + } +} + func (s *server) clearBufferedToolResultsLocked(turn *turnState) { clear(turn.bufferedResults) clear(turn.bufferedDigests) @@ -1464,6 +1482,7 @@ func (s *server) appendFailedLocked(turn *turnState, reason, msg string) { if turn.completed { return } + s.suppressPendingToolCallsLocked(turn) s.clearBufferedToolResultsLocked(turn) failedFrame := s.newFrame( turn, diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index ee063a8da..f90696fd2 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -1468,8 +1468,8 @@ func TestResponsesAdapterStateLossContinueFailsSafely(t *testing.T) { context.Background(), goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)), ) - if err == nil || !strings.Contains(err.Error(), "turn not found") { - t.Fatalf("restart continue error = %v, want clear turn not found", err) + if err == nil || !strings.Contains(err.Error(), "410") { + t.Fatalf("restart continue error = %v, want non-retryable gone state", err) } if foundry.postCount.Load() != 1 { t.Fatalf( @@ -2232,6 +2232,59 @@ func TestResponsesOversizedBatchValidatesAllResultsBeforeFailure(t *testing.T) { } } +func TestResponsesTerminalFailureSuppressesPendingToolFrames(t *testing.T) { + server := newServer(config{ + adapterBearer: "adapter-auth-value", + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-suppress-pending") + turn := &turnState{ + request: request, + pendingTools: map[string]string{}, + suppressedToolCalls: map[string]struct{}{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + server.turns[request.TurnID] = turn + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-1", + Status: "completed", + Output: []responsesOutput{ + { + Type: "function_call", CallID: "call-1", Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }, + { + Type: "function_call", CallID: "call-2", Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-2"}`), + }, + }, + }) + first := toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true}`), nil) + toSubmit, err := server.recordContinueResults(turn, []harness.ToolCallResult{first}) + if err != nil || len(toSubmit) != 0 { + t.Fatalf("buffer first result = %#v, %v", toSubmit, err) + } + oversizedOutput, err := json.Marshal(map[string]any{"payload": strings.Repeat("x", harness.MaxSSEFrameBytes)}) + if err != nil { + t.Fatalf("marshal oversized output: %v", err) + } + second := toolResultForRequest(request, "call-2", true, oversizedOutput, nil) + if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{second}); err == nil { + t.Fatal("oversized second result error = nil") + } + + adapter := httptest.NewServer(server.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + frames := streamCurrentFrames(t, client, request.TurnID) + if hasFrameType(frames, harness.FrameToolCallRequested) || !hasFrameType(frames, harness.FrameTurnFailed) { + t.Fatalf("replayed frames = %#v, want pending tool requests suppressed before terminal failure", frames) + } +} + func TestResponsesOversizedResultTombstonesPreviouslyBufferedResults(t *testing.T) { server := newServer(config{}, &http.Client{Timeout: time.Second}) request := brokeredReadRequest("foundry-partial-then-oversized") diff --git a/internal/controller/harness_wrapper.go b/internal/controller/harness_wrapper.go index cf6e1fbfe..524066f39 100644 --- a/internal/controller/harness_wrapper.go +++ b/internal/controller/harness_wrapper.go @@ -1495,7 +1495,7 @@ func harnessWrapperStreamErrorIsTerminal(err error) bool { } func harnessWrapperStreamErrorIsBrokeredPause(err error) bool { - if err == nil { + if err == nil || harnessWrapperStreamErrorIsTerminal(err) { return false } return strings.Contains(err.Error(), "continue brokered tool call") || errors.Is(err, errHarnessBrokeredApprovalPending) diff --git a/internal/controller/harness_wrapper_test.go b/internal/controller/harness_wrapper_test.go index 82a432b16..43c3f32af 100644 --- a/internal/controller/harness_wrapper_test.go +++ b/internal/controller/harness_wrapper_test.go @@ -2598,6 +2598,16 @@ func TestHarnessWrapperCancelMissingTurnErrorClassification(t *testing.T) { } } +func TestHarnessWrapperBrokeredPauseExcludesTerminalGone(t *testing.T) { + err := fmt.Errorf("continue brokered tool call \"call-1\": stream_frames failed (410): gone") + if harnessWrapperStreamErrorIsBrokeredPause(err) { + t.Fatal("terminal 410 continuation error should not be classified as a brokered pause") + } + if !harnessWrapperStreamErrorIsBrokeredPause(fmt.Errorf("continue brokered tool call \"call-1\": approval pending")) { + t.Fatal("ordinary continuation pause should remain classified as brokered pause") + } +} + func TestHarnessWrapperStreamTerminalErrorClassification(t *testing.T) { for _, message := range []string{ "stream_frames failed (410): terminal turn expired from runtime retention", From 928ea103c1afd3798b7fcab42a24f95808987b53 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 17:45:30 -0700 Subject: [PATCH 44/51] fix: require Foundry continuation proof Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/README.md | 2 +- .../foundry-responses/kubernetes.example.yaml | 1 - .../harness/foundry-responses/live-smoke.sh | 5 +- examples/harness/foundry-responses/main.go | 47 +++++++--- .../harness/foundry-responses/main_test.go | 86 +++++++++++++++++-- .../05_hosted_continuation_request.json | 9 +- .../harness/foundry-responses/validate.sh | 37 ++++++++ 7 files changed, 165 insertions(+), 22 deletions(-) diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 965aef036..80181f805 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -28,7 +28,7 @@ Use this adapter for AgentKit agents deployed as Foundry hosted agents. Use `exa | `ORKA_FOUNDRY_RESPONSES_API_KEY` | Static API-key auth mode. Tests/demo only unless your deployment standard permits it. | | `ORKA_FOUNDRY_RESPONSES_AUTH_BEARER` | Static bearer auth mode. Tests/demo only unless supplied by a production token refresher sidecar. | | `ORKA_FOUNDRY_RESPONSES_TOKEN_AUDIENCE` | Reserved for future workload-identity token refresh support; currently not used. | -| `ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF` | Optional Orka-only proof value sent on hosted Responses continuations in both the `X-AgentKit-Brokered-Continuation-Proof` header and `brokered_continuation_proof` request-body field, so gateways that strip custom headers can still forward it. Set it to match AgentKit's `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` when that guard is enabled. | +| `ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF` | Required whenever brokered tool classes are enabled. The Orka-only proof is sent on hosted Responses continuations in both the `X-AgentKit-Brokered-Continuation-Proof` header and `brokered_continuation_proof` request-body field, so gateways that strip custom headers can still forward it. Set it to match AgentKit's `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF`. | | `ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES` | Comma-separated static classes the hosted AgentKit deployment has been configured and conformance-tested to request, e.g. `read` or `read,write`. Empty means observed-only. | | `ORKA_FOUNDRY_RESPONSES_POLL_TIMEOUT` | Per-request timeout for hosted Responses calls, default `20s`. | | `ORKA_FOUNDRY_RESPONSES_STATE_RETENTION` | How long terminal in-memory turn/session state is retained, default `10m`. | diff --git a/examples/harness/foundry-responses/kubernetes.example.yaml b/examples/harness/foundry-responses/kubernetes.example.yaml index 1eca993b5..04659452f 100644 --- a/examples/harness/foundry-responses/kubernetes.example.yaml +++ b/examples/harness/foundry-responses/kubernetes.example.yaml @@ -68,7 +68,6 @@ spec: secretKeyRef: name: sample-foundry-responses-adapter-config key: continuation-proof - optional: true # The hosted agent must statically expose probe-only conformance_read # before read can pass AgentRuntime readiness. Advertise only classes # whose live brokered conformance passed. diff --git a/examples/harness/foundry-responses/live-smoke.sh b/examples/harness/foundry-responses/live-smoke.sh index 9657ce8f0..f9c501ceb 100755 --- a/examples/harness/foundry-responses/live-smoke.sh +++ b/examples/harness/foundry-responses/live-smoke.sh @@ -29,7 +29,7 @@ Optional environment: Set explicitly to an empty string for observed-only mode. Every advertised class requires the hosted agent to statically expose the matching probe-only conformance_read/conformance_write schema. - ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF optional + ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF required when brokered classes are enabled The script never prints secret values. Do not run with shell tracing (set -x). USAGE @@ -210,6 +210,9 @@ preflight() { [[ "$class" == "read" || "$class" == "write" ]] || fail "unsupported brokered class '$class' (expected read/write)" done fi + if [[ -n "${brokered_classes//[[:space:],]/}" && -z "${continuation_proof//[[:space:]]/}" ]]; then + fail "ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF is required when brokered classes are enabled" + fi if [[ "$apply" == "1" && -z "$adapter_image" ]]; then fail "ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE is required with --apply" diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 97d8bb839..b6454a4b1 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -226,6 +226,16 @@ func loadConfig() config { return cfg } +func (c config) validationError() error { + if c.configError != "" { + return errors.New(c.configError) + } + if len(c.brokeredToolClasses) > 0 && strings.TrimSpace(c.continuationProof) == "" { + return fmt.Errorf("%s is required when brokered tool classes are enabled", envContinuationProof) + } + return nil +} + func newServer(cfg config, client *http.Client) *server { if client == nil { client = &http.Client{Timeout: cfg.requestTimeout} @@ -261,7 +271,8 @@ func (s *server) health(w http.ResponseWriter, r *http.Request) { return } _, endpointErr := s.responsesEndpoint() - ready := s.cfg.configError == "" && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + configErr := s.cfg.validationError() + ready := configErr == nil && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) status := harness.HealthStatusOK msg := "ready" if !ready { @@ -269,8 +280,8 @@ func (s *server) health(w http.ResponseWriter, r *http.Request) { parts := []string{ "adapter bearer, safe Foundry hosted Responses endpoint, and exactly one Foundry auth mode are required", } - if s.cfg.configError != "" { - parts = append(parts, s.cfg.configError) + if configErr != nil { + parts = append(parts, configErr.Error()) } if endpointErr != nil { parts = append(parts, endpointErr.Error()) @@ -293,7 +304,8 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { return } _, endpointErr := s.responsesEndpoint() - ready := s.cfg.configError == "" && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + ready := s.cfg.validationError() == nil && s.cfg.adapterBearer != "" && + endpointErr == nil && exactlyOneFoundryAuth(s.cfg) if !ready { harness.WriteError(w, http.StatusServiceUnavailable, "adapter is not ready") return @@ -308,7 +320,8 @@ func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { } modes := []harness.ToolExecutionMode{harness.ToolExecutionModeObserved} maxTurnSeconds := int(s.cfg.requestTimeout.Seconds()) - if len(s.cfg.brokeredToolClasses) > 0 { + brokeredEnabled := len(s.cfg.brokeredToolClasses) > 0 && s.cfg.validationError() == nil + if brokeredEnabled { modes = append(modes, harness.ToolExecutionModeBrokered) // A brokered turn can contain multiple hosted request/tool-result rounds, // each with its own request timeout and approval wait. The harness contract @@ -316,6 +329,10 @@ func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { // understated duration when brokered mode is available. maxTurnSeconds = 0 } + advertisedClasses := []harness.BrokeredToolClass(nil) + if brokeredEnabled { + advertisedClasses = append(advertisedClasses, s.cfg.brokeredToolClasses...) + } harness.WriteJSON(w, http.StatusOK, harness.CapabilitiesResponse{ Version: harness.ProtocolVersion, ProtocolVersion: harness.ProtocolVersion, @@ -324,10 +341,10 @@ func (s *server) capabilities(w http.ResponseWriter, r *http.Request) { RuntimeVersion: "foundry-responses-adapter", ProviderKind: harness.ProviderKindRemote, ToolExecutionModes: modes, - BrokeredToolClasses: append([]harness.BrokeredToolClass(nil), s.cfg.brokeredToolClasses...), + BrokeredToolClasses: advertisedClasses, SupportsCancel: true, SupportsRuntimeSessions: true, - SupportsContinuation: len(s.cfg.brokeredToolClasses) > 0, + SupportsContinuation: brokeredEnabled, SupportsArtifacts: false, MaxConcurrentTurns: 1, MaxTurnSeconds: maxTurnSeconds, @@ -1073,7 +1090,17 @@ func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { if err != nil { return "", fmt.Errorf("tool result %q output must be valid JSON: %w", result.ToolCallID, err) } - output = compacted + if trimmed := bytes.TrimSpace(compacted); len(trimmed) > 0 && trimmed[0] != '{' { + wrapped, err := json.Marshal(struct { + Result json.RawMessage `json:"result"` + }{Result: compacted}) + if err != nil { + return "", fmt.Errorf("wrap tool result %q output: %w", result.ToolCallID, err) + } + output = wrapped + } else { + output = compacted + } } return compactJSON(struct { Approved bool `json:"approved"` @@ -1304,8 +1331,8 @@ func (s *server) foundryRequestContext( } func (s *server) validateStartRequest(req harness.StartTurnRequest) error { - if s.cfg.configError != "" { - return errors.New(s.cfg.configError) + if err := s.cfg.validationError(); err != nil { + return err } if _, err := s.responsesEndpoint(); err != nil { return err diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index f90696fd2..4e1c295e9 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "reflect" + "slices" "strings" "sync" "sync/atomic" @@ -21,7 +22,10 @@ import ( "github.com/orka-agents/orka/internal/harness/conformance" ) -const fakeSessionID = "session-1" +const ( + fakeSessionID = "session-1" + testContinuationProof = "proof-for-test" +) func TestResponsesAdapterObservedTurnCompletes(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) @@ -614,6 +618,47 @@ func TestResponsesAdapterInitialPostHonorsTurnDeadline(t *testing.T) { } } +func TestResponsesAdapterRequiresProofForBrokeredReadinessAndStart(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "observed"}) + server := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: foundry.endpoint(), + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + adapter := httptest.NewServer(server.handler()) + t.Cleanup(adapter.Close) + + response, err := http.Get(adapter.URL + readinessPath) + if err != nil { + t.Fatalf("GET ready: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("ready status = %d, want 503", response.StatusCode) + } + client := newHarnessClient(t, adapter) + capabilities, err := client.Capabilities(context.Background()) + if err != nil { + t.Fatalf("Capabilities: %v", err) + } + if slices.Contains(capabilities.ToolExecutionModes, harness.ToolExecutionModeBrokered) || + capabilities.SupportsContinuation || len(capabilities.BrokeredToolClasses) != 0 { + t.Fatalf("capabilities = %#v, missing proof must disable brokered advertisement", capabilities) + } + request := brokeredReadRequest("foundry-missing-proof") + if _, err := client.StartTurn(context.Background(), request); err == nil || + !strings.Contains(err.Error(), envContinuationProof) { + t.Fatalf("StartTurn missing proof error = %v", err) + } + if got := foundry.postCount.Load(); got != 0 { + t.Fatalf("hosted post count = %d, want no request without continuation proof", got) + } +} + func TestResponsesAdapterReadyEndpointReflectsConfigReadiness(t *testing.T) { unready := httptest.NewServer(newServer(config{ runtimeName: "foundry-responses-test", @@ -936,7 +981,7 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { foundry := newFakeResponses(t, fakeResponsesConfig{ scenario: "function_call", toolName: "support-ticket-lookup", - requiredProof: "proof-for-test", + requiredProof: testContinuationProof, }) s := newServer(config{ runtimeName: "foundry-responses-test", @@ -945,7 +990,7 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: time.Second, stateRetention: time.Minute, - continuationProof: "proof-for-test", + continuationProof: testContinuationProof, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) @@ -965,10 +1010,10 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { t.Fatalf("ContinueTurn: %v", err) } - if got := foundry.requestHeader(1).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "proof-for-test" { + if got := foundry.requestHeader(1).Get("X-AgentKit-Brokered-Continuation-Proof"); got != testContinuationProof { t.Fatalf("continuation proof header = %q, want proof-for-test", got) } - if got := requestMap(t, foundry.requestBody(1))["brokered_continuation_proof"]; got != "proof-for-test" { + if got := requestMap(t, foundry.requestBody(1))["brokered_continuation_proof"]; got != testContinuationProof { t.Fatalf("continuation proof body = %#v, want proof-for-test", got) } if got := foundry.requestHeader(0).Get("X-AgentKit-Brokered-Continuation-Proof"); got != "" { @@ -1418,6 +1463,7 @@ func TestResponsesAdapterBrokeredMaxTurnIsUnknown(t *testing.T) { foundryAuth: "foundry-auth-value", requestTimeout: 2 * time.Second, stateRetention: time.Minute, + continuationProof: testContinuationProof, brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) @@ -2591,6 +2637,31 @@ func TestResponsesFunctionCallWithoutResponseIDFailsBeforeToolRequest(t *testing } } +func TestCanonicalToolResultOutputWrapsNonObjectJSON(t *testing.T) { + tests := []struct { + name string + output json.RawMessage + want string + }{ + {name: "object", output: json.RawMessage(`{"value":1}`), want: `{"approved":true,"output":{"value":1}}`}, + {name: "array", output: json.RawMessage(`[1,2]`), want: `{"approved":true,"output":{"result":[1,2]}}`}, + {name: "string", output: json.RawMessage(`"ok"`), want: `{"approved":true,"output":{"result":"ok"}}`}, + {name: "boolean", output: json.RawMessage(`true`), want: `{"approved":true,"output":{"result":true}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := baseToolResult("call-1", true, tt.output, nil) + got, err := canonicalToolResultOutput(result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + if got != tt.want { + t.Fatalf("canonical output = %s, want %s", got, tt.want) + } + }) + } +} + func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { tests := []struct { name string @@ -2811,6 +2882,10 @@ func newTestResponsesAdapterWithServer( classes []harness.BrokeredToolClass, ) (*httptest.Server, *server) { t.Helper() + continuationProof := "" + if len(classes) > 0 { + continuationProof = testContinuationProof + } s := newServer(config{ addr: ":0", runtimeName: "foundry-responses-test", @@ -2820,6 +2895,7 @@ func newTestResponsesAdapterWithServer( apiVersion: "v1", requestTimeout: time.Second, stateRetention: time.Minute, + continuationProof: continuationProof, brokeredToolClasses: append([]harness.BrokeredToolClass(nil), classes...), }, &http.Client{Timeout: time.Second}) adapter := httptest.NewServer(s.handler()) diff --git a/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json b/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json index 2107d892c..1c67dca1f 100644 --- a/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json +++ b/examples/harness/foundry-responses/testdata/golden/05_hosted_continuation_request.json @@ -1,12 +1,13 @@ { - "previous_response_id": "resp-1", "agent_session_id": "session-1", + "brokered_continuation_proof": "proof-for-test", "input": [ { - "type": "function_call_output", "call_id": "call-1", "output": "{\"approved\":true,\"output\":{\"success\":true}}", - "status": "completed" + "status": "completed", + "type": "function_call_output" } - ] + ], + "previous_response_id": "resp-1" } diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 16a361e04..90f315be6 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -119,6 +119,7 @@ ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ ORKA_FOUNDRY_RESPONSES_ADAPTER_IMAGE="" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES="" \ examples/harness/foundry-responses/live-smoke.sh --apply >/dev/null 2>"$missing_image_err" missing_image_code=$? set -e @@ -130,6 +131,42 @@ if [[ "$missing_image_code" == "0" ]] || ! grep -q "ADAPTER_IMAGE is required" " fi rm -f "$missing_image_err" +missing_proof_err="$(mktemp)" +set +e +ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES="read" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF="" \ + examples/harness/foundry-responses/live-smoke.sh >/dev/null 2>"$missing_proof_err" +missing_proof_code=$? +set -e +if [[ "$missing_proof_code" == "0" ]] || ! grep -q "CONTINUATION_PROOF is required" "$missing_proof_err"; then + cat "$missing_proof_err" >&2 + rm -f "$missing_proof_err" + echo "expected brokered live smoke without continuation proof to fail" >&2 + exit 1 +fi +rm -f "$missing_proof_err" + +whitespace_proof_err="$(mktemp)" +set +e +ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ + ORKA_FOUNDRY_RESPONSES_API_KEY="placeholder" \ + ORKA_FOUNDRY_RESPONSES_AUTH_BEARER="" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_TOOL_CLASSES="read" \ + ORKA_FOUNDRY_RESPONSES_BROKERED_CONTINUATION_PROOF=" " \ + examples/harness/foundry-responses/live-smoke.sh >/dev/null 2>"$whitespace_proof_err" +whitespace_proof_code=$? +set -e +if [[ "$whitespace_proof_code" == "0" ]] || ! grep -q "CONTINUATION_PROOF is required" "$whitespace_proof_err"; then + cat "$whitespace_proof_err" >&2 + rm -f "$whitespace_proof_err" + echo "expected whitespace-only brokered continuation proof to fail" >&2 + exit 1 +fi +rm -f "$whitespace_proof_err" + invalid_class_err="$(mktemp)" set +e ORKA_FOUNDRY_RESPONSES_ENDPOINT="http://127.0.0.1/agents/test/endpoint/protocols/openai/responses" \ From 2512b914d95ebbf842796182b73e530d63a77b19 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 19:37:30 -0700 Subject: [PATCH 45/51] fix: cancel active Foundry requests Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 221 ++++++++---- .../harness/foundry-responses/main_test.go | 315 ++++++++++++++---- internal/controller/harness_wrapper.go | 4 +- internal/controller/harness_wrapper_test.go | 5 + 4 files changed, 414 insertions(+), 131 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index b6454a4b1..2b9cf165f 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -26,19 +26,20 @@ import ( ) const ( - defaultAddr = ":8090" - defaultAPIVersion = "v1" - defaultRequestTimeout = 20 * time.Second - defaultStateRetention = 10 * time.Minute - defaultReadHeaderTimeout = 5 * time.Second - defaultReadTimeout = 30 * time.Second - defaultIdleTimeout = 60 * time.Second - maxFoundryOutputBytes = 1 << 20 - maxFoundryBodyBytes = 4 << 20 - maxConsumedTurnIDs = 1024 - maxBrokeredToolCalls = 32 - readinessPath = "/v1/ready" - foundryInitialUnknown = "foundry_initial_unknown" + defaultAddr = ":8090" + defaultAPIVersion = "v1" + defaultRequestTimeout = 20 * time.Second + defaultStateRetention = 10 * time.Minute + defaultReadHeaderTimeout = 5 * time.Second + defaultReadTimeout = 30 * time.Second + defaultIdleTimeout = 60 * time.Second + maxFoundryOutputBytes = 1 << 20 + maxFoundryBodyBytes = 4 << 20 + maxConsumedTurnIDs = 1024 + maxQuarantinedRuntimeSessions = 1024 + maxBrokeredToolCalls = 32 + readinessPath = "/v1/ready" + foundryInitialUnknown = "foundry_initial_unknown" envAddr = "ORKA_FOUNDRY_RESPONSES_ADAPTER_ADDR" envRuntimeName = "ORKA_FOUNDRY_RESPONSES_RUNTIME_NAME" @@ -77,11 +78,13 @@ type server struct { cfg config client *http.Client - mu sync.Mutex - turns map[harness.HarnessTurnID]*turnState - consumedTurns map[harness.HarnessTurnID]struct{} - consumedOrder []harness.HarnessTurnID - runtimeSessions map[harness.RuntimeSessionID]foundrySession + mu sync.Mutex + turns map[harness.HarnessTurnID]*turnState + consumedTurns map[harness.HarnessTurnID]struct{} + consumedOrder []harness.HarnessTurnID + runtimeSessions map[harness.RuntimeSessionID]foundrySession + quarantinedSessions map[harness.RuntimeSessionID]struct{} + quarantineSaturated bool } type toolResultDigest [sha256.Size]byte @@ -100,21 +103,24 @@ type foundrySession struct { } type turnState struct { - request harness.StartTurnRequest - initializing bool - responseID string - foundrySessionID string - pendingTools map[string]string - suppressedToolCalls map[string]struct{} - requestedToolCalls int - bufferedResults map[string]harness.ToolCallResult - bufferedDigests map[string]toolResultDigest - submittedDigests map[string]toolResultDigest - frames []harness.HarnessEventFrame - completed bool - continuationInFlight bool - frameUpdates chan struct{} - continueMu sync.Mutex + request harness.StartTurnRequest + initializing bool + responseID string + foundrySessionID string + pendingTools map[string]string + suppressedToolCalls map[string]struct{} + requestedToolCalls int + bufferedResults map[string]harness.ToolCallResult + bufferedDigests map[string]toolResultDigest + submittedDigests map[string]toolResultDigest + frames []harness.HarnessEventFrame + completed bool + continuationInFlight bool + hostedRequestSequence uint64 + activeHostedRequestID uint64 + activeHostedRequestCancel context.CancelFunc + frameUpdates chan struct{} + continueMu sync.Mutex } type responsesRequest struct { @@ -247,11 +253,12 @@ func newServer(cfg config, client *http.Client) *server { } } return &server{ - cfg: cfg, - client: &clientCopy, - turns: map[harness.HarnessTurnID]*turnState{}, - consumedTurns: map[harness.HarnessTurnID]struct{}{}, - runtimeSessions: map[harness.RuntimeSessionID]foundrySession{}, + cfg: cfg, + client: &clientCopy, + turns: map[harness.HarnessTurnID]*turnState{}, + consumedTurns: map[harness.HarnessTurnID]struct{}{}, + runtimeSessions: map[harness.RuntimeSessionID]foundrySession{}, + quarantinedSessions: map[harness.RuntimeSessionID]struct{}{}, } } @@ -272,7 +279,11 @@ func (s *server) health(w http.ResponseWriter, r *http.Request) { } _, endpointErr := s.responsesEndpoint() configErr := s.cfg.validationError() - ready := configErr == nil && s.cfg.adapterBearer != "" && endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + s.mu.Lock() + quarantineSaturated := s.quarantineSaturated + s.mu.Unlock() + ready := configErr == nil && s.cfg.adapterBearer != "" && endpointErr == nil && + exactlyOneFoundryAuth(s.cfg) && !quarantineSaturated status := harness.HealthStatusOK msg := "ready" if !ready { @@ -286,6 +297,9 @@ func (s *server) health(w http.ResponseWriter, r *http.Request) { if endpointErr != nil { parts = append(parts, endpointErr.Error()) } + if quarantineSaturated { + parts = append(parts, "runtime session quarantine capacity exhausted; restart required") + } msg = strings.Join(parts, "; ") } harness.WriteJSON(w, http.StatusOK, harness.HealthResponse{ @@ -304,8 +318,11 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { return } _, endpointErr := s.responsesEndpoint() + s.mu.Lock() + quarantineSaturated := s.quarantineSaturated + s.mu.Unlock() ready := s.cfg.validationError() == nil && s.cfg.adapterBearer != "" && - endpointErr == nil && exactlyOneFoundryAuth(s.cfg) + endpointErr == nil && exactlyOneFoundryAuth(s.cfg) && !quarantineSaturated if !ready { harness.WriteError(w, http.StatusServiceUnavailable, "adapter is not ready") return @@ -404,6 +421,24 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteError(w, http.StatusConflict, "turn already completed") return } + if s.quarantineSaturated { + s.mu.Unlock() + harness.WriteError( + w, + http.StatusServiceUnavailable, + "adapter unavailable after runtime session quarantine capacity exhausted", + ) + return + } + if _, quarantined := s.quarantinedSessions[req.RuntimeSessionID]; quarantined { + s.mu.Unlock() + harness.WriteError( + w, + http.StatusGone, + "runtime session unavailable after unconfirmed hosted cancellation", + ) + return + } if s.activeTurnCountLocked() >= 1 { s.mu.Unlock() harness.WriteError(w, http.StatusConflict, "maximum concurrent turns reached") @@ -424,20 +459,43 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { s.mu.Unlock() ctx, cancel := s.foundryRequestContext(r.Context(), req.Deadline) + s.mu.Lock() + if turn.completed { + turn.initializing = false + s.mu.Unlock() + cancel() + harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) + return + } + existingFoundrySessionID := s.runtimeSessions[req.RuntimeSessionID].ID + hostedRequestID := s.activateHostedRequestLocked(turn, cancel) + s.mu.Unlock() defer cancel() var response responsesResponse - initialRequest := responsesRequest{Input: req.Input.Prompt} - foundrySessionID, err := s.postResponses(ctx, req.RuntimeSessionID, initialRequest, &response) + initialRequest := responsesRequest{ + Input: req.Input.Prompt, + AgentSessionID: existingFoundrySessionID, + } + foundrySessionID, err := s.postResponses(ctx, initialRequest, &response) + s.mu.Lock() + s.clearHostedRequestLocked(turn, hostedRequestID) if err != nil { - s.mu.Lock() turn.initializing = false - s.appendFailedLocked( - turn, - foundryInitialUnknown, - "initial hosted request failed after submission was attempted; "+ - "failing closed to avoid a duplicate hosted turn", - ) + s.quarantineRuntimeSessionLocked(turn) + completed := turn.completed + if !completed { + s.appendFailedLocked( + turn, + foundryInitialUnknown, + "initial hosted request failed after submission was attempted; "+ + "failing closed to avoid a duplicate hosted turn", + ) + } s.mu.Unlock() + if completed { + harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) + return + } log.Printf( "Foundry hosted Responses initial request failed after submission for turn %q (error type %T)", req.TurnID, @@ -449,7 +507,6 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { harness.WriteJSON(w, http.StatusAccepted, startTurnResponse(req, eventsPath)) return } - s.mu.Lock() if !turn.completed { disposition := s.handleResponsesResponseLocked(turn, response) if disposition != responseRejected { @@ -614,6 +671,7 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn return } turn.continuationInFlight = true + hostedRequestID := s.activateHostedRequestLocked(turn, cancel) s.mu.Unlock() defer func() { cancel() @@ -627,9 +685,11 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn AgentSessionID: foundrySessionID, Input: outputs, } - updatedSessionID, err := s.postResponses(ctx, req.RuntimeSessionID, continuation, &response) + updatedSessionID, err := s.postResponses(ctx, continuation, &response) + s.mu.Lock() + s.clearHostedRequestLocked(turn, hostedRequestID) if err != nil { - s.mu.Lock() + s.quarantineRuntimeSessionLocked(turn) completed := turn.completed if !completed { s.appendFailedLocked( @@ -652,7 +712,6 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn harness.WriteError(w, http.StatusBadGateway, "hosted continuation failed after submission was attempted") return } - s.mu.Lock() if turn.completed { s.mu.Unlock() harness.WriteError(w, http.StatusConflict, "turn completed while hosted continuation was in flight") @@ -705,8 +764,13 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt harness.WriteError(w, http.StatusBadRequest, "cancel request does not match started turn") return } + var cancelHostedRequest context.CancelFunc s.mu.Lock() if !turn.completed { + cancelHostedRequest = turn.activeHostedRequestCancel + if cancelHostedRequest != nil { + s.quarantineRuntimeSessionLocked(turn) + } s.suppressPendingToolCallsLocked(turn) s.clearBufferedToolResultsLocked(turn) s.appendFrameLocked(turn, harness.FrameTurnCancelled, "turn cancelled") @@ -714,6 +778,9 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt s.scheduleTurnCleanupLocked(turn) } s.mu.Unlock() + if cancelHostedRequest != nil { + cancelHostedRequest() + } harness.WriteJSON( w, http.StatusAccepted, @@ -1126,7 +1193,6 @@ func compactJSON(value any) (string, error) { func (s *server) postResponses( ctx context.Context, - runtimeSessionID harness.RuntimeSessionID, body responsesRequest, out *responsesResponse, ) (string, error) { @@ -1137,13 +1203,7 @@ func (s *server) postResponses( if err != nil { return "", err } - s.mu.Lock() - session := s.runtimeSessions[runtimeSessionID] - s.mu.Unlock() - sessionID := firstNonBlank(body.AgentSessionID, session.ID) - if body.AgentSessionID == "" && sessionID != "" { - body.AgentSessionID = sessionID - } + sessionID := strings.TrimSpace(body.AgentSessionID) // Carry the continuation proof in the body as well as the header, so it // survives hosted-agent gateways (e.g. Foundry) that strip custom request // headers before forwarding to the runtime container. Only on continuations. @@ -1315,6 +1375,45 @@ func exactlyOneFoundryAuth(cfg config) bool { return hasKey != hasBearer } +func (s *server) quarantineRuntimeSessionLocked(turn *turnState) { + // A local request failure or cancellation does not prove that Foundry + // stopped an already accepted operation. Invalidate and quarantine this + // logical runtime session for the adapter process lifetime so no later turn + // can overlap with or inherit state from an uncertain hosted outcome. + sessionID := turn.request.RuntimeSessionID + delete(s.runtimeSessions, sessionID) + if s.quarantineSaturated { + return + } + if _, exists := s.quarantinedSessions[sessionID]; exists { + return + } + if len(s.quarantinedSessions) >= maxQuarantinedRuntimeSessions { + // Keep memory bounded and fail closed globally until process restart. + s.quarantineSaturated = true + return + } + s.quarantinedSessions[sessionID] = struct{}{} +} + +func (s *server) activateHostedRequestLocked( + turn *turnState, + cancel context.CancelFunc, +) uint64 { + turn.hostedRequestSequence++ + turn.activeHostedRequestID = turn.hostedRequestSequence + turn.activeHostedRequestCancel = cancel + return turn.activeHostedRequestID +} + +func (s *server) clearHostedRequestLocked(turn *turnState, requestID uint64) { + if turn.activeHostedRequestID != requestID { + return + } + turn.activeHostedRequestID = 0 + turn.activeHostedRequestCancel = nil +} + func (s *server) foundryRequestContext( parent context.Context, turnDeadline time.Time, diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 4e1c295e9..95990cdaa 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -216,7 +216,6 @@ func TestResponsesAdapterInterleavedResponsesRetainResponseSpecificSession(t *te var firstResponse responsesResponse firstSession, err := server.postResponses( context.Background(), - firstRequest.RuntimeSessionID, responsesRequest{Input: "a"}, &firstResponse, ) @@ -226,7 +225,6 @@ func TestResponsesAdapterInterleavedResponsesRetainResponseSpecificSession(t *te var secondResponse responsesResponse secondSession, err := server.postResponses( context.Background(), - secondRequest.RuntimeSessionID, responsesRequest{Input: "b"}, &secondResponse, ) @@ -435,27 +433,47 @@ func TestResponsesAdapterRejectedResponseDoesNotRetainSession(t *testing.T) { } } -func TestResponsesAdapterCancelDuringInitialPostDoesNotRetainSession(t *testing.T) { +func TestResponsesAdapterCancelDuringInitialPostCancelsHostedRequest(t *testing.T) { received := make(chan struct{}) - release := make(chan struct{}) + hostedRequestCancelled := make(chan struct{}) + releaseHostedResponse := make(chan struct{}) var releaseOnce sync.Once - releaseFoundry := func() { releaseOnce.Do(func() { close(release) }) } - foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - select { - case <-received: - default: - close(received) + releaseResponse := func() { releaseOnce.Do(func() { close(releaseHostedResponse) }) } + var postCount atomic.Int32 + backendClient := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if postCount.Add(1) > 1 { + return jsonHTTPResponse(r, finalResponsesMessage()) } - <-release - w.Header().Set("x-agent-session-id", "cancelled-session") - writeJSON(w, finalResponsesMessage()) - })) - t.Cleanup(foundry.Close) - endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" - adapter, server := newTestResponsesAdapterWithServer(t, endpoint, nil) - t.Cleanup(releaseFoundry) + body, err := ioReadAll(r.Body) + if err != nil { + return nil, err + } + var decoded map[string]any + if err := json.Unmarshal(body, &decoded); err != nil { + return nil, err + } + if got := decoded["agent_session_id"]; got != "existing-session" { + return nil, fmt.Errorf("agent_session_id = %#v, want existing-session", got) + } + close(received) + <-r.Context().Done() + close(hostedRequestCancelled) + // Model a hosted service that accepted the request before client-side + // cancellation and later returns success despite the canceled context. + <-releaseHostedResponse + return jsonHTTPResponseWithSession(r, finalResponsesMessage(), "cancelled-session") + })} + endpoint := "https://foundry.example/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter, server := newTestResponsesAdapterWithHTTPClient(t, endpoint, nil, backendClient) + t.Cleanup(releaseResponse) client := newHarnessClient(t, adapter) request := responsesStartTurnRequest("foundry-cancel-initial") + server.mu.Lock() + server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ + ID: "existing-session", + LastSeen: time.Now().UTC(), + } + server.mu.Unlock() startErr := make(chan error, 1) go func() { @@ -470,40 +488,53 @@ func TestResponsesAdapterCancelDuringInitialPostDoesNotRetainSession(t *testing. if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { t.Fatalf("CancelTurn: %v", err) } - second := responsesStartTurnRequest("foundry-after-cancel-initial") - second.RuntimeSessionID = request.RuntimeSessionID - if _, err := client.StartTurn(context.Background(), second); err == nil || - !strings.Contains(err.Error(), "maximum concurrent turns reached") { - t.Fatalf("StartTurn during cancelled in-flight request error = %v, want admission rejection", err) + select { + case <-hostedRequestCancelled: + case <-time.After(time.Second): + t.Fatal("cancellation did not cancel the initial hosted request context") } - releaseFoundry() + releaseResponse() select { case err := <-startErr: if err != nil { t.Fatalf("StartTurn after cancellation: %v", err) } case <-time.After(time.Second): - t.Fatal("StartTurn did not finish after releasing Foundry response") + t.Fatal("StartTurn did not finish after the late hosted response") } server.mu.Lock() turn := server.turns[request.TurnID] _, sessionRetained := server.runtimeSessions[request.RuntimeSessionID] + _, sessionQuarantined := server.quarantinedSessions[request.RuntimeSessionID] foundrySessionID := "" if turn != nil { foundrySessionID = turn.foundrySessionID } server.mu.Unlock() - if turn == nil || !turn.completed || !hasFrameType(turn.frames, harness.FrameTurnCancelled) { + if turn == nil || !turn.completed || !hasFrameType(turn.frames, harness.FrameTurnCancelled) || + hasFrameType(turn.frames, harness.FrameTurnCompleted) { t.Fatalf("turn = %#v, want retained terminal cancellation", turn) } - if sessionRetained || foundrySessionID != "" { + if sessionRetained || foundrySessionID != "" || !sessionQuarantined { t.Fatalf( - "cancelled initial post retained session map=%v turnSession=%q", + "cancelled initial post retained session map=%v turnSession=%q quarantined=%v", sessionRetained, foundrySessionID, + sessionQuarantined, ) } + + quarantined := responsesStartTurnRequest("foundry-quarantined-after-cancel-initial") + quarantined.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), quarantined); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with quarantined runtime session error = %v", err) + } + second := responsesStartTurnRequest("foundry-after-cancel-initial") + if _, err := client.StartTurn(context.Background(), second); err != nil { + t.Fatalf("StartTurn with a different runtime session after cancellation: %v", err) + } } func TestResponsesAdapterInitialPostSurvivesControlDisconnect(t *testing.T) { @@ -565,6 +596,55 @@ func TestResponsesAdapterInitialPostSurvivesControlDisconnect(t *testing.T) { } } +func TestResponsesAdapterQuarantineCapacityFailsClosed(t *testing.T) { + server := newServer(config{ + runtimeName: "foundry-responses-test", + adapterBearer: "adapter-auth-value", + endpoint: "https://foundry.example/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1", + foundryAuth: "foundry-auth-value", + requestTimeout: time.Second, + stateRetention: time.Minute, + }, &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("hosted request should not be attempted after quarantine saturation") + })}) + server.mu.Lock() + for i := range maxQuarantinedRuntimeSessions { + server.quarantineRuntimeSessionLocked(&turnState{request: responsesStartTurnRequest(fmt.Sprintf("quarantine-%d", i))}) + } + if got := len(server.quarantinedSessions); got != maxQuarantinedRuntimeSessions { + server.mu.Unlock() + t.Fatalf("quarantine count = %d, want %d", got, maxQuarantinedRuntimeSessions) + } + if server.quarantineSaturated { + server.mu.Unlock() + t.Fatal("quarantine saturated before exceeding the bounded tombstone capacity") + } + server.quarantineRuntimeSessionLocked(&turnState{request: responsesStartTurnRequest("quarantine-overflow")}) + gotCount := len(server.quarantinedSessions) + saturated := server.quarantineSaturated + server.mu.Unlock() + if gotCount != maxQuarantinedRuntimeSessions || !saturated { + t.Fatalf("quarantine count=%d saturated=%v, want bounded saturated state", gotCount, saturated) + } + + adapter := httptest.NewServer(server.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + afterSaturation := responsesStartTurnRequest("after-quarantine-saturation") + if _, err := client.StartTurn(context.Background(), afterSaturation); err == nil || + !strings.Contains(err.Error(), "quarantine capacity exhausted") { + t.Fatalf("StartTurn after quarantine saturation error = %v", err) + } + response, err := http.Get(adapter.URL + readinessPath) //nolint:gosec // Test-only loopback server. + if err != nil { + t.Fatalf("GET readiness: %v", err) + } + defer response.Body.Close() //nolint:errcheck + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("readiness status = %d, want 503 after quarantine saturation", response.StatusCode) + } +} + func TestResponsesAdapterFoundryRequestContextDetachesAndUsesEarlierTurnDeadline(t *testing.T) { server := newServer(config{requestTimeout: time.Second}, &http.Client{Timeout: time.Second}) parent, cancelParent := context.WithCancel(context.Background()) @@ -599,7 +679,7 @@ func TestResponsesAdapterInitialPostHonorsTurnDeadline(t *testing.T) { })) t.Cleanup(foundry.Close) endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" - adapter := newTestResponsesAdapter(t, endpoint, nil) + adapter, server := newTestResponsesAdapterWithServer(t, endpoint, nil) client := newHarnessClient(t, adapter) request := responsesStartTurnRequest("foundry-initial-deadline") request.Deadline = time.Now().Add(150 * time.Millisecond) @@ -616,6 +696,18 @@ func TestResponsesAdapterInitialPostHonorsTurnDeadline(t *testing.T) { failed.Failed.Reason != foundryInitialUnknown { t.Fatalf("failed frame = %#v, want foundry_initial_unknown", failed) } + server.mu.Lock() + _, quarantined := server.quarantinedSessions[request.RuntimeSessionID] + server.mu.Unlock() + if !quarantined { + t.Fatal("deadline-uncertain initial request did not quarantine its runtime session") + } + retry := responsesStartTurnRequest("foundry-initial-deadline-retry") + retry.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), retry); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with deadline-quarantined runtime session error = %v", err) + } } func TestResponsesAdapterRequiresProofForBrokeredReadinessAndStart(t *testing.T) { @@ -1024,46 +1116,52 @@ func TestResponsesAdapterSendsBrokeredContinuationProof(t *testing.T) { } } -func TestResponsesAdapterCancelDuringHostedContinuationWins(t *testing.T) { +func TestResponsesAdapterCancelDuringHostedContinuationCancelsRequest(t *testing.T) { continuationReceived := make(chan struct{}) - releaseContinuation := make(chan struct{}) + hostedRequestCancelled := make(chan struct{}) + releaseHostedResponse := make(chan struct{}) var releaseOnce sync.Once - releaseFoundry := func() { releaseOnce.Do(func() { close(releaseContinuation) }) } - foundry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + releaseResponse := func() { releaseOnce.Do(func() { close(releaseHostedResponse) }) } + backendClient := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { body, err := ioReadAll(r.Body) if err != nil { - http.Error(w, "read body", http.StatusBadRequest) - return + return nil, err } var decoded map[string]any if err := json.Unmarshal(body, &decoded); err != nil { - http.Error(w, "decode body", http.StatusBadRequest) - return + return nil, err } if _, continuing := decoded["previous_response_id"]; !continuing { - w.Header().Set("x-agent-session-id", fakeSessionID) - writeJSON(w, functionCallResponse("support-ticket-lookup")) - return - } - select { - case <-continuationReceived: - default: - close(continuationReceived) + return jsonHTTPResponseWithSession( + r, + functionCallResponse("support-ticket-lookup"), + "existing-session", + ) } - <-releaseContinuation - w.Header().Set("x-agent-session-id", "session-after-cancel") - writeJSON(w, finalResponsesMessage()) - })) - t.Cleanup(foundry.Close) - endpoint := foundry.URL + "/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" - adapter, server := newTestResponsesAdapterWithServer( + close(continuationReceived) + <-r.Context().Done() + close(hostedRequestCancelled) + // Model a hosted continuation that ignores client cancellation and + // eventually reports success with a replacement session identifier. + <-releaseHostedResponse + return jsonHTTPResponseWithSession(r, finalResponsesMessage(), "session-after-cancel") + })} + endpoint := "https://foundry.example/agents/test-agent/endpoint/protocols/openai/responses?api-version=v1" + adapter, server := newTestResponsesAdapterWithHTTPClient( t, endpoint, []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + backendClient, ) - t.Cleanup(releaseFoundry) + t.Cleanup(releaseResponse) client := newHarnessClient(t, adapter) request := brokeredReadRequest("foundry-cancel-continuation") + server.mu.Lock() + server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ + ID: "existing-session", + LastSeen: time.Now().UTC(), + } + server.mu.Unlock() if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } @@ -1073,10 +1171,10 @@ func TestResponsesAdapterCancelDuringHostedContinuationWins(t *testing.T) { t.Fatalf("frames = %#v, want tool request", frames) } server.mu.Lock() - _, publishedBeforeCompletion := server.runtimeSessions[request.RuntimeSessionID] + publishedBeforeCompletion := server.runtimeSessions[request.RuntimeSessionID] server.mu.Unlock() - if publishedBeforeCompletion { - t.Fatal("pending function call published runtime session before completion") + if publishedBeforeCompletion.ID != "existing-session" { + t.Fatalf("runtime session before continuation = %#v, want existing session", publishedBeforeCompletion) } continueRequest := goldenContinueRequest(request, requested.ToolCallID, json.RawMessage(`{"success":true}`)) @@ -1093,30 +1191,51 @@ func TestResponsesAdapterCancelDuringHostedContinuationWins(t *testing.T) { if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { t.Fatalf("CancelTurn during continuation: %v", err) } - second := responsesStartTurnRequest("foundry-after-cancel-continuation") - second.RuntimeSessionID = request.RuntimeSessionID - if _, err := client.StartTurn(context.Background(), second); err == nil || - !strings.Contains(err.Error(), "maximum concurrent turns reached") { - t.Fatalf("StartTurn during cancelled continuation error = %v, want admission rejection", err) + select { + case <-hostedRequestCancelled: + case <-time.After(time.Second): + t.Fatal("cancellation did not cancel the hosted continuation request context") } - releaseFoundry() + server.mu.Lock() + _, publishedAfterCancellation := server.runtimeSessions[request.RuntimeSessionID] + _, sessionQuarantined := server.quarantinedSessions[request.RuntimeSessionID] + server.mu.Unlock() + if publishedAfterCancellation || !sessionQuarantined { + t.Fatalf( + "runtime session after cancellation published=%v quarantined=%v", + publishedAfterCancellation, + sessionQuarantined, + ) + } + releaseResponse() select { case err := <-continueErr: if err == nil || !strings.Contains(err.Error(), "turn completed while hosted continuation was in flight") { t.Fatalf("ContinueTurn after cancellation error = %v", err) } case <-time.After(time.Second): - t.Fatal("ContinueTurn did not finish after releasing hosted response") + t.Fatal("ContinueTurn did not finish after the late hosted response") + } + + quarantined := responsesStartTurnRequest("foundry-quarantined-after-cancel-continuation") + quarantined.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), quarantined); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with quarantined runtime session error = %v", err) + } + second := responsesStartTurnRequest("foundry-after-cancel-continuation") + if _, err := client.StartTurn(context.Background(), second); err != nil { + t.Fatalf("StartTurn with a different runtime session after cancellation: %v", err) } frames = streamCurrentFrames(t, client, request.TurnID) if !hasFrameType(frames, harness.FrameTurnCancelled) || hasFrameType(frames, harness.FrameTurnCompleted) { t.Fatalf("frames = %#v, want cancellation to win continuation race", frames) } server.mu.Lock() - _, publishedAfterCancel := server.runtimeSessions[request.RuntimeSessionID] + _, publishedAfterLateResponse := server.runtimeSessions[request.RuntimeSessionID] server.mu.Unlock() - if publishedAfterCancel { - t.Fatal("cancelled continuation published runtime session") + if publishedAfterLateResponse { + t.Fatal("late continuation response restored the cancelled runtime session") } } @@ -1282,7 +1401,7 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t toolName: "support-ticket-lookup", continuationStatus: http.StatusInternalServerError, }) - adapter := newTestResponsesAdapter( + adapter, server := newTestResponsesAdapterWithServer( t, foundry.endpoint(), []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, @@ -1324,6 +1443,18 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t strings.Contains(failed.Failed.Message, "HTTP 500") { t.Fatalf("failed frame leaked upstream detail: %#v", failed.Failed) } + server.mu.Lock() + _, quarantined := server.quarantinedSessions[request.RuntimeSessionID] + server.mu.Unlock() + if !quarantined { + t.Fatal("uncertain continuation failure did not quarantine its runtime session") + } + retry := responsesStartTurnRequest("foundry-continuation-failure-retry") + retry.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), retry); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with continuation-quarantined runtime session error = %v", err) + } } func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { @@ -1560,7 +1691,6 @@ func TestPostResponsesRejectsOversizedAndTrailingBodies(t *testing.T) { var response responsesResponse _, err := server.postResponses( context.Background(), - "runtime-session", responsesRequest{Input: "test"}, &response, ) @@ -2880,6 +3010,21 @@ func newTestResponsesAdapterWithServer( t *testing.T, endpoint string, classes []harness.BrokeredToolClass, +) (*httptest.Server, *server) { + t.Helper() + return newTestResponsesAdapterWithHTTPClient( + t, + endpoint, + classes, + &http.Client{Timeout: time.Second}, + ) +} + +func newTestResponsesAdapterWithHTTPClient( + t *testing.T, + endpoint string, + classes []harness.BrokeredToolClass, + backendClient *http.Client, ) (*httptest.Server, *server) { t.Helper() continuationProof := "" @@ -2897,12 +3042,44 @@ func newTestResponsesAdapterWithServer( stateRetention: time.Minute, continuationProof: continuationProof, brokeredToolClasses: append([]harness.BrokeredToolClass(nil), classes...), - }, &http.Client{Timeout: time.Second}) + }, backendClient) adapter := httptest.NewServer(s.handler()) t.Cleanup(adapter.Close) return adapter, s } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +func jsonHTTPResponse(r *http.Request, value any) (*http.Response, error) { + payload, err := json.Marshal(value) + if err != nil { + return nil, err + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(payload)), + Request: r, + }, nil +} + +func jsonHTTPResponseWithSession( + r *http.Request, + value any, + sessionID string, +) (*http.Response, error) { + response, err := jsonHTTPResponse(r, value) + if err != nil { + return nil, err + } + response.Header.Set("x-agent-session-id", sessionID) + return response, nil +} + func newHarnessClient(t *testing.T, adapter *httptest.Server) *harness.Client { t.Helper() client, err := harness.NewClient( diff --git a/internal/controller/harness_wrapper.go b/internal/controller/harness_wrapper.go index 524066f39..5ae9268c7 100644 --- a/internal/controller/harness_wrapper.go +++ b/internal/controller/harness_wrapper.go @@ -1402,7 +1402,9 @@ func harnessWrapperStartTurnErrorIsRetryable(err error) bool { return false } message := err.Error() - for _, marker := range []string{"(400)", "(401)", "(403)", "unsupported version", "harness did not accept"} { + for _, marker := range []string{ + "(400)", "(401)", "(403)", "(410)", "unsupported version", "harness did not accept", + } { if strings.Contains(message, marker) { return false } diff --git a/internal/controller/harness_wrapper_test.go b/internal/controller/harness_wrapper_test.go index 43c3f32af..6fb3480f1 100644 --- a/internal/controller/harness_wrapper_test.go +++ b/internal/controller/harness_wrapper_test.go @@ -2724,6 +2724,11 @@ func TestHarnessWrapperStartTurnErrorClassification(t *testing.T) { if harnessWrapperStartTurnErrorIsRetryable(fmt.Errorf("start_turn failed (401): unauthorized")) { t.Fatal("expected auth start error to remain terminal") } + if harnessWrapperStartTurnErrorIsRetryable( + fmt.Errorf("start_turn failed (410): runtime session unavailable after unconfirmed hosted cancellation"), + ) { + t.Fatal("expected quarantined runtime session error to remain terminal") + } } func TestHarnessWrapperTurnMetadataDefaultsMaxTurns(t *testing.T) { From 7c9da8a298a9c2cd1fed893db4f433c60878805c Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 19:46:02 -0700 Subject: [PATCH 46/51] fix: keep lost harness turns terminal Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 2 +- .../harness/foundry-responses/main_test.go | 31 +++++++++++++++++++ internal/controller/harness_wrapper.go | 15 ++++++--- internal/controller/harness_wrapper_test.go | 8 +++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 2b9cf165f..62fafe163 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -812,7 +812,7 @@ func (s *server) handleResponsesResponseLocked(turn *turnState, response respons s.appendFailedLocked( turn, "foundry_response_error", - firstNonBlank(response.Error.Message, response.Error.Code, "Foundry hosted Responses returned an error"), + "Foundry hosted Responses returned an error", ) return responseRejected } diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 95990cdaa..04755bcf9 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -1457,6 +1457,37 @@ func TestResponsesAdapterContinuationFailureFailsClosedWithoutDuplicatePost(t *t } } +func TestResponsesAdapterHostedErrorDoesNotExposeUpstreamDiagnostics(t *testing.T) { + server := newServer(config{}, &http.Client{Timeout: time.Second}) + request := responsesStartTurnRequest("foundry-safe-hosted-error") + turn := &turnState{request: request, frameUpdates: make(chan struct{})} + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + secret := "proof-that-must-not-reach-task-events" + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-error", + Status: "failed", + Error: &responsesError{ + Code: secret, + Message: "upstream echoed brokered_continuation_proof=" + secret, + }, + }) + failed := findFrame(turn.frames, harness.FrameTurnFailed) + if failed == nil || failed.Failed == nil || failed.Error == nil { + t.Fatalf("failed frame = %#v, want safe terminal failure", failed) + } + if failed.Failed.Reason != "foundry_response_error" || + failed.Failed.Message != "Foundry hosted Responses returned an error" { + t.Fatalf("failed frame = %#v, want fixed safe upstream error", failed) + } + payload, err := json.Marshal(failed) + if err != nil { + t.Fatalf("marshal failed frame: %v", err) + } + if strings.Contains(string(payload), secret) { + t.Fatalf("failed frame leaked upstream diagnostics: %s", payload) + } +} + func TestResponsesRepeatedSubmittedFunctionCallFailsTurn(t *testing.T) { server := newServer(config{ runtimeName: "test", diff --git a/internal/controller/harness_wrapper.go b/internal/controller/harness_wrapper.go index 5ae9268c7..62074026f 100644 --- a/internal/controller/harness_wrapper.go +++ b/internal/controller/harness_wrapper.go @@ -1457,13 +1457,18 @@ func harnessWrapperStreamErrorIsMissingTurn(err error) bool { if err == nil { return false } + var clientErr harness.ClientError + if errors.As(err, &clientErr) && clientErr.StatusCode > 0 { + return clientErr.StatusCode == 404 + } message := err.Error() - for _, marker := range []string{"(404)", "turn not found"} { - if strings.Contains(message, marker) { - return true - } + if strings.Contains(message, "(404)") { + return true } - return false + if strings.Contains(message, "(410)") { + return false + } + return strings.Contains(message, "turn not found") } func harnessWrapperCancelErrorIsMissingTurn(err error) bool { diff --git a/internal/controller/harness_wrapper_test.go b/internal/controller/harness_wrapper_test.go index 6fb3480f1..c129add22 100644 --- a/internal/controller/harness_wrapper_test.go +++ b/internal/controller/harness_wrapper_test.go @@ -2577,8 +2577,16 @@ func TestHarnessWrapperStreamMissingTurnErrorClassification(t *testing.T) { if !harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("stream_frames failed (404): turn not found")) { t.Fatal("404 turn-not-found stream error should be classified as retryable missing turn") } + if harnessWrapperStreamErrorIsMissingTurn(harness.ClientError{ + Op: "stream_frames", + StatusCode: http.StatusGone, + Message: "turn not found", + }) { + t.Fatal("typed 410 turn-not-found stream error should remain terminal") + } for _, message := range []string{ "stream_frames failed (410): terminal turn expired from runtime retention", + "stream_frames failed (410): turn not found", "stream_frames failed (401): unauthorized", } { if harnessWrapperStreamErrorIsMissingTurn(fmt.Errorf("%s", message)) { From b1fe290b1d1eaf119d1f7142ee694ad3e5b7c924 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 20:01:42 -0700 Subject: [PATCH 47/51] fix: harden Foundry continuation evidence Signed-off-by: Sertac Ozercan --- ...onses-events-generic-idempotency-only.json | 66 +++++++++++++++++++ .../verify-foundry-responses.sh | 36 +++++----- examples/harness/foundry-responses/README.md | 7 +- examples/harness/foundry-responses/main.go | 18 ++++- .../harness/foundry-responses/main_test.go | 22 ++++++- .../approval_declined_payload.json | 2 +- .../tool_execution_failure_payload.json | 2 +- .../tool_policy_rejection_payload.json | 2 +- .../golden/07_approval_declined_output.json | 2 +- .../08_tool_policy_rejection_output.json | 2 +- .../09_tool_execution_failure_output.json | 2 +- .../harness/foundry-responses/validate.sh | 6 +- 12 files changed, 136 insertions(+), 31 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json new file mode 100644 index 000000000..edb1e5768 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json @@ -0,0 +1,66 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 3, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 4, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 5, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 6, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 7, + "eventType": "TaskSucceeded" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 1f7fca1a2..5ae1536d4 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -14,7 +14,7 @@ Expected evidence: - read brokered tool request for check-network-telemetry or get-active-incidents - write brokered tool request for dispatch-work-order or escalate-incident - matching ApprovalRequested and ApprovalApproved events precede write execution - - an idempotency key is present in the write execution ledger event + - an executionIdempotencyKey is present in the write execution ledger event - terminal TaskSucceeded/AgentRuntimeCompleted/TurnCompleted-style event exists This verifier does not approve tasks and never reads Foundry credentials. @@ -189,18 +189,18 @@ def seq(event): return None -def idempotency_value(value): +def execution_idempotency_value(value): if isinstance(value, dict): for key, nested in value.items(): - if key in {"idempotencyKey", "Idempotency-Key"}: + if key in {"executionIdempotencyKey", "Execution-Idempotency-Key"}: if isinstance(nested, str) and nested.strip(): return nested.strip() - found = idempotency_value(nested) + found = execution_idempotency_value(nested) if found: return found elif isinstance(value, list): for item in value: - found = idempotency_value(item) + found = execution_idempotency_value(item) if found: return found elif isinstance(value, str): @@ -208,7 +208,7 @@ def idempotency_value(value): decoded = json.loads(value) except Exception: # noqa: BLE001 return "" - return idempotency_value(decoded) + return execution_idempotency_value(decoded) return "" @@ -267,7 +267,7 @@ write_exec_events = [e for e in write_events if is_write_execution_start(e)] write_start_events = write_exec_events terminal_events = [e for e in events if event_type(e) in TERMINAL_TYPES] task_terminal_events = [e for e in events if event_type(e) in TASK_TERMINAL_TYPES] -idempotency_events = [e for e in write_exec_events if idempotency_value(e)] +execution_idempotency_events = [e for e in write_exec_events if execution_idempotency_value(e)] failures = [] if not read_events: @@ -284,8 +284,8 @@ if write_exec_events: for event in write_exec_events: write_tool = tool_name(event) write_order = seq(event) - if not idempotency_value(event): - failures.append(f"write execution for {write_tool} is missing idempotency key evidence") + if not execution_idempotency_value(event): + failures.append(f"write execution for {write_tool} is missing execution idempotency key evidence") write_tool_call_id = tool_call_id(event) if not write_tool_call_id: failures.append(f"write execution for {write_tool} is missing toolCallID") @@ -327,12 +327,12 @@ if write_exec_events: ] if matching_declined: failures.append(f"write execution for {write_tool} follows ApprovalDeclined") -if not idempotency_events: - failures.append("missing write ToolCallStarted idempotency key evidence") +if not execution_idempotency_events: + failures.append("missing write ToolCallStarted execution idempotency key evidence") missing_idempotency_tools = sorted( tool for tool in {tool_name(event) for event in write_exec_events} - if tool not in {tool_name(event) for event in idempotency_events} + if tool not in {tool_name(event) for event in execution_idempotency_events} ) if missing_idempotency_tools: failures.append( @@ -347,12 +347,12 @@ for write_tool, count in starts_by_tool.items(): if count > 1: failures.append(f"duplicate write execution starts for {write_tool}") -idempotency_by_tool = {} -for event in idempotency_events: - idempotency_by_tool.setdefault(tool_name(event), set()).add(idempotency_value(event)) -for write_tool, keys in idempotency_by_tool.items(): +execution_idempotency_by_tool = {} +for event in execution_idempotency_events: + execution_idempotency_by_tool.setdefault(tool_name(event), set()).add(execution_idempotency_value(event)) +for write_tool, keys in execution_idempotency_by_tool.items(): if len(keys) > 1: - failures.append(f"multiple write idempotency keys for {write_tool}") + failures.append(f"multiple write execution idempotency keys for {write_tool}") if not terminal_events: failures.append("missing terminal completion event") elif write_exec_events: @@ -383,6 +383,6 @@ print(f"- read events: {len(read_events)}") print(f"- write requests: {len(write_request_events)}") print(f"- approval requests: {len(approval_request_events)}") print(f"- approval decisions: {len(approval_approved_events)}") -print(f"- idempotency evidence events: {len(idempotency_events)}") +print(f"- execution idempotency evidence events: {len(execution_idempotency_events)}") print(f"- terminal events: {len(terminal_events)}") PY diff --git a/examples/harness/foundry-responses/README.md b/examples/harness/foundry-responses/README.md index 80181f805..e17f68282 100644 --- a/examples/harness/foundry-responses/README.md +++ b/examples/harness/foundry-responses/README.md @@ -86,10 +86,11 @@ The hosted continuation request includes `previous_response_id`, `agent_session_ `function_call_output.output` is always a compact JSON string: -- successful tool result: `{"approved":true,"output":}` -- declined approval or policy/execution error: `{"approved":false,"error":}` +- successful object result: `{"approved":true,"output":}` +- successful array or scalar result: `{"approved":true,"output":{"result":}}` +- declined approval or policy/execution error: `{"approved":false,"error":}` -Approval decline, tool policy rejection, and tool execution failure fixtures live under `testdata/golden/`. +Object outputs remain unchanged. Arrays and scalars are wrapped under `output.result` so the AgentKit continuation always receives an object-shaped `output`. Error codes are allowlisted and messages are replaced with stable generic text before crossing the hosted-provider boundary. Approval decline, tool policy rejection, and tool execution failure fixtures live under `testdata/golden/`. ## State, restart, and sessions diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 62fafe163..4a15bf1b5 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -1143,7 +1143,7 @@ func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { return compactJSON(struct { Approved bool `json:"approved"` Error *harness.ErrorInfo `json:"error"` - }{Approved: false, Error: result.Error}) + }{Approved: false, Error: hostedToolError(result.Error)}) } if !result.Approved { return compactJSON(struct { @@ -1175,6 +1175,22 @@ func canonicalToolResultOutput(result harness.ToolCallResult) (string, error) { }{Approved: true, Output: output}) } +func hostedToolError(err *harness.ErrorInfo) *harness.ErrorInfo { + if err == nil { + return nil + } + switch strings.TrimSpace(err.Code) { + case "approval_declined": + return &harness.ErrorInfo{Code: "approval_declined", Message: "tool call was not approved"} + case "tool_policy_rejected": + return &harness.ErrorInfo{Code: "tool_policy_rejected", Message: "tool call was rejected by policy"} + case "tool_execution_failed": + return &harness.ErrorInfo{Code: "tool_execution_failed", Message: "tool execution failed"} + default: + return &harness.ErrorInfo{Code: "tool_execution_failed", Message: "tool execution failed"} + } +} + func compactRawJSON(raw json.RawMessage) (json.RawMessage, error) { var compacted bytes.Buffer if err := json.Compact(&compacted, raw); err != nil { diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 04755bcf9..81d900dd3 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -924,7 +924,7 @@ func TestResponsesAdapterWriteParksUntilDeclinedApprovalContinue(t *testing.T) { if !ok { t.Fatalf("continuation item = %#v, want object", items[0]) } - wantOutput := `{"approved":false,"error":{"code":"approval_declined","message":"human declined"}}` + wantOutput := `{"approved":false,"error":{"code":"approval_declined","message":"tool call was not approved"}}` if got := item["output"]; got != wantOutput { t.Fatalf("declined output = %#v, want %s", got, wantOutput) } @@ -1381,7 +1381,7 @@ func TestResponsesAdapterContinuesToolExecutionFailurePayload(t *testing.T) { if !ok { t.Fatalf("continuation item = %#v, want object", items[0]) } - wantOutput := `{"approved":false,"error":{"code":"tool_execution_failed","message":"downstream failed"}}` + wantOutput := `{"approved":false,"error":{"code":"tool_execution_failed","message":"tool execution failed"}}` if got := item["output"]; got != wantOutput { t.Fatalf("failure output = %#v, want %s", got, wantOutput) } @@ -2823,6 +2823,24 @@ func TestCanonicalToolResultOutputWrapsNonObjectJSON(t *testing.T) { } } +func TestCanonicalToolResultErrorUsesSafeHostedEnvelope(t *testing.T) { + result := baseToolResult("call-1", true, nil, &harness.ErrorInfo{ + Code: "transport_failed", + Message: "POST https://tools.internal.example/v1/run: connection refused", + }) + got, err := canonicalToolResultOutput(result) + if err != nil { + t.Fatalf("canonicalToolResultOutput: %v", err) + } + want := `{"approved":false,"error":{"code":"tool_execution_failed","message":"tool execution failed"}}` + if got != want { + t.Fatalf("canonical error output = %s, want %s", got, want) + } + if strings.Contains(got, "tools.internal.example") || strings.Contains(got, "https://") { + t.Fatalf("canonical error output leaked tool URL: %s", got) + } +} + func TestCanonicalErrorAndDeclineOutputFixtures(t *testing.T) { tests := []struct { name string diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json index facb8ed7b..07aef6b58 100644 --- a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/approval_declined_payload.json @@ -2,6 +2,6 @@ "approved": false, "error": { "code": "approval_declined", - "message": "Human declined dispatch-work-order" + "message": "tool call was not approved" } } diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json index b0955fef2..dc529a152 100644 --- a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_execution_failure_payload.json @@ -2,6 +2,6 @@ "approved": false, "error": { "code": "tool_execution_failed", - "message": "downstream timed out" + "message": "tool execution failed" } } diff --git a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json index d32368fb6..6ce46ae57 100644 --- a/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json +++ b/examples/harness/foundry-responses/testdata/agentkit-foundry-brokered/tool_policy_rejection_payload.json @@ -2,6 +2,6 @@ "approved": false, "error": { "code": "tool_policy_rejected", - "message": "writes are disabled" + "message": "tool call was rejected by policy" } } diff --git a/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json b/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json index 645b97fd1..b0cadb210 100644 --- a/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json +++ b/examples/harness/foundry-responses/testdata/golden/07_approval_declined_output.json @@ -1,6 +1,6 @@ { "type": "function_call_output", "call_id": "call-1", - "output": "{\"approved\":false,\"error\":{\"code\":\"approval_declined\",\"message\":\"human declined\"}}", + "output": "{\"approved\":false,\"error\":{\"code\":\"approval_declined\",\"message\":\"tool call was not approved\"}}", "status": "completed" } diff --git a/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json b/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json index 6ae0eeee9..acf356a26 100644 --- a/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json +++ b/examples/harness/foundry-responses/testdata/golden/08_tool_policy_rejection_output.json @@ -1,6 +1,6 @@ { "type": "function_call_output", "call_id": "call-1", - "output": "{\"approved\":false,\"error\":{\"code\":\"tool_policy_rejected\",\"message\":\"tool is not allowed\"}}", + "output": "{\"approved\":false,\"error\":{\"code\":\"tool_policy_rejected\",\"message\":\"tool call was rejected by policy\"}}", "status": "completed" } diff --git a/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json b/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json index 039e1277c..e0ce3703d 100644 --- a/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json +++ b/examples/harness/foundry-responses/testdata/golden/09_tool_execution_failure_output.json @@ -1,6 +1,6 @@ { "type": "function_call_output", "call_id": "call-1", - "output": "{\"approved\":false,\"error\":{\"code\":\"tool_execution_failed\",\"message\":\"downstream failed\"}}", + "output": "{\"approved\":false,\"error\":{\"code\":\"tool_execution_failed\",\"message\":\"tool execution failed\"}}", "status": "completed" } diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 90f315be6..4765c8c26 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -258,8 +258,12 @@ expect_verifier_failure \ "mismatched-write-request" expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-partial-idempotency.json \ - "write execution for escalate-incident is missing idempotency key evidence" \ + "write execution for escalate-incident is missing execution idempotency key evidence" \ "partial-idempotency" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json \ + "write execution for dispatch-work-order is missing execution idempotency key evidence" \ + "generic-idempotency-only" expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json \ "event JSON is incomplete" \ From 36de6ff1e6e0e232aaa5a87c3d860fde2a16957a Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 20:26:12 -0700 Subject: [PATCH 48/51] fix: verify brokered tool completion Signed-off-by: Sertac Ozercan --- ...nses-events-completion-after-terminal.json | 86 ++++++++++++++++++ ...-responses-events-failure-after-start.json | 88 +++++++++++++++++++ ...onses-events-generic-idempotency-only.json | 28 ++++-- .../foundry-responses-events-pass.json | 29 ++++-- .../verify-foundry-responses.sh | 31 +++++++ .../foundry-responses/live-evidence.sh | 6 +- .../harness/foundry-responses/validate.sh | 8 ++ 7 files changed, 265 insertions(+), 11 deletions(-) create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-completion-after-terminal.json create mode 100644 examples/fibey-custom-agent-demo/testdata/foundry-responses-events-failure-after-start.json diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-completion-after-terminal.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-completion-after-terminal.json new file mode 100644 index 000000000..b3f79d2e7 --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-completion-after-terminal.json @@ -0,0 +1,86 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallCompleted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": { + "approved": true + } + }, + { + "seq": 3, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 4, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 5, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 6, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 7, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 8, + "eventType": "ToolCallCompleted", + "toolName": "dispatch-work-order", + "toolCallID": "write-call-1", + "content": { + "approved": true, + "executionIdempotencyKey": "approval-1" + } + }, + { + "seq": 9, + "eventType": "TaskSucceeded" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-failure-after-start.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-failure-after-start.json new file mode 100644 index 000000000..40c146fcf --- /dev/null +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-failure-after-start.json @@ -0,0 +1,88 @@ +{ + "events": [ + { + "seq": 1, + "eventType": "ToolCallStarted", + "toolName": "check-network-telemetry", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "read-call-1" + }, + { + "seq": 2, + "eventType": "ToolCallCompleted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": { + "approved": true + } + }, + { + "seq": 3, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "harness": { + "frameType": "ToolCallRequested" + } + }, + "toolCallID": "write-call-1" + }, + { + "seq": 4, + "eventType": "ApprovalRequested", + "toolName": "dispatch-work-order", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "targetTool": "dispatch-work-order", + "toolCallID": "write-call-1" + } + }, + { + "seq": 5, + "eventType": "ApprovalApproved", + "toolCallID": "approval-1", + "content": { + "approvalID": "approval-1", + "decision": "approve" + } + }, + { + "seq": 6, + "eventType": "ToolCallStarted", + "toolName": "dispatch-work-order", + "content": { + "approvalID": "approval-1", + "brokeredClass": "write", + "executionState": "started", + "idempotencyKey": "approval-1", + "executionIdempotencyKey": "approval-1" + }, + "toolCallID": "write-call-1" + }, + { + "seq": 7, + "eventType": "ToolCallFailed", + "toolName": "dispatch-work-order", + "toolCallID": "write-call-1", + "content": { + "approved": false, + "toolError": { + "code": "tool_execution_failed" + } + } + }, + { + "seq": 8, + "eventType": "AgentRuntimeCompleted" + }, + { + "seq": 9, + "eventType": "TaskSucceeded" + } + ] +} diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json index edb1e5768..684dd3c3d 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json @@ -13,6 +13,15 @@ }, { "seq": 2, + "eventType": "ToolCallCompleted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": { + "approved": true + } + }, + { + "seq": 3, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", "content": { @@ -23,7 +32,7 @@ "toolCallID": "write-call-1" }, { - "seq": 3, + "seq": 4, "eventType": "ApprovalRequested", "toolName": "dispatch-work-order", "toolCallID": "approval-1", @@ -34,7 +43,7 @@ } }, { - "seq": 4, + "seq": 5, "eventType": "ApprovalApproved", "toolCallID": "approval-1", "content": { @@ -43,7 +52,7 @@ } }, { - "seq": 5, + "seq": 6, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", "content": { @@ -55,11 +64,20 @@ "toolCallID": "write-call-1" }, { - "seq": 6, + "seq": 7, + "eventType": "ToolCallCompleted", + "toolName": "dispatch-work-order", + "toolCallID": "write-call-1", + "content": { + "approved": true + } + }, + { + "seq": 8, "eventType": "AgentRuntimeCompleted" }, { - "seq": 7, + "seq": 9, "eventType": "TaskSucceeded" } ] diff --git a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json index 7231ad15f..e49fcce4d 100644 --- a/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json +++ b/examples/fibey-custom-agent-demo/testdata/foundry-responses-events-pass.json @@ -13,6 +13,15 @@ }, { "seq": 2, + "eventType": "ToolCallCompleted", + "toolName": "check-network-telemetry", + "toolCallID": "read-call-1", + "content": { + "approved": true + } + }, + { + "seq": 3, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", "content": { @@ -23,7 +32,7 @@ "toolCallID": "write-call-1" }, { - "seq": 3, + "seq": 4, "eventType": "ApprovalRequested", "toolName": "dispatch-work-order", "toolCallID": "approval-1", @@ -34,7 +43,7 @@ } }, { - "seq": 4, + "seq": 5, "eventType": "ApprovalApproved", "toolCallID": "approval-1", "content": { @@ -43,7 +52,7 @@ } }, { - "seq": 5, + "seq": 6, "eventType": "ToolCallStarted", "toolName": "dispatch-work-order", "content": { @@ -56,11 +65,21 @@ "toolCallID": "write-call-1" }, { - "seq": 6, + "seq": 7, + "eventType": "ToolCallCompleted", + "toolName": "dispatch-work-order", + "toolCallID": "write-call-1", + "content": { + "approved": true, + "executionIdempotencyKey": "approval-1" + } + }, + { + "seq": 8, "eventType": "AgentRuntimeCompleted" }, { - "seq": 7, + "seq": 9, "eventType": "TaskSucceeded" } ] diff --git a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh index 5ae1536d4..e2d5e4d72 100755 --- a/examples/fibey-custom-agent-demo/verify-foundry-responses.sh +++ b/examples/fibey-custom-agent-demo/verify-foundry-responses.sh @@ -15,6 +15,7 @@ Expected evidence: - write brokered tool request for dispatch-work-order or escalate-incident - matching ApprovalRequested and ApprovalApproved events precede write execution - an executionIdempotencyKey is present in the write execution ledger event + - matching ToolCallCompleted events exist for each brokered read and write execution - terminal TaskSucceeded/AgentRuntimeCompleted/TurnCompleted-style event exists This verifier does not approve tasks and never reads Foundry credentials. @@ -247,6 +248,22 @@ def is_harness_tool_request(event): ) +def matching_tool_events(source, candidates, before_order=None): + source_tool = tool_name(source) + source_call_id = tool_call_id(source) + source_order = seq(source) + if not source_tool or not source_call_id or source_order is None: + return [] + return [ + candidate for candidate in candidates + if tool_name(candidate) == source_tool + and tool_call_id(candidate) == source_call_id + and seq(candidate) is not None + and seq(candidate) > source_order + and (before_order is None or seq(candidate) < before_order) + ] + + ordered_events = [] for index, event in enumerate(events, start=1): if isinstance(event, dict) and seq(event) is None: @@ -265,8 +282,11 @@ approval_approved_events = [e for e in events if event_type(e) == "ApprovalAppro approval_declined_events = [e for e in events if event_type(e) == "ApprovalDeclined"] write_exec_events = [e for e in write_events if is_write_execution_start(e)] write_start_events = write_exec_events +tool_completed_events = [e for e in events if event_type(e) == "ToolCallCompleted"] +tool_failed_events = [e for e in events if event_type(e) == "ToolCallFailed"] terminal_events = [e for e in events if event_type(e) in TERMINAL_TYPES] task_terminal_events = [e for e in events if event_type(e) in TASK_TERMINAL_TYPES] +first_terminal_order = min((seq(event) for event in terminal_events), default=None) execution_idempotency_events = [e for e in write_exec_events if execution_idempotency_value(e)] failures = [] @@ -280,12 +300,22 @@ if not approval_approved_events: failures.append("missing ApprovalApproved event") if not write_exec_events: failures.append("missing write ToolCallStarted event after approval") +for event in read_events: + read_tool = tool_name(event) + if matching_tool_events(event, tool_failed_events, first_terminal_order): + failures.append(f"read tool call for {read_tool} has matching ToolCallFailed") + elif not matching_tool_events(event, tool_completed_events, first_terminal_order): + failures.append(f"read tool call for {read_tool} is missing ToolCallCompleted") if write_exec_events: for event in write_exec_events: write_tool = tool_name(event) write_order = seq(event) if not execution_idempotency_value(event): failures.append(f"write execution for {write_tool} is missing execution idempotency key evidence") + if matching_tool_events(event, tool_failed_events, first_terminal_order): + failures.append(f"write execution for {write_tool} has matching ToolCallFailed") + elif not matching_tool_events(event, tool_completed_events, first_terminal_order): + failures.append(f"write execution for {write_tool} is missing ToolCallCompleted") write_tool_call_id = tool_call_id(event) if not write_tool_call_id: failures.append(f"write execution for {write_tool} is missing toolCallID") @@ -384,5 +414,6 @@ print(f"- write requests: {len(write_request_events)}") print(f"- approval requests: {len(approval_request_events)}") print(f"- approval decisions: {len(approval_approved_events)}") print(f"- execution idempotency evidence events: {len(execution_idempotency_events)}") +print(f"- completed tool calls: {len(tool_completed_events)}") print(f"- terminal events: {len(terminal_events)}") PY diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index a0b1f13bb..b81b64f85 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -260,7 +260,11 @@ def error_marker_is_set(value): def has_direct_error(value): if not isinstance(value, dict): return False - return error_marker_is_set(value.get("error")) or error_marker_is_set(value.get("errorCode")) + return ( + error_marker_is_set(value.get("error")) + or error_marker_is_set(value.get("errorCode")) + or error_marker_is_set(value.get("toolError")) + ) def safe_content(value): diff --git a/examples/harness/foundry-responses/validate.sh b/examples/harness/foundry-responses/validate.sh index 4765c8c26..5a4cf42c8 100755 --- a/examples/harness/foundry-responses/validate.sh +++ b/examples/harness/foundry-responses/validate.sh @@ -264,6 +264,14 @@ expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-generic-idempotency-only.json \ "write execution for dispatch-work-order is missing execution idempotency key evidence" \ "generic-idempotency-only" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-failure-after-start.json \ + "write execution for dispatch-work-order has matching ToolCallFailed" \ + "failure-after-start" +expect_verifier_failure \ + examples/fibey-custom-agent-demo/testdata/foundry-responses-events-completion-after-terminal.json \ + "write execution for dispatch-work-order is missing ToolCallCompleted" \ + "completion-after-terminal" expect_verifier_failure \ examples/fibey-custom-agent-demo/testdata/foundry-responses-events-truncated-page.json \ "event JSON is incomplete" \ From 58e13d8a84e6a2a4cb7b0bd3c9c010588a1aa4f3 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 20:54:14 -0700 Subject: [PATCH 49/51] fix: serialize brokered tool delivery Signed-off-by: Sertac Ozercan --- .../foundry-responses/live-evidence.sh | 7 +- examples/harness/foundry-responses/main.go | 35 ++- .../harness/foundry-responses/main_test.go | 217 +++++++++++++++--- 3 files changed, 226 insertions(+), 33 deletions(-) diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index b81b64f85..6417edf61 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -275,7 +275,12 @@ def safe_content(value): found = scalar(value.get(key)) if found not in (None, ""): safe[key] = found - for key in ("executionIdempotencyKey", "idempotencyKey", "Idempotency-Key"): + for key in ("executionIdempotencyKey", "Execution-Idempotency-Key"): + idempotency = scalar(value.get(key)) + if isinstance(idempotency, str) and idempotency.strip(): + safe["executionIdempotencyKey"] = idempotency.strip() + break + for key in ("idempotencyKey", "Idempotency-Key"): idempotency = scalar(value.get(key)) if isinstance(idempotency, str) and idempotency.strip(): safe["idempotencyKey"] = idempotency.strip() diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 4a15bf1b5..3b4f436a0 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -509,7 +509,11 @@ func (s *server) startTurn(w http.ResponseWriter, r *http.Request) { } if !turn.completed { disposition := s.handleResponsesResponseLocked(turn, response) - if disposition != responseRejected { + if disposition == responseRejected { + if _, reused := s.runtimeSessions[turn.request.RuntimeSessionID]; reused { + s.quarantineRuntimeSessionLocked(turn) + } + } else { s.setTurnSessionLocked(turn, foundrySessionID) } if disposition == responseCompleted { @@ -573,7 +577,6 @@ func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turn for { s.mu.Lock() frames := append([]harness.HarnessEventFrame(nil), turn.frames...) - suppressed := maps.Clone(turn.suppressedToolCalls) completed := turn.completed updates := turn.frameUpdates if updates == nil { @@ -586,7 +589,11 @@ func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turn continue } if frame.Type == harness.FrameToolCallRequested { - if _, skip := suppressed[frame.ToolCallID]; skip { + s.mu.Lock() + _, suppressed := turn.suppressedToolCalls[frame.ToolCallID] + terminal := turn.completed + s.mu.Unlock() + if suppressed || terminal { continue } } @@ -594,6 +601,12 @@ func (s *server) streamEvents(w http.ResponseWriter, r *http.Request, turn *turn return } nextSeq = frame.Seq + if frame.Type == harness.FrameToolCallRequested { + // Pace brokered calls one at a time. A partial /continue result, + // cancellation, or terminal transition wakes the stream before the + // next request is considered, so suppression is observed live. + break + } } if completed { _ = harness.WriteSSEDone(w) @@ -736,7 +749,11 @@ func (s *server) continueTurn(w http.ResponseWriter, r *http.Request, turn *turn delete(turn.bufferedDigests, result.ToolCallID) } disposition := s.handleResponsesResponseLocked(turn, response) - if disposition != responseRejected { + if disposition == responseRejected { + if _, reused := s.runtimeSessions[turn.request.RuntimeSessionID]; reused { + s.quarantineRuntimeSessionLocked(turn) + } + } else { s.setTurnSessionLocked(turn, updatedSessionID) } if disposition == responseCompleted { @@ -768,7 +785,8 @@ func (s *server) cancelTurn(w http.ResponseWriter, r *http.Request, turn *turnSt s.mu.Lock() if !turn.completed { cancelHostedRequest = turn.activeHostedRequestCancel - if cancelHostedRequest != nil { + _, reusedRuntimeSession := s.runtimeSessions[turn.request.RuntimeSessionID] + if cancelHostedRequest != nil || reusedRuntimeSession { s.quarantineRuntimeSessionLocked(turn) } s.suppressPendingToolCallsLocked(turn) @@ -1081,6 +1099,9 @@ func (s *server) recordContinueResults( } } if readyCount < len(turn.pendingTools) { + if len(newResults) > 0 { + s.notifyTurnUpdatedLocked(turn) + } return nil, nil } toSubmit := make([]harness.ToolCallResult, 0, len(unsubmittedIDs)) @@ -1757,6 +1778,10 @@ func harnessFrameFitsSSE(frame harness.HarnessEventFrame) bool { func (s *server) appendPreparedFrameLocked(turn *turnState, frame harness.HarnessEventFrame) { turn.frames = append(turn.frames, frame) + s.notifyTurnUpdatedLocked(turn) +} + +func (s *server) notifyTurnUpdatedLocked(turn *turnState) { if turn.frameUpdates != nil { close(turn.frameUpdates) } diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 81d900dd3..4aec9b377 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -24,6 +24,8 @@ import ( const ( fakeSessionID = "session-1" + existingSessionID = "existing-session" + foundryFailedReason = "foundry_failed" testContinuationProof = "proof-for-test" ) @@ -413,7 +415,7 @@ func TestResponsesAdapterRejectedResponseDoesNotRetainSession(t *testing.T) { t.Fatalf("first StartTurn: %v", err) } frames := streamCurrentFrames(t, client, first.TurnID) - if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || failed.Failed.Reason != "foundry_failed" { + if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || failed.Failed.Reason != foundryFailedReason { t.Fatalf("failed frame = %#v, want foundry_failed", failed) } server.mu.Lock() @@ -433,6 +435,93 @@ func TestResponsesAdapterRejectedResponseDoesNotRetainSession(t *testing.T) { } } +func TestResponsesAdapterRejectedReusedSessionIsQuarantined(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{scenario: "failed_with_session"}) + adapter, server := newTestResponsesAdapterWithServer(t, foundry.endpoint(), nil) + client := newHarnessClient(t, adapter) + request := responsesStartTurnRequest("foundry-rejected-reused-session") + server.mu.Lock() + server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ + ID: existingSessionID, + LastSeen: time.Now().UTC(), + } + server.mu.Unlock() + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + if got := requestMap(t, foundry.requestBody(0))["agent_session_id"]; got != existingSessionID { + t.Fatalf("initial agent_session_id = %#v, want existing-session", got) + } + frames := streamCurrentFrames(t, client, request.TurnID) + if failed := findFrame(frames, harness.FrameTurnFailed); failed == nil || failed.Failed.Reason != foundryFailedReason { + t.Fatalf("failed frame = %#v, want foundry_failed", failed) + } + server.mu.Lock() + _, retained := server.runtimeSessions[request.RuntimeSessionID] + _, quarantined := server.quarantinedSessions[request.RuntimeSessionID] + server.mu.Unlock() + if retained || !quarantined { + t.Fatalf("rejected reused session retained=%v quarantined=%v", retained, quarantined) + } + retry := responsesStartTurnRequest("foundry-rejected-reused-session-retry") + retry.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), retry); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with rejected reused session error = %v", err) + } + if got := foundry.postCount.Load(); got != 1 { + t.Fatalf("hosted post count = %d, want no retry for quarantined session", got) + } +} + +func TestResponsesAdapterCancelPendingReusedSessionQuarantinesIt(t *testing.T) { + foundry := newFakeResponses(t, fakeResponsesConfig{ + scenario: "function_call", + toolName: "support-ticket-lookup", + }) + adapter, server := newTestResponsesAdapterWithServer( + t, + foundry.endpoint(), + []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + ) + client := newHarnessClient(t, adapter) + request := brokeredReadRequest("foundry-cancel-pending-reused-session") + server.mu.Lock() + server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ + ID: existingSessionID, + LastSeen: time.Now().UTC(), + } + server.mu.Unlock() + + if _, err := client.StartTurn(context.Background(), request); err != nil { + t.Fatalf("StartTurn: %v", err) + } + frames := streamCurrentFrames(t, client, request.TurnID) + if findFrame(frames, harness.FrameToolCallRequested) == nil { + t.Fatalf("frames = %#v, want pending tool call", frames) + } + if _, err := client.CancelTurn(context.Background(), cancelRequestForStart(request)); err != nil { + t.Fatalf("CancelTurn: %v", err) + } + server.mu.Lock() + _, retained := server.runtimeSessions[request.RuntimeSessionID] + _, quarantined := server.quarantinedSessions[request.RuntimeSessionID] + server.mu.Unlock() + if retained || !quarantined { + t.Fatalf("cancelled pending reused session retained=%v quarantined=%v", retained, quarantined) + } + retry := brokeredReadRequest("foundry-cancel-pending-reused-session-retry") + retry.RuntimeSessionID = request.RuntimeSessionID + if _, err := client.StartTurn(context.Background(), retry); err == nil || + !strings.Contains(err.Error(), "runtime session unavailable after unconfirmed hosted cancellation") { + t.Fatalf("StartTurn with cancelled pending reused session error = %v", err) + } + if got := foundry.postCount.Load(); got != 1 { + t.Fatalf("hosted post count = %d, want no retry for quarantined session", got) + } +} + func TestResponsesAdapterCancelDuringInitialPostCancelsHostedRequest(t *testing.T) { received := make(chan struct{}) hostedRequestCancelled := make(chan struct{}) @@ -452,7 +541,7 @@ func TestResponsesAdapterCancelDuringInitialPostCancelsHostedRequest(t *testing. if err := json.Unmarshal(body, &decoded); err != nil { return nil, err } - if got := decoded["agent_session_id"]; got != "existing-session" { + if got := decoded["agent_session_id"]; got != existingSessionID { return nil, fmt.Errorf("agent_session_id = %#v, want existing-session", got) } close(received) @@ -470,7 +559,7 @@ func TestResponsesAdapterCancelDuringInitialPostCancelsHostedRequest(t *testing. request := responsesStartTurnRequest("foundry-cancel-initial") server.mu.Lock() server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ - ID: "existing-session", + ID: existingSessionID, LastSeen: time.Now().UTC(), } server.mu.Unlock() @@ -1000,26 +1089,38 @@ func TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued(t *testing.T) if _, err := client.StartTurn(context.Background(), request); err != nil { t.Fatalf("StartTurn: %v", err) } - frames := streamCurrentFrames(t, client, request.TurnID) - requests := findFrames(frames, harness.FrameToolCallRequested) - if len(requests) != 2 { - t.Fatalf("tool request frames = %#v, want 2", requests) - } - continueRequest := harness.ContinueTurnRequest{ - Version: harness.ProtocolVersion, - Namespace: request.Namespace, - TaskName: request.TaskName, - SessionName: request.SessionName, - RuntimeSessionID: request.RuntimeSessionID, - TurnID: request.TurnID, - CorrelationID: request.CorrelationID, - ToolResults: []harness.ToolCallResult{ - toolResultForRequest(request, "call-1", true, json.RawMessage(`{"success":true,"call":1}`), nil), - toolResultForRequest(request, "call-2", true, json.RawMessage(`{"success":true,"call":2}`), nil), - }, + var frames []harness.HarnessEventFrame + var requested []string + err := client.StreamFrames(context.Background(), request.TurnID, 0, func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + if frame.Type != harness.FrameToolCallRequested { + return nil + } + requested = append(requested, frame.ToolCallID) + result := toolResultForRequest( + request, + frame.ToolCallID, + true, + json.RawMessage(fmt.Sprintf(`{"success":true,"call":%d}`, len(requested))), + nil, + ) + _, continueErr := client.ContinueTurn(context.Background(), harness.ContinueTurnRequest{ + Version: harness.ProtocolVersion, + Namespace: request.Namespace, + TaskName: request.TaskName, + SessionName: request.SessionName, + RuntimeSessionID: request.RuntimeSessionID, + TurnID: request.TurnID, + CorrelationID: request.CorrelationID, + ToolResults: []harness.ToolCallResult{result}, + }) + return continueErr + }) + if err != nil { + t.Fatalf("StreamFrames: %v", err) } - if _, err := client.ContinueTurn(context.Background(), continueRequest); err != nil { - t.Fatalf("ContinueTurn: %v", err) + if !reflect.DeepEqual(requested, []string{"call-1", "call-2"}) { + t.Fatalf("requested tool calls = %#v, want call-1 then call-2", requested) } continuation := requestMap(t, foundry.requestBody(1)) items, ok := continuation["input"].([]any) @@ -1029,7 +1130,6 @@ func TestResponsesAdapterMultipleFunctionCallsBufferedAndContinued(t *testing.T) if got := continuation["agent_session_id"]; got != fakeSessionID { t.Fatalf("agent_session_id = %#v, want %q", got, fakeSessionID) } - frames = streamCurrentFrames(t, client, request.TurnID) if !hasFrameType(frames, harness.FrameToolResultReceived) || !hasFrameType(frames, harness.FrameTurnCompleted) { t.Fatalf("frames = %#v, want tool results and completion", frames) } @@ -1135,7 +1235,7 @@ func TestResponsesAdapterCancelDuringHostedContinuationCancelsRequest(t *testing return jsonHTTPResponseWithSession( r, functionCallResponse("support-ticket-lookup"), - "existing-session", + existingSessionID, ) } close(continuationReceived) @@ -1158,7 +1258,7 @@ func TestResponsesAdapterCancelDuringHostedContinuationCancelsRequest(t *testing request := brokeredReadRequest("foundry-cancel-continuation") server.mu.Lock() server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ - ID: "existing-session", + ID: existingSessionID, LastSeen: time.Now().UTC(), } server.mu.Unlock() @@ -1173,7 +1273,7 @@ func TestResponsesAdapterCancelDuringHostedContinuationCancelsRequest(t *testing server.mu.Lock() publishedBeforeCompletion := server.runtimeSessions[request.RuntimeSessionID] server.mu.Unlock() - if publishedBeforeCompletion.ID != "existing-session" { + if publishedBeforeCompletion.ID != existingSessionID { t.Fatalf("runtime session before continuation = %#v, want existing session", publishedBeforeCompletion) } @@ -2439,6 +2539,69 @@ func TestResponsesOversizedBatchValidatesAllResultsBeforeFailure(t *testing.T) { } } +func TestResponsesStreamRechecksSuppressionBetweenToolFrames(t *testing.T) { + server := newServer(config{ + adapterBearer: "adapter-auth-value", + brokeredToolClasses: []harness.BrokeredToolClass{harness.BrokeredToolClassRead}, + }, &http.Client{Timeout: time.Second}) + request := brokeredReadRequest("foundry-live-suppression") + turn := &turnState{ + request: request, + pendingTools: map[string]string{}, + suppressedToolCalls: map[string]struct{}{}, + bufferedResults: map[string]harness.ToolCallResult{}, + bufferedDigests: map[string]toolResultDigest{}, + submittedDigests: map[string]toolResultDigest{}, + } + server.turns[request.TurnID] = turn + server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") + server.handleResponsesResponse(turn, responsesResponse{ + ID: "resp-live-suppression", + Status: "completed", + Output: []responsesOutput{ + { + Type: "function_call", + CallID: "call-1", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-1"}`), + }, + { + Type: "function_call", + CallID: "call-2", + Name: "support-ticket-lookup", + Arguments: json.RawMessage(`{"incident":"inc-2"}`), + }, + }, + }) + + adapter := httptest.NewServer(server.handler()) + t.Cleanup(adapter.Close) + client := newHarnessClient(t, adapter) + var requested []string + var frames []harness.HarnessEventFrame + err := client.StreamFrames(context.Background(), request.TurnID, 0, func(frame harness.HarnessEventFrame) error { + frames = append(frames, frame) + if frame.Type != harness.FrameToolCallRequested { + return nil + } + requested = append(requested, frame.ToolCallID) + if len(requested) == 1 { + _, cancelErr := client.CancelTurn(context.Background(), cancelRequestForStart(request)) + return cancelErr + } + return nil + }) + if err != nil { + t.Fatalf("StreamFrames: %v", err) + } + if !reflect.DeepEqual(requested, []string{"call-1"}) { + t.Fatalf("requested tool calls = %#v, want only first call before cancellation", requested) + } + if !hasFrameType(frames, harness.FrameTurnCancelled) { + t.Fatalf("frames = %#v, want terminal cancellation", frames) + } +} + func TestResponsesTerminalFailureSuppressesPendingToolFrames(t *testing.T) { server := newServer(config{ adapterBearer: "adapter-auth-value", @@ -2683,7 +2846,7 @@ func TestResponsesFailureStatusDoesNotCompleteWithPartialText(t *testing.T) { t.Fatalf("frames = %#v, failed response should not complete", turn.frames) } failed := findFrame(turn.frames, harness.FrameTurnFailed) - if failed == nil || failed.Failed.Reason != "foundry_failed" { + if failed == nil || failed.Failed.Reason != foundryFailedReason { t.Fatalf("failed frame = %#v, want foundry_failed", failed) } } From 1227c3a34daca4b0dc7d920c37ce0d36713d50cc Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 21:34:22 -0700 Subject: [PATCH 50/51] fix: stream live evidence log scans Signed-off-by: Sertac Ozercan --- .../foundry-responses/live-evidence.sh | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/examples/harness/foundry-responses/live-evidence.sh b/examples/harness/foundry-responses/live-evidence.sh index 6417edf61..aa25d0697 100755 --- a/examples/harness/foundry-responses/live-evidence.sh +++ b/examples/harness/foundry-responses/live-evidence.sh @@ -72,6 +72,7 @@ require_cmd() { command -v "$1" >/dev/null 2>&1 || { echo "error: $1 is required" >&2; exit 2; } } +require_cmd awk require_cmd kubectl require_cmd orka require_cmd python3 @@ -86,11 +87,15 @@ runtime_tmp="" events_tmp="" approvals_tmp="" pods_tmp="" +log_scan_tmp="" +log_pipe_dir="" cleanup() { [[ -z "${runtime_tmp:-}" ]] || rm -f "$runtime_tmp" [[ -z "${events_tmp:-}" ]] || rm -f "$events_tmp" [[ -z "${approvals_tmp:-}" ]] || rm -f "$approvals_tmp" [[ -z "${pods_tmp:-}" ]] || rm -f "$pods_tmp" + [[ -z "${log_scan_tmp:-}" ]] || rm -f "$log_scan_tmp" + [[ -z "${log_pipe_dir:-}" ]] || rm -rf "$log_pipe_dir" } trap cleanup EXIT @@ -399,8 +404,14 @@ if [[ -z "$pods" ]]; then cat "$log_scan" >&2 exit 1 fi -log_text="" +log_scan_tmp="$(mktemp)" +log_pipe_dir="$(mktemp -d)" +log_match_fifo="$log_pipe_dir/match" +log_count_fifo="$log_pipe_dir/count" +mkfifo "$log_match_fifo" "$log_count_fifo" log_pods=0 +log_lines=0 +log_nonblank_lines=0 while IFS= read -r pod; do [[ -n "$pod" ]] || continue log_pods=$((log_pods + 1)) @@ -408,7 +419,23 @@ while IFS= read -r pod; do if [[ -n "$logs_since" ]]; then log_args+=(--since "$logs_since") fi - if ! pod_logs="$(kubectl -n "$namespace" "${log_args[@]}" 2>>"$log_err")"; then + : >"$log_scan_tmp" + grep -Ei "$log_forbidden_pattern" <"$log_match_fifo" >/dev/null & + match_pid=$! + awk ' + { count++; if ($0 ~ /[^[:space:]]/) nonblank++ } + END { printf "%d %d\n", count, nonblank } + ' <"$log_count_fifo" >"$log_scan_tmp" & + count_pid=$! + set +e + kubectl -n "$namespace" "${log_args[@]}" 2>>"$log_err" | tee "$log_match_fifo" >"$log_count_fifo" + pipeline_status=("${PIPESTATUS[@]}") + wait "$match_pid" + match_status=$? + wait "$count_pid" + count_status=$? + set -e + if (( pipeline_status[0] != 0 )); then { echo "adapter log scan: FAILED" echo "Could not retrieve adapter logs from pod/${pod}. Raw logs were not stored." @@ -418,10 +445,30 @@ while IFS= read -r pod; do cat "$log_scan" >&2 exit 1 fi - log_text+=$'\n'"${pod_logs}" + if (( pipeline_status[1] != 0 || count_status != 0 || (match_status != 0 && match_status != 1) )); then + { + echo "adapter log scan: FAILED" + echo "Could not scan adapter logs from pod/${pod}. Raw logs were not stored." + } >"$log_scan" + rm -f "$log_err" + cat "$log_scan" >&2 + exit 1 + fi + if (( match_status == 0 )); then + { + echo "adapter log scan: FAILED" + echo "A forbidden credential/tool-url pattern was detected in adapter logs. Raw logs were not stored." + } >"$log_scan" + rm -f "$log_err" + cat "$log_scan" >&2 + exit 1 + fi + read -r pod_lines pod_nonblank_lines <"$log_scan_tmp" + log_lines=$((log_lines + pod_lines)) + log_nonblank_lines=$((log_nonblank_lines + pod_nonblank_lines)) done <<<"$pods" rm -f "$log_err" -if [[ -z "${log_text//[[:space:]]/}" ]]; then +if (( log_nonblank_lines == 0 )); then { echo "adapter log scan: FAILED" echo "No adapter logs were returned from pods for deployment/${runtime}; evidence is indeterminate." @@ -429,18 +476,11 @@ if [[ -z "${log_text//[[:space:]]/}" ]]; then cat "$log_scan" >&2 exit 1 fi -if grep -Eiq "$log_forbidden_pattern" <<<"$log_text"; then - { - echo "adapter log scan: FAILED" - echo "A forbidden credential/tool-url pattern was detected in adapter logs. Raw logs were not stored." - } >"$log_scan" - cat "$log_scan" >&2 - exit 1 -fi { echo "adapter log scan: passed" echo "scanned pods: ${log_pods}" - echo "scanned tail lines: $(wc -l <<<"$log_text" | tr -d ' ')" + echo "scanned lines: ${log_lines}" + echo "scanned nonblank lines: ${log_nonblank_lines}" } >"$log_scan" { From 4b7b02f1db845641d14db40dd7d0f202e7a09808 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 22:27:41 -0700 Subject: [PATCH 51/51] fix: fence uncertain harness submissions Signed-off-by: Sertac Ozercan --- examples/harness/foundry-responses/main.go | 3 + .../harness/foundry-responses/main_test.go | 10 ++ internal/controller/harness_wrapper.go | 76 ++++++++++-- internal/controller/harness_wrapper_test.go | 115 +++++++++++++++++- 4 files changed, 196 insertions(+), 8 deletions(-) diff --git a/examples/harness/foundry-responses/main.go b/examples/harness/foundry-responses/main.go index 3b4f436a0..3898e4b6e 100644 --- a/examples/harness/foundry-responses/main.go +++ b/examples/harness/foundry-responses/main.go @@ -1075,6 +1075,9 @@ func (s *server) recordContinueResults( if !toolResultFrameFitsSSE(frame) { maps.Copy(turn.submittedDigests, turn.bufferedDigests) maps.Copy(turn.submittedDigests, newDigests) + if _, reused := s.runtimeSessions[turn.request.RuntimeSessionID]; reused { + s.quarantineRuntimeSessionLocked(turn) + } s.appendFailedLocked( turn, "brokered_tool_result_frame_too_large", diff --git a/examples/harness/foundry-responses/main_test.go b/examples/harness/foundry-responses/main_test.go index 4aec9b377..26ce9660c 100644 --- a/examples/harness/foundry-responses/main_test.go +++ b/examples/harness/foundry-responses/main_test.go @@ -2617,6 +2617,10 @@ func TestResponsesTerminalFailureSuppressesPendingToolFrames(t *testing.T) { submittedDigests: map[string]toolResultDigest{}, } server.turns[request.TurnID] = turn + server.runtimeSessions[request.RuntimeSessionID] = foundrySession{ + ID: existingSessionID, + LastSeen: time.Now().UTC(), + } server.appendFrameLocked(turn, harness.FrameTurnStarted, "foundry hosted response started") server.handleResponsesResponse(turn, responsesResponse{ ID: "resp-1", @@ -2645,6 +2649,12 @@ func TestResponsesTerminalFailureSuppressesPendingToolFrames(t *testing.T) { if _, err := server.recordContinueResults(turn, []harness.ToolCallResult{second}); err == nil { t.Fatal("oversized second result error = nil") } + if _, retained := server.runtimeSessions[request.RuntimeSessionID]; retained { + t.Fatal("oversized result failure retained reused runtime session") + } + if _, quarantined := server.quarantinedSessions[request.RuntimeSessionID]; !quarantined { + t.Fatal("oversized result failure did not quarantine reused runtime session") + } adapter := httptest.NewServer(server.handler()) t.Cleanup(adapter.Close) diff --git a/internal/controller/harness_wrapper.go b/internal/controller/harness_wrapper.go index 62074026f..4577f6ca4 100644 --- a/internal/controller/harness_wrapper.go +++ b/internal/controller/harness_wrapper.go @@ -57,6 +57,7 @@ const ( harnessWrapperCorrelationIDAnno = "orka.ai/harness-wrapper-correlation-id" harnessWrapperLastFrameSeqAnno = "orka.ai/harness-wrapper-last-frame-seq" harnessWrapperStartedAnno = "orka.ai/harness-wrapper-started" + harnessWrapperSubmissionAttemptedAnno = "orka.ai/harness-wrapper-submission-attempted" harnessWrapperPlannedAtAnno = "orka.ai/harness-wrapper-planned-at" harnessWrapperMetadataAnno = "orka.ai/harness-wrapper-metadata" harnessWrapperRuntimeRefAnno = "orka.ai/harness-wrapper-runtime-ref" @@ -81,6 +82,16 @@ func taskHasHarnessWrapperTurn(task *corev1alpha1.Task) bool { taskHasPlannedHarnessWrapperTurn(task) } +func taskHasHarnessWrapperSubmissionAttempt(task *corev1alpha1.Task) bool { + if task == nil || task.Annotations == nil { + return false + } + return strings.EqualFold( + strings.TrimSpace(task.Annotations[harnessWrapperSubmissionAttemptedAnno]), + scheduledRunLabelValue, + ) +} + func taskHasPlannedHarnessWrapperTurn(task *corev1alpha1.Task) bool { if task == nil || task.Annotations == nil { return false @@ -418,7 +429,7 @@ func (r *TaskReconciler) runHarnessWrapperTask(ctx context.Context, task *corev1 var err error startedPlannedTurn := false if taskHasPlannedHarnessWrapperTurn(task) { - if taskHasHarnessWrapperTurn(task) { + if taskHasHarnessWrapperTurn(task) || taskHasHarnessWrapperSubmissionAttempt(task) { startedPlannedTurn = true } request, err = r.plannedHarnessWrapperStartTurnRequest(ctx, task, agent, now.Time) @@ -509,7 +520,7 @@ func (r *TaskReconciler) runHarnessWrapperTask(ctx context.Context, task *corev1 if !hasFrames { if err := r.validateHarnessWrapperCapabilities(ctx, client, request); err != nil { if target.RuntimeRefName != "" && harnessWrapperAuthError(err) { - if shouldWait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task); waitErr != nil { + if shouldWait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task, false); waitErr != nil { return ctrl.Result{}, waitErr } else if shouldWait { return ctrl.Result{RequeueAfter: time.Second}, nil @@ -520,6 +531,9 @@ func (r *TaskReconciler) runHarnessWrapperTask(ctx context.Context, task *corev1 } return r.failTask(ctx, task, err.Error()) } + if err := r.patchHarnessWrapperSubmissionAttempted(ctx, task); err != nil { + return ctrl.Result{}, err + } if _, err := client.StartTurn(ctx, request); err != nil { message := err.Error() switch { @@ -535,7 +549,7 @@ func (r *TaskReconciler) runHarnessWrapperTask(ctx context.Context, task *corev1 } return ctrl.Result{RequeueAfter: time.Second}, nil case target.RuntimeRefName != "" && harnessWrapperAuthError(err): - if wait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task); waitErr != nil { + if wait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task, true); waitErr != nil { return ctrl.Result{}, waitErr } else if wait { return ctrl.Result{RequeueAfter: time.Second}, nil @@ -549,7 +563,14 @@ func (r *TaskReconciler) runHarnessWrapperTask(ctx context.Context, task *corev1 } } turnAccepted = true - if err := r.patchHarnessWrapperStarted(ctx, task); err != nil { + if !taskHasHarnessWrapperTurn(task) { + if err := r.patchHarnessWrapperStarted(ctx, task, false); err != nil { + return ctrl.Result{}, err + } + } + } + if startedPlannedTurn && !taskHasHarnessWrapperTurn(task) { + if err := r.patchHarnessWrapperStarted(ctx, task, taskHasHarnessWrapperSubmissionAttempt(task)); err != nil { return ctrl.Result{}, err } } @@ -708,12 +729,20 @@ func (r *TaskReconciler) finishHarnessWrapperTask(ctx context.Context, task *cor } if err != nil && result.Completed == nil && result.Failed == nil && !result.Cancelled { if target.RuntimeRefName != "" && harnessWrapperAuthError(err) { - if wait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task); waitErr != nil { + if wait, waitErr := r.waitForHarnessWrapperAuthRetry(ctx, task, false); waitErr != nil { return ctrl.Result{}, waitErr } else if wait { return ctrl.Result{RequeueAfter: time.Second}, nil } } + if harnessWrapperStreamErrorIsMissingTurn(err) && taskHasHarnessWrapperSubmissionAttempt(task) { + return r.completeTask( + ctx, + task, + corev1alpha1.TaskPhaseFailed, + "harness runtime lost a turn after StartTurn submission outcome became unknown", + ) + } if harnessWrapperStreamErrorIsMissingTurn(err) && r.shouldRetry(task) { if clearErr := r.clearHarnessWrapperTurnState(ctx, task); clearErr != nil { return ctrl.Result{}, clearErr @@ -926,7 +955,11 @@ func harnessWrapperAuthRetries(task *corev1alpha1.Task) int { return retries } -func (r *TaskReconciler) waitForHarnessWrapperAuthRetry(ctx context.Context, task *corev1alpha1.Task) (bool, error) { +func (r *TaskReconciler) waitForHarnessWrapperAuthRetry( + ctx context.Context, + task *corev1alpha1.Task, + clearSubmissionAttempt bool, +) (bool, error) { retries := harnessWrapperAuthRetries(task) if retries >= harnessWrapperMaxAuthRetries { return false, nil @@ -936,6 +969,9 @@ func (r *TaskReconciler) waitForHarnessWrapperAuthRetry(ctx context.Context, tas task.Annotations = map[string]string{} } task.Annotations[harnessWrapperAuthRetriesAnno] = strconv.Itoa(retries + 1) + if clearSubmissionAttempt { + delete(task.Annotations, harnessWrapperSubmissionAttemptedAnno) + } if err := r.Patch(ctx, task, patch); err != nil { return false, err } @@ -1015,6 +1051,7 @@ func (r *TaskReconciler) patchHarnessWrapperPlannedTurn( task.Annotations[harnessWrapperCorrelationIDAnno] = request.CorrelationID task.Annotations[harnessWrapperLastFrameSeqAnno] = "0" task.Annotations[harnessWrapperStartedAnno] = "false" + delete(task.Annotations, harnessWrapperSubmissionAttemptedAnno) task.Annotations[harnessWrapperPlannedAtAnno] = time.Now().UTC().Format(time.RFC3339Nano) if runtimeRefName := strings.TrimSpace(request.Metadata["runtimeRef"]); runtimeRefName != "" { task.Annotations[harnessWrapperRuntimeRefAnno] = runtimeRefName @@ -1043,7 +1080,28 @@ func (r *TaskReconciler) patchHarnessWrapperPlannedTurn( return r.Patch(ctx, task, patch) } -func (r *TaskReconciler) patchHarnessWrapperStarted(ctx context.Context, task *corev1alpha1.Task) error { +func (r *TaskReconciler) patchHarnessWrapperSubmissionAttempted(ctx context.Context, task *corev1alpha1.Task) error { + latest := &corev1alpha1.Task{} + if err := r.Get(ctx, ctrlclient.ObjectKey{Name: task.Name, Namespace: task.Namespace}, latest); err != nil { + return err + } + patch := ctrlclient.MergeFrom(latest.DeepCopy()) + if latest.Annotations == nil { + latest.Annotations = map[string]string{} + } + latest.Annotations[harnessWrapperSubmissionAttemptedAnno] = scheduledRunLabelValue + if err := r.Patch(ctx, latest, patch); err != nil { + return err + } + latest.DeepCopyInto(task) + return nil +} + +func (r *TaskReconciler) patchHarnessWrapperStarted( + ctx context.Context, + task *corev1alpha1.Task, + preserveSubmissionAttempt bool, +) error { latest := &corev1alpha1.Task{} if err := r.Get(ctx, ctrlclient.ObjectKey{Name: task.Name, Namespace: task.Namespace}, latest); err != nil { return err @@ -1067,6 +1125,9 @@ func (r *TaskReconciler) patchHarnessWrapperStarted(ctx context.Context, task *c } } latest.Annotations[harnessWrapperStartedAnno] = scheduledRunLabelValue + if !preserveSubmissionAttempt { + delete(latest.Annotations, harnessWrapperSubmissionAttemptedAnno) + } if err := r.Patch(ctx, latest, patch); err != nil { return err } @@ -1436,6 +1497,7 @@ func (r *TaskReconciler) clearHarnessWrapperTurnState(ctx context.Context, task delete(task.Annotations, harnessWrapperCorrelationIDAnno) delete(task.Annotations, harnessWrapperLastFrameSeqAnno) delete(task.Annotations, harnessWrapperStartedAnno) + delete(task.Annotations, harnessWrapperSubmissionAttemptedAnno) delete(task.Annotations, harnessWrapperPlannedAtAnno) delete(task.Annotations, harnessWrapperMetadataAnno) delete(task.Annotations, harnessWrapperRuntimeRefAnno) diff --git a/internal/controller/harness_wrapper_test.go b/internal/controller/harness_wrapper_test.go index c129add22..eef56efe9 100644 --- a/internal/controller/harness_wrapper_test.go +++ b/internal/controller/harness_wrapper_test.go @@ -107,7 +107,7 @@ func TestPatchHarnessWrapperStartedPreservesPlannedTurnAnnotationsFromLocalTask( local.Annotations[harnessWrapperOutputFetchRetriesAnno] = "1" r := newUnitReconciler(newTestScheme(), task) - if err := r.patchHarnessWrapperStarted(context.Background(), local); err != nil { + if err := r.patchHarnessWrapperStarted(context.Background(), local, false); err != nil { t.Fatalf("patchHarnessWrapperStarted: %v", err) } @@ -1615,6 +1615,119 @@ func TestHarnessWrapperPendingFirstOnlyPlansTurn(t *testing.T) { } } +func TestHarnessWrapperStartTurnAttemptIsPersistedBeforeSubmission(t *testing.T) { + startCalls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == harness.CapabilitiesPath: + harness.WriteJSON(w, http.StatusOK, harness.CapabilitiesResponse{ + Version: harness.ProtocolVersion, + ProtocolVersion: harness.ProtocolVersion, + Transport: harness.HTTPTransport, + RuntimeName: "codex", + ProviderKind: harness.ProviderKindKubernetesService, + ToolExecutionModes: []harness.ToolExecutionMode{harness.ToolExecutionModeObserved}, + SupportsCancel: true, + SupportsRuntimeSessions: true, + }) + case r.Method == http.MethodPost && r.URL.Path == harness.TurnsPath: + startCalls++ + harness.WriteError(w, http.StatusInternalServerError, "submission outcome unknown") + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + t.Setenv(harnessWrapperEndpointEnv, srv.URL) + + task, agent := harnessWrapperTaskAndAgent() + secret := attachHarnessWrapperRuntimeSecret(task, agent) + r := newUnitReconciler(newTestScheme(), task, agent, secret) + if _, err := r.handlePending(context.Background(), task); err != nil { + t.Fatalf("planning handlePending: %v", err) + } + var planned corev1alpha1.Task + key := types.NamespacedName{Name: task.Name, Namespace: task.Namespace} + if err := r.Get(context.Background(), key, &planned); err != nil { + t.Fatalf("get planned task: %v", err) + } + + result, err := r.handlePending(context.Background(), &planned) + if err != nil { + t.Fatalf("submission handlePending: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatalf("submission requeue = %s, want positive delay", result.RequeueAfter) + } + if startCalls != 1 { + t.Fatalf("StartTurn calls after uncertain submission = %d, want 1", startCalls) + } + var attempted corev1alpha1.Task + if err := r.Get(context.Background(), key, &attempted); err != nil { + t.Fatalf("get attempted task: %v", err) + } + if !taskHasHarnessWrapperSubmissionAttempt(&attempted) || taskHasHarnessWrapperTurn(&attempted) { + t.Fatalf("submission annotations = %#v, want attempted but not started", attempted.Annotations) + } + + if _, err := r.handlePending(context.Background(), &attempted); err != nil { + t.Fatalf("recovery handlePending: %v", err) + } + if startCalls != 1 { + t.Fatalf("StartTurn calls after recovery = %d, want no duplicate", startCalls) + } + var running corev1alpha1.Task + if err := r.Get(context.Background(), key, &running); err != nil { + t.Fatalf("get running task: %v", err) + } + if running.Status.Phase != corev1alpha1.TaskPhaseRunning || !taskHasHarnessWrapperTurn(&running) { + t.Fatalf("recovered task phase=%s annotations=%#v, want Running started turn", running.Status.Phase, running.Annotations) + } + if !taskHasHarnessWrapperSubmissionAttempt(&running) { + t.Fatal("unknown submission marker was not preserved through Running recovery") + } + if _, err := r.handleRunning(context.Background(), &running); err != nil { + t.Fatalf("handleRunning after lost submission: %v", err) + } + if startCalls != 1 { + t.Fatalf("StartTurn calls after missing-turn failure = %d, want no duplicate", startCalls) + } + var failed corev1alpha1.Task + if err := r.Get(context.Background(), key, &failed); err != nil { + t.Fatalf("get failed task: %v", err) + } + if failed.Status.Phase != corev1alpha1.TaskPhaseFailed || + !strings.Contains(failed.Status.Message, "submission outcome became unknown") { + t.Fatalf("failed task phase=%s message=%q", failed.Status.Phase, failed.Status.Message) + } +} + +func TestHarnessWrapperAuthRetryClearsSubmissionAttemptAtomically(t *testing.T) { + task, _ := harnessWrapperTaskAndAgent() + task.Annotations = map[string]string{ + harnessWrapperSubmissionAttemptedAnno: scheduledRunLabelValue, + } + r := newUnitReconciler(newTestScheme(), task) + wait, err := r.waitForHarnessWrapperAuthRetry(context.Background(), task, true) + if err != nil { + t.Fatalf("waitForHarnessWrapperAuthRetry: %v", err) + } + if !wait { + t.Fatal("auth retry did not request a wait") + } + var updated corev1alpha1.Task + key := types.NamespacedName{Name: task.Name, Namespace: task.Namespace} + if err := r.Get(context.Background(), key, &updated); err != nil { + t.Fatalf("get updated task: %v", err) + } + if got := harnessWrapperAuthRetries(&updated); got != 1 { + t.Fatalf("auth retries = %d, want 1", got) + } + if taskHasHarnessWrapperSubmissionAttempt(&updated) { + t.Fatal("submission-attempt marker remained after definite auth rejection") + } +} + func TestHarnessRuntimeRunningTaskFinishesAfterStart(t *testing.T) { cfg := cliwrapper.DefaultConfig() cfg.AllowUnauthenticated = true