diff --git a/bin/llm b/bin/llm index e6a5b00..45214af 100755 --- a/bin/llm +++ b/bin/llm @@ -77,6 +77,14 @@ LLM_OR_ZDR="${LLM_OR_ZDR:-}" # routing constraint nobody typed on this command line still changes where the # prompt may go, so it is announced once rather than applied invisibly. _llm_or_env="$LLM_OR_ONLY$LLM_OR_DATA_COLLECTION$LLM_OR_ZDR" +# Completion wire protocol. "chat" preserves the existing API and output +# contract; "responses" selects OpenAI's Responses create/stream protocol. +# Lifecycle operations (retrieve/cancel/delete, conversations, compaction) +# belong in a follow-on surface rather than overloading this completion CLI. +LLM_API_FORMAT="${LLM_API_FORMAT:-chat}" +LLM_RESPONSES_BODY_FILE="${LLM_RESPONSES_BODY_FILE:-}" +LLM_PREVIOUS_RESPONSE_ID="${LLM_PREVIOUS_RESPONSE_ID:-}" +LLM_RESPONSE_FILE="${LLM_RESPONSE_FILE:-}" # Token usage is captured on every call. A caller that wants this call's # numbers sets LLM_USAGE_FILE and gets a one-line JSON record # ({in_tok, out_tok, think_tok}) written there after the call (bin/shellm @@ -326,6 +334,18 @@ Environment: LLM_API_KEY Optional key for openai-compatible, sent as Authorization: Bearer when set (local servers like Ollama need none) + LLM_API_FORMAT Completion protocol: chat (default) or responses. + Responses is supported by openai, openrouter, and + openai-compatible providers + LLM_RESPONSES_BODY_FILE Optional JSON object merged into a Responses create + request. Supports tools, include, store, text format, + metadata, truncation, service tier, and other create + fields. llm owns model, input, instructions, + previous_response_id, max_output_tokens, stream, and + command-line reasoning settings + LLM_PREVIOUS_RESPONSE_ID Continue a stored Responses chain from this response + LLM_RESPONSE_FILE Atomically write the full terminal Response object, + or a provider error envelope, to this mode-0600 file LLM_ADAPTER Path to an adapter executable; required with --provider adapter (contract: design/providers.md) LLM_PROVIDER Provider to use when the model name implies none @@ -361,6 +381,8 @@ Examples: llm -m z-ai/glm-5.2 "explain quicksort" LLM_API_URL=http://localhost:11434/v1/chat/completions \ llm --provider openai-compatible -m qwen3:8b "explain quicksort" + LLM_API_FORMAT=responses LLM_RESPONSE_FILE=/tmp/response.json \ + llm --provider openai -m gpt-5.5 --thinking high "solve carefully" EOF exit "${1:-0}" } @@ -459,6 +481,29 @@ if [[ -z "$LLM_PROVIDER" ]]; then LLM_PROVIDER=$(detect_provider "$LLM_MODEL") fi +case "$LLM_API_FORMAT" in + chat) ;; + responses) + case "$LLM_PROVIDER" in + openai|openrouter|openai-compatible) ;; + *) die "Responses format is not supported for provider $LLM_PROVIDER (use openai, openrouter, or openai-compatible)" ;; + esac + if [[ -n "$LLM_RESPONSES_BODY_FILE" ]]; then + [[ -f "$LLM_RESPONSES_BODY_FILE" ]] \ + || die "Responses body file not found: $LLM_RESPONSES_BODY_FILE" + jq -e 'type == "object"' "$LLM_RESPONSES_BODY_FILE" >/dev/null 2>&1 \ + || die "Responses body file must contain a JSON object: $LLM_RESPONSES_BODY_FILE" + if [[ "$(jq -r '.background // false' "$LLM_RESPONSES_BODY_FILE")" == true ]]; then + die "background responses require lifecycle operations; use foreground Responses completion for now" + fi + if jq -e 'has("conversation")' "$LLM_RESPONSES_BODY_FILE" >/dev/null 2>&1; then + die "conversation state cannot be combined with llm's previous_response_id continuation; omit conversation" + fi + fi + ;; + *) die "Unknown LLM_API_FORMAT '$LLM_API_FORMAT' (expected chat or responses)" ;; +esac + # An environment-supplied provider reroutes every call in the process tree, # so it must never do it silently: when it disagrees with what the model # name implies, say so on stderr. (detect_provider dies on names it cannot @@ -726,6 +771,58 @@ build_payload_openai() { printf '%s' "$msgs" | jq "${jq_args[@]}" "$jq_filter" } +# OpenAI Responses create request. Input stays as the typed JSON supplied by +# --messages/--messages-file: message items, images/files, prior reasoning +# items, tool calls, and tool outputs must not be flattened into chat strings. +# The optional extra body owns the broad protocol surface while llm overwrites +# the fields coupled to its existing CLI contract. +build_payload_responses() { + local reasoning_effort="${LLM_THINKING_LEVEL:-${LLM_EFFORT:-}}" + local -a jq_args=( + --arg model "$LLM_MODEL" + --arg system "$LLM_SYSTEM" + --arg previous "$LLM_PREVIOUS_RESPONSE_ID" + --arg effort "$reasoning_effort" + --argjson max_tokens "$LLM_MAX_TOKENS" + --argjson stream "$( [[ "$LLM_STREAM" -eq 1 ]] && echo true || echo false )" + ) + + if [[ -n "$LLM_RESPONSES_BODY_FILE" ]]; then + jq_args+=(--slurpfile extra "$LLM_RESPONSES_BODY_FILE") + else + # Match --slurpfile's array shape without putting any caller content + # on argv. + jq_args+=(--argjson extra '[{}]') + fi + + if [[ -n "$reasoning_effort" && "$LLM_PROVIDER" == openai ]] \ + && ! supports_openai_reasoning; then + echo "llm: --thinking/--effort ignored: $LLM_MODEL not in the known-reasoning model list (LLM_ASSUME_THINKING=1 to send anyway)" >&2 + reasoning_effort="" + # Replace the earlier jq value; the final duplicate option wins. + jq_args+=(--arg effort "") + fi + + printf '%s' "$LLM_MESSAGES" | jq "${jq_args[@]}" ' + ($extra[0] // {}) + { + model: $model, + input: ., + max_output_tokens: $max_tokens, + stream: $stream + } + | if $system == "" then del(.instructions) + else .instructions = $system end + | if $previous == "" then del(.previous_response_id) + else .previous_response_id = $previous end + | if $effort == "" then . + else .reasoning = ((.reasoning // {}) + {effort: $effort, summary: "auto"}) end + | .include = ((.include // []) + | if index("reasoning.encrypted_content") == null + then . + ["reasoning.encrypted_content"] + else . end) + ' +} + build_payload_gemini() { # Convert messages to Gemini format: {contents:[{role,parts:[{text}]}]} local contents @@ -753,10 +850,9 @@ build_payload_gemini() { printf '%s' "$contents" | jq "${jq_args[@]}" "$jq_filter" } -# OpenRouter is OpenAI-compatible (chat/completions wire format) -# OpenRouter is OpenAI-compatible (chat/completions wire format), plus an -# optional "provider" object carrying the routing preferences above. Without -# that object OpenRouter is free to pick any host serving the model. +# OpenRouter is OpenAI-compatible (chat/completions or Responses wire format), +# plus an optional "provider" object carrying the routing preferences above. +# Without that object OpenRouter is free to pick any host serving the model. build_payload_openrouter() { # Unpinned callers pay nothing: no extra jq pass, byte-identical to before. if [[ -z "$LLM_OR_ONLY$LLM_OR_DATA_COLLECTION$LLM_OR_ZDR" ]]; then @@ -839,13 +935,21 @@ get_url_headers() { ;; openai) [[ -z "${OPENAI_API_KEY:-}" ]] && die "OPENAI_API_KEY is not set" - url="${LLM_API_URL:-https://api.openai.com/v1/chat/completions}" + if [[ "$LLM_API_FORMAT" == responses ]]; then + url="${LLM_API_URL:-https://api.openai.com/v1/responses}" + else + url="${LLM_API_URL:-https://api.openai.com/v1/chat/completions}" + fi headers=(-H "Content-Type: application/json") add_auth_header "Authorization: Bearer $OPENAI_API_KEY" ;; openrouter) [[ -z "${OPENROUTER_API_KEY:-}" ]] && die "OPENROUTER_API_KEY is not set" - url="${LLM_API_URL:-https://openrouter.ai/api/v1/chat/completions}" + if [[ "$LLM_API_FORMAT" == responses ]]; then + url="${LLM_API_URL:-https://openrouter.ai/api/v1/responses}" + else + url="${LLM_API_URL:-https://openrouter.ai/api/v1/chat/completions}" + fi headers=(-H "Content-Type: application/json") add_auth_header "Authorization: Bearer $OPENROUTER_API_KEY" ;; @@ -901,6 +1005,27 @@ extract_text_openai() { jq -r '.choices[0].message.content // empty' 2>/dev/null } +extract_text_responses() { + jq -j ' + .output[]? + | select(.type == "message") + | .content[]? + | if .type == "output_text" then .text + elif .type == "refusal" then .refusal + else empty end + ' 2>/dev/null +} + +extract_reasoning_responses() { + jq -j ' + .output[]? + | select(.type == "reasoning") + | .summary[]? + | select(.type == "summary_text") + | .text + ' 2>/dev/null +} + extract_text_gemini() { jq -r '.candidates[0].content.parts[0].text // empty' 2>/dev/null } @@ -937,6 +1062,14 @@ check_response_openai() { ] | .[0] // empty' 2>/dev/null } +check_response_responses() { + jq -r ' + if .error != null then .error.message // "unknown error" + elif .status == "failed" then .error.message // "response failed" + else empty end + ' 2>/dev/null +} + check_response_gemini() { jq -r '.error.message // empty' 2>/dev/null } @@ -962,6 +1095,22 @@ check_truncated_openai() { jq -r 'if (.choices[0].finish_reason // "") == "length" then "y" else empty end' 2>/dev/null } +check_truncated_responses() { + jq -r ' + if .status == "incomplete" + and (.incomplete_details.reason // "") == "max_output_tokens" + then "y" else empty end + ' 2>/dev/null +} + +check_incomplete_responses() { + jq -r ' + if .status == "incomplete" + then .incomplete_details.reason // "unknown reason" + else empty end + ' 2>/dev/null +} + check_truncated_gemini() { jq -r 'if (.candidates[0].finishReason // "") == "MAX_TOKENS" then "y" else empty end' 2>/dev/null } @@ -978,6 +1127,33 @@ warn_truncated() { echo "llm: warning: output truncated at max_tokens=$LLM_MAX_TOKENS (reasoning tokens count against it) — raise with -t" >&2 } +warn_incomplete_response() { + local reason="$1" + [[ -z "$reason" || "$reason" == max_output_tokens ]] && return 0 + echo "llm: warning: response incomplete: $reason" >&2 +} + +# Write the full terminal Responses object (or an API error envelope) for +# machine callers. A same-directory temporary plus rename makes each write +# atomic, and mode 0600 keeps prompts, outputs, and reasoning private. +write_response_file() { + [[ -n "$LLM_RESPONSE_FILE" ]] || return 0 + local response_dir response_tmp + response_dir=$(dirname "$LLM_RESPONSE_FILE") + [[ -d "$response_dir" ]] \ + || die "Response file directory does not exist: $response_dir" + response_tmp=$(mktemp "$response_dir/.llm-response.XXXXXX") \ + || die "Cannot create temporary Response file in: $response_dir" + if ( umask 077; cat > "$response_tmp" ); then + chmod 600 "$response_tmp" 2>/dev/null || true + mv -f "$response_tmp" "$LLM_RESPONSE_FILE" \ + || { rm -f "$response_tmp"; die "Cannot write Response file: $LLM_RESPONSE_FILE"; } + else + rm -f "$response_tmp" + die "Cannot write Response file: $LLM_RESPONSE_FILE" + fi +} + # Write the LLM_USAGE_FILE record. Args: input tokens, output tokens, # reasoning/thinking tokens, cached input tokens — non-numeric args are # omitted from the record. cache_tok is the part of in_tok the provider @@ -1036,12 +1212,31 @@ _llm_ledger_append() { # Streaming handlers # --------------------------------------------------------------------------- +# True when a Responses error names previous_response_id (expiry, ZDR, etc.). +_responses_prev_id_error() { + jq -e '[.error.param?,.error.code?,.error.message?,.param?,.code?,.message?,.detail?] + | map(select(. != null) | tostring | ascii_downcase) | join(" ") + | test("previous[ _]response[ _]id|previous response")' >/dev/null 2>&1 +} + # Shared stream tail: if no SSE data arrived, the buffered lines are an API # error body. Reads the caller's _saw_data/_err_buf via dynamic scoping. _die_if_no_stream_data() { [[ "$_saw_data" -eq 0 && -n "$_err_buf" ]] || return 0 - local _err - _err=$(printf '%s' "$_err_buf" | jq -r '.error.message // empty' 2>/dev/null || true) + local _err _continuation_error=0 + if [[ "$LLM_API_FORMAT" == responses && -n "$LLM_RESPONSE_FILE" ]]; then + if printf '%s' "$_err_buf" | jq -e . >/dev/null 2>&1; then + printf '%s' "$_err_buf" | write_response_file + printf '%s' "$_err_buf" | _responses_prev_id_error && _continuation_error=1 + else + jq -nc --arg message "$_err_buf" '{error:{message:$message}}' \ + | write_response_file + fi + fi + _err=$(printf '%s' "$_err_buf" | jq -r '.error.message // .message // empty' 2>/dev/null || true) + if [[ "$_continuation_error" -eq 1 ]]; then + die "API error: ${_err:-previous_response_id was rejected}" + fi [[ -n "$_err" ]] && die_retryable "API error: $_err" die_retryable "API error: $_err_buf" } @@ -1158,6 +1353,116 @@ stream_openai() { _die_if_no_stream_data } +stream_responses() { + local _saw_data=0 _err_buf="" _chunk="" _event="" _response="" + local _text_emitted=0 _reasoning_emitted=0 _terminal=0 + while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do + local line="${raw_line%$'\r'}" + if [[ "$line" == data:\ * ]]; then + _saw_data=1 + local json="${line#data: }" + [[ "$json" == "[DONE]" ]] && continue + _event=$(printf '%s' "$json" | jq -r '.type // empty' 2>/dev/null) || _event="" + case "$_event" in + response.output_text.delta|response.refusal.delta) + # Preserve newline-only deltas; see stream_openai. + _chunk=$(printf '%s' "$json" | jq -j '.delta // empty'; printf X) + _chunk="${_chunk%X}" + if [[ -n "$_chunk" ]]; then + mark_emitted + _text_emitted=1 + printf '%s' "$_chunk" + fi + ;; + response.reasoning_summary_text.delta) + _chunk=$(printf '%s' "$json" | jq -j '.delta // empty'; printf X) + _chunk="${_chunk%X}" + if [[ -n "$_chunk" ]]; then + mark_emitted + _reasoning_emitted=1 + printf '%s' "$_chunk" >&2 + fi + ;; + response.output_item.done) + # A function call or other typed item is valid protocol + # output even when there is no visible assistant text. + mark_emitted + ;; + response.completed|response.incomplete) + _response=$(printf '%s' "$json" | jq -c '.response // empty' 2>/dev/null) || _response="" + if [[ -n "$_response" ]] \ + && printf '%s' "$_response" | jq -e --arg status "${_event#response.}" \ + '.status == $status' >/dev/null 2>&1; then + _terminal=1 + printf '%s' "$_response" | write_response_file + if [[ "$_reasoning_emitted" -eq 0 ]]; then + printf '%s' "$_response" | extract_reasoning_responses >&2 + fi + if [[ "$_text_emitted" -eq 0 ]]; then + printf '%s' "$_response" | extract_text_responses + fi + + local _u="" + _u=$(printf '%s' "$_response" | jq -r '[ + (.usage.input_tokens // ""), + (.usage.output_tokens // ""), + (.usage.output_tokens_details.reasoning_tokens // ""), + (.usage.input_tokens_details.cached_tokens // "") + ] | @tsv' 2>/dev/null) || _u="" + if [[ -n "$_u" ]]; then + local _u_in _u_out _u_think _u_cache + IFS=$'\t' read -r _u_in _u_out _u_think _u_cache <<<"$_u" + write_usage "$_u_in" "$_u_out" "$_u_think" "$_u_cache" + fi + + if [[ "$_event" == response.incomplete ]]; then + local _reason="" + _reason=$(printf '%s' "$_response" | check_incomplete_responses) + if [[ "$_reason" == max_output_tokens ]]; then + warn_truncated + else + warn_incomplete_response "$_reason" + fi + fi + fi + mark_emitted + ;; + response.failed) + _response=$(printf '%s' "$json" | jq -c '.response // empty' 2>/dev/null) || _response="" + [[ -n "$_response" ]] && printf '%s' "$_response" | write_response_file + local _failed_msg="" + _failed_msg=$(printf '%s' "$json" | jq -r '.response.error.message // .error.message // "response failed"' 2>/dev/null) || _failed_msg="response failed" + if [[ -e "$LLM_DATA_MARKER" ]]; then + die "API stream error: $_failed_msg" + else + die_retryable "API stream error: $_failed_msg" + fi + ;; + error) + printf '%s' "$json" | write_response_file + local _error_msg="" + _error_msg=$(printf '%s' "$json" | jq -r '.error.message // .message // "unknown error"' 2>/dev/null) || _error_msg="unknown error" + if [[ -e "$LLM_DATA_MARKER" ]] || printf '%s' "$json" | _responses_prev_id_error; then + die "API stream error: $_error_msg" + else + die_retryable "API stream error: $_error_msg" + fi + ;; + esac + elif [[ "$_saw_data" -eq 0 ]]; then + _err_buf="${_err_buf}${line} +" + fi + done + _die_if_no_stream_data + if [[ "$_terminal" -ne 1 ]]; then + if [[ -e "$LLM_DATA_MARKER" ]]; then + die "API stream error: Responses stream ended without a terminal response" + fi + die_retryable "API stream error: Responses stream ended without a terminal response" + fi +} + stream_gemini() { local _saw_data=0 _err_buf="" _chunk="" while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do @@ -1339,9 +1644,13 @@ fi # A typo'd provider name would otherwise surface as bash's exit-127 # "build_payload_...: command not found" instead of an error naming the # problem, because payload build runs before get_url_headers' own check. -declare -f "build_payload_${LLM_PROVIDER}" >/dev/null \ - || die "Unknown provider: $LLM_PROVIDER (see --provider in --help for the list)" -"build_payload_${LLM_PROVIDER}" > "$payload_file" +if [[ "$LLM_API_FORMAT" == responses ]]; then + build_payload_responses > "$payload_file" +else + declare -f "build_payload_${LLM_PROVIDER}" >/dev/null \ + || die "Unknown provider: $LLM_PROVIDER (see --provider in --help for the list)" + "build_payload_${LLM_PROVIDER}" > "$payload_file" +fi # Get URL and headers url="" @@ -1377,7 +1686,9 @@ if [[ "$LLM_STREAM" -eq 1 ]]; then LLM_DATA_MARKER="${LLM_RETRY_ERR}.emitted" LLM_STOP_MARKER="${LLM_RETRY_ERR}.stopped" _rc=0 - ( "stream_${LLM_PROVIDER}" < <(curl -sS -N ${_net_args[@]+"${_net_args[@]}"} "${headers[@]}" -d @"$payload_file" "$url" 2>"$_curl_err") ) || _rc=$? + _stream_handler="stream_${LLM_PROVIDER}" + [[ "$LLM_API_FORMAT" == responses ]] && _stream_handler=stream_responses + ( "$_stream_handler" < <(curl -sS -N ${_net_args[@]+"${_net_args[@]}"} "${headers[@]}" -d @"$payload_file" "$url" 2>"$_curl_err") ) || _rc=$? _emitted=0 [[ -e "$LLM_DATA_MARKER" ]] && _emitted=1 @@ -1411,7 +1722,11 @@ if [[ "$LLM_STREAM" -eq 1 ]]; then _retry_msg="empty response: stream ended without emitting anything" _rc=1 fi - if [[ "$_emitted" -eq 0 && "$_attempt" -lt "$_max_attempts" ]]; then + _retryable=0 + [[ "$_rc" -eq 75 || -n "$_ce" ]] && _retryable=1 + [[ "$_retry_msg" == empty\ response:* ]] && _retryable=1 + if [[ "$_emitted" -eq 0 && "$_retryable" -eq 1 \ + && "$_attempt" -lt "$_max_attempts" ]]; then echo "llm: transient API failure (attempt $_attempt/$_max_attempts): ${_retry_msg:-${_ce:-exit $_rc}} — retrying" >&2 sleep $(( LLM_RETRY_BACKOFF * _attempt )) continue @@ -1443,7 +1758,11 @@ else # Nothing has been emitted yet on this path, so retrying is safe. if [[ "$http_code" != "000" && "$http_code" -lt 400 ]]; then if grep -q '[^[:space:]]' "$local_resp"; then - embedded_err=$("check_response_${LLM_PROVIDER}" < "$local_resp") + if [[ "$LLM_API_FORMAT" == responses ]]; then + embedded_err=$(check_response_responses < "$local_resp") + else + embedded_err=$("check_response_${LLM_PROVIDER}" < "$local_resp") + fi else # A whitespace-only 200 is keep-alive padding from a provider # that died mid-generation. The embedded-error checkers @@ -1452,12 +1771,22 @@ else embedded_err="whitespace-only response body" fi if [[ -n "$embedded_err" ]]; then - rm -f "$local_resp" + if [[ "$LLM_API_FORMAT" == responses ]] \ + && jq -e '.status == "failed"' "$local_resp" >/dev/null 2>&1; then + write_response_file < "$local_resp" + rm -f "$local_resp" + die "API error: $embedded_err" + fi if [[ "$_attempt" -lt "$_max_attempts" ]]; then + rm -f "$local_resp" echo "llm: transient API failure (attempt $_attempt/$_max_attempts): $embedded_err — retrying" >&2 sleep $(( LLM_RETRY_BACKOFF * _attempt )) continue fi + if [[ "$LLM_API_FORMAT" == responses ]]; then + printf '%s' "$(cat "$local_resp")" | write_response_file + fi + rm -f "$local_resp" die "API error: $embedded_err" fi fi @@ -1466,6 +1795,16 @@ else if [[ "$http_code" -ge 400 || "$http_code" == "000" ]]; then error_msg=$(jq -r '.error.message // .message // empty' "$local_resp" 2>/dev/null || true) + if [[ "$LLM_API_FORMAT" == responses ]]; then + if [[ -s "$local_resp" ]] && jq -e . "$local_resp" >/dev/null 2>&1; then + write_response_file < "$local_resp" + else + jq -nc --arg message "${error_msg:-HTTP $http_code}" \ + --arg code "$http_code" \ + '{error:{message:$message,http_code:$code}}' \ + | write_response_file + fi + fi rm -f "$local_resp" _llm_note error "$http_code" "${error_msg:-HTTP $http_code}" if [[ -n "$error_msg" ]]; then @@ -1475,16 +1814,28 @@ else fi _llm_note ok - if [[ -n "$("check_truncated_${LLM_PROVIDER}" < "$local_resp")" ]]; then + if [[ "$LLM_API_FORMAT" == responses ]]; then + write_response_file < "$local_resp" + if [[ -n "$(check_truncated_responses < "$local_resp")" ]]; then + warn_truncated + else + _incomplete_reason=$(check_incomplete_responses < "$local_resp") + warn_incomplete_response "$_incomplete_reason" + fi + elif [[ -n "$("check_truncated_${LLM_PROVIDER}" < "$local_resp")" ]]; then warn_truncated fi - case "$LLM_PROVIDER" in - # opencode (Zen) speaks the Anthropic wire protocol, usage shape included. - anthropic|opencode) _u=$(jq -r '[(.usage.input_tokens // ""), (.usage.output_tokens // ""), "", (.usage.cache_read_input_tokens // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; - gemini) _u=$(jq -r '[(.usageMetadata.promptTokenCount // ""), (.usageMetadata.candidatesTokenCount // ""), (.usageMetadata.thoughtsTokenCount // ""), (.usageMetadata.cachedContentTokenCount // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; - *) _u=$(jq -r '[(.usage.prompt_tokens // ""), (.usage.completion_tokens // ""), (.usage.completion_tokens_details.reasoning_tokens // ""), (.usage.prompt_tokens_details.cached_tokens // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; - esac + if [[ "$LLM_API_FORMAT" == responses ]]; then + _u=$(jq -r '[(.usage.input_tokens // ""), (.usage.output_tokens // ""), (.usage.output_tokens_details.reasoning_tokens // ""), (.usage.input_tokens_details.cached_tokens // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" + else + case "$LLM_PROVIDER" in + # opencode (Zen) speaks the Anthropic wire protocol, usage shape included. + anthropic|opencode) _u=$(jq -r '[(.usage.input_tokens // ""), (.usage.output_tokens // ""), "", (.usage.cache_read_input_tokens // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; + gemini) _u=$(jq -r '[(.usageMetadata.promptTokenCount // ""), (.usageMetadata.candidatesTokenCount // ""), (.usageMetadata.thoughtsTokenCount // ""), (.usageMetadata.cachedContentTokenCount // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; + *) _u=$(jq -r '[(.usage.prompt_tokens // ""), (.usage.completion_tokens // ""), (.usage.completion_tokens_details.reasoning_tokens // ""), (.usage.prompt_tokens_details.cached_tokens // "")] | @tsv' "$local_resp" 2>/dev/null) || _u="" ;; + esac + fi if [[ -n "$_u" ]]; then IFS=$'\t' read -r _u_in _u_out _u_think _u_cache <<<"$_u" write_usage "$_u_in" "$_u_out" "$_u_think" "$_u_cache" @@ -1493,6 +1844,9 @@ else if [[ "$LLM_RAW" -eq 1 ]]; then cat "$local_resp" + elif [[ "$LLM_API_FORMAT" == responses ]]; then + extract_reasoning_responses < "$local_resp" >&2 + extract_text_responses < "$local_resp" else "extract_text_${LLM_PROVIDER}" < "$local_resp" fi diff --git a/bin/shellm b/bin/shellm index 90f6c9b..f2d1e12 100755 --- a/bin/shellm +++ b/bin/shellm @@ -71,6 +71,8 @@ SHELLM_MAX_CONSECUTIVE_FAILURES="${SHELLM_MAX_CONSECUTIVE_FAILURES:-10}" SHELLM_MAX_REPEAT_FAILURES="${SHELLM_MAX_REPEAT_FAILURES:-3}" SHELLM_TRUNCATE="${SHELLM_TRUNCATE:-2000}" SHELLM_API_URL="${SHELLM_API_URL:-}" +SHELLM_API_FORMAT="${SHELLM_API_FORMAT:-chat}" +SHELLM_RESPONSES_BODY_FILE="${SHELLM_RESPONSES_BODY_FILE:-}" # Clear any inherited LLM_API_URL so llm uses per-provider defaults unset LLM_API_URL 2>/dev/null || true if [[ -n "$SHELLM_API_URL" ]]; then @@ -143,6 +145,16 @@ _SHELLM_EXTRA_BINS=() die() { echo "shellm: error: $*" >&2; exit 1; } +case "$SHELLM_API_FORMAT" in + chat|responses) ;; + *) die "Invalid SHELLM_API_FORMAT: $SHELLM_API_FORMAT (expected chat or responses)" ;; +esac +if [[ -n "$SHELLM_RESPONSES_BODY_FILE" ]]; then + [[ -f "$SHELLM_RESPONSES_BODY_FILE" ]] \ + || die "SHELLM_RESPONSES_BODY_FILE not found: $SHELLM_RESPONSES_BODY_FILE" + SHELLM_RESPONSES_BODY_FILE=$(realpath "$SHELLM_RESPONSES_BODY_FILE") +fi + # A local model bound to the Docker host is usually configured as localhost # for thinkers and other llm calls that run on that host. Code executed in a # shellm sandbox needs Docker's hostname for the same server. The rewrite @@ -305,6 +317,13 @@ Environment: SHELLM_CONTEXT_WHOLE_BLOCKS How many newest blocks get the larger limit (default 1); in traj scope, the newest N rows + SHELLM_API_FORMAT Completion protocol: chat (default) or responses. + Responses persists and reuses previous_response_id + within a run, with one full-context fallback when a + provider rejects continuation + SHELLM_RESPONSES_BODY_FILE + Optional Responses create-body JSON object passed to + llm (tools, include, store, text format, etc.) SHELLM_EMPTY_RESPONSE_RETRIES Empty-response retries before the run dies (default 8; set empty for unlimited) @@ -1527,64 +1546,237 @@ execute_code() { call_llm() { local system_prompt="$1" - local messages_json="$2" + local full_messages_json="$2" local thinking_output_file="${3:-}" + local messages_are_delta="${4:-0}" + local api_format="${SHELLM_API_FORMAT:-chat}" + local response_file="${SHELLM_RESPONSE_FILE:-}" + local response_id_file="${SHELLM_RESPONSE_ID_FILE:-}" + local disabled_file="${SHELLM_RESPONSES_DISABLED_FILE:-}" + local replay_file="${SHELLM_RESPONSES_REPLAY_FILE:-}" + local context_file="${SHELLM_RESPONSES_CONTEXT_FILE:-}" + local stateless="${SHELLM_RESPONSES_STATELESS:-0}" + local request_messages_json="$full_messages_json" + local new_messages_json="$full_messages_json" + local previous_response_id="" + local fallback_used=0 + + export LLM_API_FORMAT="$api_format" + if [[ "$api_format" == "responses" ]]; then + if [[ -n "$response_file" ]]; then + export LLM_RESPONSE_FILE="$response_file" + else + unset LLM_RESPONSE_FILE 2>/dev/null || true + fi + if [[ -n "${SHELLM_RESPONSES_BODY_FILE:-}" ]]; then + export LLM_RESPONSES_BODY_FILE="$SHELLM_RESPONSES_BODY_FILE" + else + unset LLM_RESPONSES_BODY_FILE 2>/dev/null || true + fi + local replay_has_history=0 + if [[ -n "$replay_file" && -s "$replay_file" ]] \ + && jq -e 'type == "array" and length > 0' "$replay_file" >/dev/null 2>&1; then + replay_has_history=1 + fi + if [[ "$stateless" != "1" && -n "$response_id_file" && -f "$response_id_file" \ + && ( -z "$disabled_file" || ! -e "$disabled_file" ) ]]; then + previous_response_id=$(cat "$response_id_file") + fi + if [[ "$messages_are_delta" != "1" \ + && ( -n "$previous_response_id" || "$replay_has_history" -eq 1 ) ]]; then + [[ -n "$context_file" && -s "$context_file" ]] \ + || die "Responses continuation context is missing" + new_messages_json=$(printf '%s' "$full_messages_json" | jq --slurpfile previous "$context_file" ' + ($previous[0] // null) as $prior + | if ($prior | type) != "array" or type != "array" + or .[0:($prior | length)] != $prior + then error("previous Responses context is not a prefix of the rendered context") + else .[($prior | length):] + end + | . as $new + | (reduce range(0; length) as $i + (0; if . == $i and $new[$i].role == "assistant" then . + 1 else . end)) as $start + | $new[$start:] + ') || die "Responses continuation boundary fell outside the rendered context; increase SHELLM_CONTEXT_RUN_TAIL" + fi + if [[ "$stateless" == "1" || ( -n "$disabled_file" && -e "$disabled_file" ) ]]; then + if [[ "$replay_has_history" -eq 1 ]]; then + request_messages_json=$(printf '%s' "$new_messages_json" \ + | jq --slurpfile history "$replay_file" '($history[0] // []) + .') + else + request_messages_json="$new_messages_json" + fi + elif [[ -n "$previous_response_id" ]]; then + request_messages_json="$new_messages_json" + fi + else + unset LLM_RESPONSE_FILE LLM_RESPONSES_BODY_FILE LLM_PREVIOUS_RESPONSE_ID \ + 2>/dev/null || true + fi + + while true; do + # The message array rides a file, not argv: a long conversation (or the + # empty-response retry's fed-back thinking) can exceed the OS's per-argument + # cap (128KB on Linux), and exec would die with E2BIG before the call. + local messages_file text_file stderr_file + messages_file=$(mktemp) + text_file=$(mktemp) + stderr_file=$(mktemp) + printf '%s' "$request_messages_json" > "$messages_file" + + local -a args=(-m "$SHELLM_MODEL" + --effort "$SHELLM_EFFORT" --thinking --system-prompt "$system_prompt" --messages-file "$messages_file") + [[ -n "$SHELLM_MAX_TOKENS" ]] && args+=(-t "$SHELLM_MAX_TOKENS") + # Responses needs the terminal event (id, usage, status). Cutting the + # stream at the first fence would drop it and break continuation. + [[ "${SHELLM_STOP_AFTER_CODE_BLOCK:-1}" == 1 && "$api_format" != responses ]] \ + && args+=(--stop-after-code-block) + + if [[ "$api_format" == "responses" ]]; then + if [[ -n "$previous_response_id" ]]; then + export LLM_PREVIOUS_RESPONSE_ID="$previous_response_id" + else + unset LLM_PREVIOUS_RESPONSE_ID 2>/dev/null || true + fi + [[ -z "$response_file" ]] || rm -f "$response_file" + fi - # The message array rides a file, not argv: a long conversation (or the - # empty-response retry's fed-back thinking) can exceed the OS's per-argument - # cap (128KB on Linux), and exec would die with E2BIG before the call. - local messages_file - messages_file=$(mktemp) - printf '%s' "$messages_json" > "$messages_file" - - local -a args=(-m "$SHELLM_MODEL" - --effort "$SHELLM_EFFORT" --thinking --system-prompt "$system_prompt" --messages-file "$messages_file") - [[ -n "$SHELLM_MAX_TOKENS" ]] && args+=(-t "$SHELLM_MAX_TOKENS") - [[ "${SHELLM_STOP_AFTER_CODE_BLOCK:-1}" == 1 ]] && args+=(--stop-after-code-block) - - local text_file stderr_file - text_file=$(mktemp) - stderr_file=$(mktemp) - - # Run llm, capturing stdout to text_file and stderr to stderr_file. - # When not quiet, also tee stdout to stderr (for live display) and - # stderr to the terminal. Use a subshell to capture llm's exit code - # despite the pipeline. - local llm_exit=0 - if [[ "$QUIET" -eq 0 ]]; then - { llm "${args[@]}" 2>"$stderr_file"; echo $? > "$text_file.rc"; } | tee "$text_file" >&2 - llm_exit=$(cat "$text_file.rc" 2>/dev/null) || llm_exit=0 - rm -f "$text_file.rc" - # Replay stderr (thinking output + errors) to terminal and thinking file - if [[ -s "$stderr_file" ]]; then - cat "$stderr_file" >&2 + # Run llm, capturing stdout to text_file and stderr to stderr_file. + # When not quiet, also tee stdout to stderr (for live display) and + # stderr to the terminal. Use a subshell to capture llm's exit code + # despite the pipeline. + local llm_exit=0 + if [[ "$QUIET" -eq 0 ]]; then + { llm "${args[@]}" 2>"$stderr_file"; echo $? > "$text_file.rc"; } | tee "$text_file" >&2 + llm_exit=$(cat "$text_file.rc" 2>/dev/null) || llm_exit=0 + rm -f "$text_file.rc" + # Replay stderr (thinking output + errors) to terminal and thinking file + if [[ -s "$stderr_file" ]]; then + cat "$stderr_file" >&2 + [[ -n "$thinking_output_file" ]] && cp "$stderr_file" "$thinking_output_file" + fi + else + llm "${args[@]}" > "$text_file" 2>"$stderr_file" || llm_exit=$? [[ -n "$thinking_output_file" ]] && cp "$stderr_file" "$thinking_output_file" fi - else - llm "${args[@]}" > "$text_file" 2>"$stderr_file" || llm_exit=$? - [[ -n "$thinking_output_file" ]] && cp "$stderr_file" "$thinking_output_file" - fi - # A failed llm call must never yield its partial stream as a response: - # a truncated code block that still parses would get executed. Stall - # timeouts (bin/llm net guards) land here after llm's own retries. - if [[ "$llm_exit" -ne 0 ]]; then + if [[ "$llm_exit" -eq 0 ]]; then + if [[ "$api_format" == "responses" ]] \ + && { [[ -z "$response_file" || ! -s "$response_file" ]] \ + || ! jq -e '.status == "completed" or .status == "incomplete"' \ + "$response_file" >/dev/null 2>&1; }; then + rm -f "$text_file" "$stderr_file" "$messages_file" + die "llm returned Responses output without a terminal response" + fi + if [[ "$api_format" == "responses" && -n "$response_file" \ + && -s "$response_file" ]] \ + && jq -e '.status == "completed" or .status == "incomplete"' \ + "$response_file" >/dev/null 2>&1; then + # Keep an exact process-local replay chain in parallel with + # stateful continuation. It is the primary path for OpenRouter + # and the safe fallback for ZDR/expired compatible endpoints. + if [[ -n "$replay_file" ]]; then + local replay_dir replay_tmp turn_tmp + replay_dir=$(dirname "$replay_file") + mkdir -p "$replay_dir" + chmod 700 "$replay_dir" 2>/dev/null || true + [[ -s "$replay_file" ]] \ + || { ( umask 077; printf '[]' > "$replay_file" ); chmod 600 "$replay_file"; } + replay_tmp=$(mktemp "$replay_dir/.responses-replay.XXXXXX") + turn_tmp=$(mktemp) + chmod 600 "$replay_tmp" + printf '%s' "$new_messages_json" > "$turn_tmp" + jq -s '.[0] + .[1] + (.[2].output // [])' \ + "$replay_file" "$turn_tmp" "$response_file" > "$replay_tmp" + mv -f "$replay_tmp" "$replay_file" + rm -f "$turn_tmp" + fi + if [[ -n "$context_file" && "$messages_are_delta" != "1" ]]; then + local context_dir context_tmp + context_dir=$(dirname "$context_file") + mkdir -p "$context_dir" + chmod 700 "$context_dir" 2>/dev/null || true + context_tmp=$(mktemp "$context_dir/.responses-context.XXXXXX") + chmod 600 "$context_tmp" + printf '%s' "$full_messages_json" > "$context_tmp" + mv -f "$context_tmp" "$context_file" + fi + + local terminal_response_id + terminal_response_id=$(jq -r ' + select((.status == "completed" or .status == "incomplete") + and (.id | type == "string") and (.id | length > 0)) + | .id + ' "$response_file" 2>/dev/null || true) + if [[ "$stateless" != "1" && -n "$response_id_file" \ + && -n "$terminal_response_id" \ + && ( -z "$disabled_file" || ! -e "$disabled_file" ) ]]; then + local id_dir id_tmp + id_dir=$(dirname "$response_id_file") + mkdir -p "$id_dir" + chmod 700 "$id_dir" 2>/dev/null || true + id_tmp=$(mktemp "$id_dir/.response-id.XXXXXX") + chmod 600 "$id_tmp" + printf '%s\n' "$terminal_response_id" > "$id_tmp" + mv -f "$id_tmp" "$response_id_file" + fi + fi + + # No visible text is not a failure. `llm --thinking` streams reasoning to + # stderr, and a run that spends its whole token budget there returns 200, + # exits 0 and emits nothing on stdout, which is exactly what run_loop's + # empty-response retry feeds back as assistant context. Real failures exit + # non-zero and are caught below. + cat "$text_file" + rm -f "$text_file" "$stderr_file" "$messages_file" + return 0 + fi + + # Providers may reject a previous_response_id after expiry or when a + # compatible endpoint does not implement continuation. Retry once with + # the exact input/output-item replay chain, then leave continuation + # disabled so an endpoint cannot trap shellm in a fallback loop. + if [[ "$api_format" == "responses" && -n "$previous_response_id" \ + && "$fallback_used" -eq 0 && -n "$response_file" && -s "$response_file" ]] \ + && jq -e '[.error.param?,.error.code?,.error.message?,.param?,.code?,.message?,.detail?] + | map(select(. != null) | tostring | ascii_downcase) | join(" ") + | test("previous[ _]response[ _]id|previous response")' \ + "$response_file" >/dev/null 2>&1; then + [[ -z "$response_id_file" ]] || rm -f "$response_id_file" + if [[ -n "$disabled_file" ]]; then + local disabled_dir disabled_tmp + disabled_dir=$(dirname "$disabled_file") + mkdir -p "$disabled_dir" + chmod 700 "$disabled_dir" 2>/dev/null || true + disabled_tmp=$(mktemp "$disabled_dir/.continuation-disabled.XXXXXX") + chmod 600 "$disabled_tmp" + printf 'provider rejected previous_response_id\n' > "$disabled_tmp" + mv -f "$disabled_tmp" "$disabled_file" + fi + previous_response_id="" + unset LLM_PREVIOUS_RESPONSE_ID 2>/dev/null || true + if [[ -n "$replay_file" && -s "$replay_file" ]] \ + && jq -e 'type == "array" and length > 0' "$replay_file" >/dev/null 2>&1; then + request_messages_json=$(printf '%s' "$new_messages_json" \ + | jq --slurpfile history "$replay_file" '($history[0] // []) + .') + else + request_messages_json="$full_messages_json" + fi + fallback_used=1 + rm -f "$text_file" "$stderr_file" "$messages_file" + [[ -z "$thinking_output_file" ]] || : > "$thinking_output_file" + progress "responses continuation rejected; retrying once with exact replay context" + continue + fi + + # A failed llm call must never yield its partial stream as a response: + # a truncated code block that still parses would get executed. Stall + # timeouts (bin/llm net guards) land here after llm's own retries. local _llm_err="" [[ -s "$stderr_file" ]] && _llm_err=$(tail -c 500 "$stderr_file") rm -f "$text_file" "$stderr_file" "$messages_file" die "llm failed (exit $llm_exit): $_llm_err" - fi - - # No visible text is not a failure. `llm --thinking` streams reasoning to - # stderr, and a run that spends its whole token budget there returns 200, - # exits 0 and emits nothing on stdout, which is exactly what run_loop's - # empty-response retry feeds back as assistant context. Dying on a - # non-empty stderr here made that retry unreachable and printed the - # model's reasoning as an error. Real failures exit non-zero and are - # caught above. - - cat "$text_file" - rm -f "$text_file" "$stderr_file" "$messages_file" + done } # --------------------------------------------------------------------------- @@ -2350,6 +2542,7 @@ run_loop() { --arg wd "$workdir" \ --arg model "$SHELLM_MODEL" \ --arg effort "$SHELLM_EFFORT" \ + --arg api_format "$SHELLM_API_FORMAT" \ --arg max_iter "${SHELLM_MAX_ITERATIONS:-}" \ --arg max_tok "${SHELLM_MAX_TOKENS:-}" \ --arg inactivity_timeout "$SHELLM_INACTIVITY_TIMEOUT" \ @@ -2359,7 +2552,7 @@ run_loop() { --arg trigger "${SHELLM_TRIGGER_STEP_ID:-}" \ --arg launched_by "${SHELLM_LAUNCHED_BY:-}" \ --arg wake "${SHELLM_WAKE:-}" \ - '{type:"shellm-run", command:$cmd, workdir:$wd, model:$model, effort:$effort, max_iterations:$max_iter, max_tokens:$max_tok, inactivity_timeout:$inactivity_timeout, context_files:$ctx, env:$env, resumed:$resumed} + '{type:"shellm-run", command:$cmd, workdir:$wd, model:$model, effort:$effort, api_format:$api_format, max_iterations:$max_iter, max_tokens:$max_tok, inactivity_timeout:$inactivity_timeout, context_files:$ctx, env:$env, resumed:$resumed} + (if $trigger == "" then {} else {trigger_step: $trigger} end) + (if $launched_by == "" then {} else {launched_by: $launched_by} end) + (if $wake == "" then {} else {wake: $wake} end)') @@ -2368,6 +2561,35 @@ run_loop() { # interleave in one shared trajectory) local _run_step_id _prompt_step_id="" _run_step_id=$(printf '%s' "$_shellm_run_json" | traj append --traj_dir "$_run_traj_dir" "$_run_traj_id") + + # Responses continuation belongs to this process only. A resumed shellm + # starts a fresh chain from its trajectory; remote response IDs and exact + # output-item replay state disappear with rundir at process cleanup. + local SHELLM_RESPONSE_FILE="" + local SHELLM_RESPONSE_ID_FILE="" + local SHELLM_RESPONSES_DISABLED_FILE="" + local SHELLM_RESPONSES_REPLAY_FILE="" + local SHELLM_RESPONSES_CONTEXT_FILE="" + local SHELLM_RESPONSES_STATELESS=0 + if [[ "$SHELLM_API_FORMAT" == "responses" ]]; then + SHELLM_RESPONSE_FILE="$rundir/.last_response.json" + SHELLM_RESPONSE_ID_FILE="$rundir/.response-id" + SHELLM_RESPONSES_DISABLED_FILE="$rundir/.continuation-disabled" + SHELLM_RESPONSES_REPLAY_FILE="$rundir/.responses-replay.json" + SHELLM_RESPONSES_CONTEXT_FILE="$rundir/.responses-context.json" + ( umask 077; printf '[]' > "$SHELLM_RESPONSES_REPLAY_FILE" ) + chmod 600 "$SHELLM_RESPONSES_REPLAY_FILE" + + # OpenRouter documents /responses as stateless: store=true and a + # non-null previous_response_id are rejected. Native OpenAI and + # generic compatible endpoints optimistically use continuation and + # fall back to this same exact replay chain when needed. + if [[ "${LLM_PROVIDER:-}" == "openrouter" \ + || ( -z "${LLM_PROVIDER:-}" && "$SHELLM_MODEL" == */* ) ]]; then + SHELLM_RESPONSES_STATELESS=1 + fi + fi + # Launcher opt-in: report this run's header step_id to the given file so # the caller can pair its own steps with this exact run — the trajectory # alone can't tell whose header is whose when concurrent runs interleave. @@ -2507,44 +2729,67 @@ run_loop() { fi rm -f "$thinking_file" - # Retry on empty response: feed captured thinking back as assistant - # context so the model continues from where it left off (max_tokens - # gets hit during thinking before any visible output is emitted). + # Retry on empty Chat output by feeding captured thinking back as + # assistant context. Responses mode keeps reasoning in the terminal + # Response item: continue from its ID/exact replay instead of inventing + # an assistant message containing a reasoning summary. # Default: 8 retries; SHELLM_EMPTY_RESPONSE_RETRIES overrides (set it # to the empty string for unlimited). local empty_retries=0 local max_empty_retries="${SHELLM_EMPTY_RESPONSE_RETRIES-8}" local retry_messages_json="$messages_json" while [[ -z "$response" ]]; do + if [[ "$SHELLM_API_FORMAT" == "responses" ]]; then + if [[ -z "${SHELLM_RESPONSE_FILE:-}" || ! -s "$SHELLM_RESPONSE_FILE" ]]; then + die "Empty Responses output without a terminal Response object" + fi + if jq -e '.output[]? | select(.type == "function_call")' \ + "$SHELLM_RESPONSE_FILE" >/dev/null 2>&1; then + die "Responses returned function calls without visible shellm output; consume them through LLM_RESPONSE_FILE" + fi + if ! jq -e ' + .status == "incomplete" + and .incomplete_details.reason == "max_output_tokens" + ' "$SHELLM_RESPONSE_FILE" >/dev/null 2>&1; then + local response_status + response_status=$(jq -r '.status // "unknown"' \ + "$SHELLM_RESPONSE_FILE" 2>/dev/null || printf 'unknown') + die "Empty Responses output with terminal status $response_status" + fi + fi if [[ -n "$max_empty_retries" && "$empty_retries" -ge "$max_empty_retries" ]]; then break fi empty_retries=$((empty_retries + 1)) - # llm's own stderr lines ("llm: warning: output truncated...") - # arrive mixed into the captured thinking; they are harness noise, - # not the model's reasoning, and must not become assistant content. - local feedback - feedback=$(printf '%s\n' "$thinking" | grep -Ev '^llm: (warning|note): ') || feedback="" - if [[ -n "$feedback" ]]; then - progress "Empty response — feeding thinking back to continue (retry $empty_retries${max_empty_retries:+/$max_empty_retries})..." - debug "empty_response_retry: fed back ${#feedback} chars of thinking" - # --rawfile, not --arg: a full-budget thinking block can exceed - # the OS's 128KB per-argument cap. - local feedback_file - feedback_file=$(mktemp) - printf '%s' "$feedback" > "$feedback_file" - retry_messages_json=$(printf '%s' "$retry_messages_json" | jq \ - --rawfile t "$feedback_file" \ - '. + [{"role":"assistant","content":$t},{"role":"user","content":"Your previous response ran out of output tokens during the reasoning above before you could emit any visible text. Continue from where you left off and produce the bash code for the next step now."}]') - rm -f "$feedback_file" - sleep 1 + if [[ "$SHELLM_API_FORMAT" == "responses" ]]; then + progress "Incomplete Responses output — continuing from terminal response state (retry $empty_retries${max_empty_retries:+/$max_empty_retries})..." + retry_messages_json='[{"role":"user","content":"Continue from the incomplete response and emit the bash code for the next step."}]' else - progress "Empty response from API (retry $empty_retries${max_empty_retries:+/$max_empty_retries})..." - sleep 1 + # llm's own stderr lines ("llm: warning: output truncated...") + # arrive mixed into the captured thinking; they are harness noise, + # not the model's reasoning, and must not become assistant content. + local feedback + feedback=$(printf '%s\n' "$thinking" | grep -Ev '^llm: (warning|note): ') || feedback="" + if [[ -n "$feedback" ]]; then + progress "Empty response — feeding thinking back to continue (retry $empty_retries${max_empty_retries:+/$max_empty_retries})..." + debug "empty_response_retry: fed back ${#feedback} chars of thinking" + # --rawfile, not --arg: a full-budget thinking block can exceed + # the OS's 128KB per-argument cap. + local feedback_file + feedback_file=$(mktemp) + printf '%s' "$feedback" > "$feedback_file" + retry_messages_json=$(printf '%s' "$retry_messages_json" | jq \ + --rawfile t "$feedback_file" \ + '. + [{"role":"assistant","content":$t},{"role":"user","content":"Your previous response ran out of output tokens during the reasoning above before you could emit any visible text. Continue from where you left off and produce the bash code for the next step now."}]') + rm -f "$feedback_file" + else + progress "Empty response from API (retry $empty_retries${max_empty_retries:+/$max_empty_retries})..." + fi fi + sleep 1 : > "$thinking_file" - response=$(call_llm "$system_prompt" "$retry_messages_json" "$thinking_file") + response=$(call_llm "$system_prompt" "$retry_messages_json" "$thinking_file" 1) thinking="" if [[ -s "$thinking_file" ]]; then @@ -2640,6 +2885,10 @@ exit \$__shellm_rc SHELLM_MAX_ITERATIONS="$SHELLM_MAX_ITERATIONS" SHELLM_EFFORT="$SHELLM_EFFORT" SHELLM_TRUNCATE="$SHELLM_TRUNCATE" + SHELLM_API_FORMAT="$SHELLM_API_FORMAT" + LLM_API_FORMAT="$SHELLM_API_FORMAT" + SHELLM_RESPONSES_BODY_FILE="$SHELLM_RESPONSES_BODY_FILE" + LLM_RESPONSES_BODY_FILE="$SHELLM_RESPONSES_BODY_FILE" SHELLM_API_URL="$execution_api_url" LLM_API_URL="$execution_api_url" SHELLM_DOCKER_IMAGE="$SHELLM_DOCKER_IMAGE" diff --git a/design/providers.md b/design/providers.md index a6bf718..40050e7 100644 --- a/design/providers.md +++ b/design/providers.md @@ -58,8 +58,9 @@ llm -m qwen3:8b "hello" | Variable | Meaning | |---|---| | `LLM_PROVIDER=openai-compatible` | Selects the provider (or pass `--provider openai-compatible`) | -| `LLM_API_URL` | The chat-completions endpoint. Required, no default | +| `LLM_API_URL` | The exact chat-completions or Responses endpoint. Required, no default | | `LLM_API_KEY` | Optional. When set, sent as `Authorization: Bearer` | +| `LLM_API_FORMAT` | `chat` (default) or `responses` | What "compatible" means here, concretely: a chat-completions endpoint that accepts `model`, `messages`, and `max_tokens`; non-streaming @@ -70,6 +71,15 @@ That subset is what the code exercises and the tests pin. An endpoint that diverges from it is best effort — it may well work, but the divergence is not a core bug to absorb. +With `LLM_API_FORMAT=responses`, compatibility instead means the synchronous +OpenAI Responses create protocol at the exact configured URL: typed `input` +items, terminal `output` items and status, Responses SSE events, and structured +errors. `LLM_RESPONSES_BODY_FILE` carries create fields beyond the completion +CLI's stable flags, while `LLM_RESPONSE_FILE` receives the full terminal object +or error envelope. Server-side lifecycle operations (retrieve, cancel, delete, +Conversations, background jobs, and WebSocket sessions) are not part of this +completion-provider seam. + `LLM_PROVIDER` is process-wide by design (decided 2026-08-26: environment overrides are authoritative, never pattern-guessed around), so selecting `openai-compatible` routes every completion in the diff --git a/design/responses-api.md b/design/responses-api.md new file mode 100644 index 0000000..d2890c1 --- /dev/null +++ b/design/responses-api.md @@ -0,0 +1,93 @@ +# OpenAI Responses completion protocol + +Status: implemented 2026-09-01. + +## Scope + +Headlong's completion boundary remains `bin/llm`. Chat Completions remains the +default. Operators opt into the OpenAI Responses create protocol with +`LLM_API_FORMAT=responses` for the `openai`, `openrouter`, or +`openai-compatible` providers. + +This change covers synchronous buffered and SSE response creation, including +reasoning summaries, function call items and outputs, structured and +multimodal input items, terminal status and errors, usage, and continuation. +Response retrieval/deletion/cancellation, input-item listing, Conversations, +background responses, and WebSocket mode are separate lifecycle work. + +## Wire contract + +- Native OpenAI defaults to `https://api.openai.com/v1/responses` in Responses + mode. OpenRouter defaults to `https://openrouter.ai/api/v1/responses`. + `openai-compatible` still requires the exact `LLM_API_URL` endpoint. +- The existing messages input is passed as the Responses `input` array without + reshaping. This preserves typed input/output items, images, files, assistant + phases, reasoning items, function calls, and `function_call_output` items. +- The system prompt maps to `instructions`, the token cap maps to + `max_output_tokens`, and an explicit thinking level maps to + `reasoning.effort` with an automatic summary. +- `LLM_RESPONSES_BODY_FILE` may name a JSON object containing other synchronous + create fields. `bin/llm` owns and overwrites `model`, `input`, `instructions`, + `max_output_tokens`, `stream`, and `previous_response_id` so command-line and + continuation semantics remain deterministic. Conversation state is rejected + because it conflicts with this continuation contract. +- Every create requests `reasoning.encrypted_content`, preserving exact + reasoning-item replay for stateless and Zero Data Retention paths while + retaining any other caller-supplied `include` values. +- `LLM_PREVIOUS_RESPONSE_ID` adds stateful continuation. +- `LLM_RESPONSE_FILE`, when set, receives the complete terminal Response object + or provider error envelope through an atomic mode-0600 write. It is the + machine-readable channel for response IDs, all output items, function calls, + encrypted reasoning, status, errors, and usage. + +The human-output contract does not change: visible `output_text` is stdout, +reasoning summaries are stderr, and `--raw` prints the buffered API object. +A function-only response is a successful protocol response even though stdout +is empty; callers consume its items from `LLM_RESPONSE_FILE`. + +## Streaming and failure semantics + +The SSE handler emits text and reasoning deltas as they arrive, records the +terminal response from `response.completed`, `response.incomplete`, or +`response.failed`, and maps Responses usage into the existing usage record. +Incomplete responses warn with their reason. Failed responses and `error` +events fail the call. + +Retries remain legal only before protocol output is emitted. A terminal output +item counts as output even when it is a function call with no visible text. +After a text, reasoning, or output-item event, a truncated or failed stream is +never replayed automatically. + +## shellm continuation + +Responses mode keeps completion state only for the current `shellm` process: + +1. The first call sends the trajectory-derived context in full. +2. Later calls send only newly appended user-side context plus the stable + instructions and the previous response ID. +3. In parallel, shellm retains the original input and every terminal output + item. This exact replay chain preserves encrypted reasoning and assistant + `phase` values for stateless endpoints and Zero Data Retention accounts. +4. If a continuation is rejected specifically because the previous response + cannot be referenced, before any output is emitted, shellm retries once with + the replay chain and remains stateless for the rest of the run. +5. A resumed process starts a new chain from the durable trajectory. Remote + response IDs are not persisted as durable trajectory state. + +OpenRouter's Responses endpoint is stateless and therefore starts directly in +replay mode. Native OpenAI and generic compatible endpoints use automatic +stateful continuation with the safe replay fallback. + +The existing thinking-text empty-response workaround remains the Chat +Completions behavior. In Responses mode, an incomplete reasoning-only Response +continues through its response ID or exact output-item replay instead of +turning a reasoning summary into an invented assistant message. + +## Verification + +Hermetic tests pin request JSON, endpoint selection, pass-through input, +extra-body validation and precedence, buffered extraction, terminal sidecar +permissions, response status and usage, SSE event classes, function-only +success, stateful shellm deltas, stateless replay, continuation fallback, and +unchanged Chat behavior. The implementation is additionally smoke-tested +against native OpenAI and an independent OpenAI-compatible Responses endpoint. diff --git a/docs/shellm.md b/docs/shellm.md index d6d5f85..aadc358 100644 --- a/docs/shellm.md +++ b/docs/shellm.md @@ -288,6 +288,39 @@ LLM_API_URL=http://localhost:11434/v1/chat/completions \ llm -m qwen3:8b "hello" ``` +OpenAI's Responses completion protocol is opt-in. Native OpenAI and +OpenRouter choose their `/responses` endpoint automatically; +`openai-compatible` uses the exact configured URL without appending a path: + +```bash +# Native OpenAI Responses, with the complete terminal object kept for tools, +# typed output items, response IDs, status, and usage. +LLM_API_FORMAT=responses \ +LLM_RESPONSE_FILE=/tmp/response.json \ +llm --provider openai -m gpt-5.5 --thinking high "solve carefully" + +# A compatible Responses endpoint. +LLM_API_FORMAT=responses \ +LLM_PROVIDER=openai-compatible \ +LLM_API_URL=https://router.example/v1/responses \ +LLM_API_KEY=... \ +llm -m routed-model "hello" +``` + +`LLM_RESPONSES_BODY_FILE` may name a JSON object with other synchronous +Responses create fields, including `tools`, `include`, `store`, `metadata`, +`text.format`, `truncation`, and provider-specific extensions. `llm` owns the +fields coupled to its CLI (`model`, `input`, `instructions`, +`previous_response_id`, `max_output_tokens`, `stream`, and command-line +reasoning settings). Typed message, image, file, reasoning, function-call, and +function-output items in `--messages-file` pass through unchanged. + +Visible output text remains stdout; reasoning summaries go to stderr. The +mode-0600 `LLM_RESPONSE_FILE` sidecar is the machine channel for the complete +terminal Response or error envelope, including function-only results. Retrieval, +cancellation, deletion, Conversations, background mode, and WebSocket sessions +are lifecycle APIs and are not completion operations in this CLI. + Set `LLM_API_KEY` if the endpoint wants a bearer token. The policy for which providers live in core is in [design/providers.md](../design/providers.md). @@ -310,13 +343,21 @@ shellm "what os is this?" nested shellm runs the same way the vendor keys are, so `llm` calls inside generated code reach the endpoint too. +To run shellm itself on Responses, set `SHELLM_API_FORMAT=responses` (and +`SHELLM_RESPONSES_BODY_FILE` for extra create fields). Native OpenAI and generic +compatible endpoints use `previous_response_id` while retaining an exact +process-local replay chain. If an endpoint rejects continuation, shellm retries +once with that chain and stays stateless for the rest of the run. OpenRouter's +documented stateless Responses endpoint uses exact replay from the first turn. +Remote response IDs and replay items are removed when the shellm process exits. + A provider that can't speak this protocol (an SDK, a vendor CLI, signed requests) runs outside core as an adapter: set `LLM_PROVIDER=adapter` and `LLM_ADAPTER=/path/to/executable`, and `llm` runs that executable in place of curl. The adapter contract is in [design/providers.md](../design/providers.md). -**Output contract:** stdout = text response, stderr = thinking tokens (Anthropic only), exit 0 = success. This makes it composable with pipes and subshells. +**Output contract:** stdout = text response, stderr = thinking/reasoning output, exit 0 = success. This makes it composable with pipes and subshells. ## mem and skills diff --git a/tests/test_llm_responses.sh b/tests/test_llm_responses.sh new file mode 100644 index 0000000..afc2529 --- /dev/null +++ b/tests/test_llm_responses.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# test_llm_responses.sh — OpenAI Responses completion protocol in bin/llm + +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(dirname "$HERE")" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +pass=0 +fail=0 +ok() { pass=$((pass+1)); printf 'ok %s\n' "$1"; } +bad() { fail=$((fail+1)); printf 'FAIL %s%s\n' "$1" "${2:+ — $2}"; } + +mkdir -p "$WORK/bin" "$WORK/home" +cat > "$WORK/bin/curl" <<'STUB' +#!/usr/bin/env bash +n=0 +[[ -f "$CURL_CALLS" ]] && read -r n < "$CURL_CALLS" +printf '%s\n' "$((n + 1))" > "$CURL_CALLS" +printf '%s\n' "$@" > "$CURL_ARGS" +out_file="" +payload_file="" +prev="" +for arg in "$@"; do + [[ "$prev" == "-o" ]] && out_file="$arg" + [[ "$prev" == "-d" && "$arg" == @* ]] && payload_file="${arg#@}" + prev="$arg" +done +[[ -n "$payload_file" ]] && cp "$payload_file" "$CURL_PAYLOAD" + +buffered_completed='{ + "id":"resp_buffered", + "object":"response", + "status":"completed", + "output":[ + {"id":"rs_1","type":"reasoning","summary":[{"type":"summary_text","text":"brief reasoning"}]}, + {"id":"msg_1","type":"message","role":"assistant","status":"completed","phase":"final_answer","content":[{"type":"output_text","text":"hello"}]} + ], + "usage":{"input_tokens":21,"output_tokens":8,"output_tokens_details":{"reasoning_tokens":3}} +}' + +case "$CURL_MODE" in + buffered-completed) + printf '%s' "$buffered_completed" > "$out_file" + printf '200' + ;; + buffered-multipart) + printf '%s' '{"id":"resp_parts","object":"response","status":"completed","output":[{"id":"msg_parts","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"alpha"},{"type":"output_text","text":"\nbeta"},{"type":"refusal","refusal":"denied"}]}]}' > "$out_file" + printf '200' + ;; + buffered-function) + printf '%s' '{"id":"resp_fn","object":"response","status":"completed","output":[{"id":"fc_1","type":"function_call","call_id":"call_1","name":"weather","arguments":"{\"city\":\"Paris\"}","status":"completed"}],"usage":{"input_tokens":9,"output_tokens":4}}' > "$out_file" + printf '200' + ;; + buffered-incomplete) + printf '%s' '{"id":"resp_short","object":"response","status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"output":[{"id":"msg_2","type":"message","role":"assistant","status":"incomplete","content":[{"type":"output_text","text":"partial"}]}],"usage":{"input_tokens":7,"output_tokens":5}}' > "$out_file" + printf '200' + ;; + buffered-failed) + printf '%s' '{"id":"resp_failed","object":"response","status":"failed","error":{"code":"server_error","message":"generation failed"},"output":[]}' > "$out_file" + printf '200' + ;; + http-400-previous) + printf '%s' '{"error":{"message":"Previous response cannot be used for this organization due to Zero Data Retention.","type":"invalid_request_error","param":"previous_response_id","code":"unsupported_parameter"}}' > "$out_file" + printf '400' + ;; + stream-completed) + printf '%s\n\n' 'event: response.created' 'data: {"type":"response.created","response":{"id":"resp_stream","status":"in_progress","output":[]}}' + printf '%s\n\n' 'event: response.reasoning_summary_text.delta' 'data: {"type":"response.reasoning_summary_text.delta","delta":"stream reasoning"}' + printf '%s\n\n' 'event: response.output_text.delta' 'data: {"type":"response.output_text.delta","delta":"line one\n"}' + printf '%s\n\n' 'event: response.output_text.delta' 'data: {"type":"response.output_text.delta","delta":"line two"}' + printf '%s\n\n' 'event: response.completed' 'data: {"type":"response.completed","response":{"id":"resp_stream","object":"response","status":"completed","output":[{"id":"rs_s","type":"reasoning","summary":[{"type":"summary_text","text":"stream reasoning"}]},{"id":"msg_s","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"line one\nline two"}]}],"usage":{"input_tokens":30,"output_tokens":12,"output_tokens_details":{"reasoning_tokens":4}}}}' + ;; + stream-function) + printf '%s\n\n' 'event: response.output_item.done' 'data: {"type":"response.output_item.done","item":{"id":"fc_s","type":"function_call","call_id":"call_s","name":"weather","arguments":"{}","status":"completed"}}' + printf '%s\n\n' 'event: response.completed' 'data: {"type":"response.completed","response":{"id":"resp_stream_fn","object":"response","status":"completed","output":[{"id":"fc_s","type":"function_call","call_id":"call_s","name":"weather","arguments":"{}","status":"completed"}],"usage":{"input_tokens":5,"output_tokens":2}}}' + ;; + stream-failed) + printf '%s\n\n' 'event: response.failed' 'data: {"type":"response.failed","response":{"id":"resp_stream_bad","object":"response","status":"failed","error":{"code":"server_error","message":"stream generation failed"},"output":[]}}' + ;; + stream-http-error) + printf '%s\n' '{"error":{"message":"previous response missing","param":"previous_response_id","code":"previous_response_not_found"}}' + ;; + stream-flat-error) + printf '%s\n\n' 'event: error' 'data: {"type":"error","message":"previous response missing","param":"previous_response_id","code":"previous_response_not_found"}' + ;; + stream-no-terminal) + printf '%s\n\n' 'event: response.output_text.delta' 'data: {"type":"response.output_text.delta","delta":"```bash\nprintf pwned\n```"}' + ;; + *) + echo "curl stub: unknown CURL_MODE=$CURL_MODE" >&2 + exit 2 + ;; +esac +STUB +chmod +x "$WORK/bin/curl" + +export PATH="$WORK/bin:$PATH" +export HEADLONG_HOME="$WORK/home" +export OPENAI_API_KEY="test-openai-key" +export OPENROUTER_API_KEY="test-openrouter-key" +export LLM_RETRIES=0 +export CURL_ARGS="$WORK/curl.args" +export CURL_PAYLOAD="$WORK/curl.payload" +export CURL_CALLS="$WORK/curl.calls" +LLM="$REPO/bin/llm" + +reset() { + : > "$CURL_ARGS" + : > "$CURL_PAYLOAD" + rm -f "$CURL_CALLS" "$WORK/response.json" "$WORK/usage.json" "$WORK/stdout" "$WORK/stderr" +} + +run_openai() { + LLM_API_FORMAT=responses \ + LLM_RESPONSE_FILE="$WORK/response.json" \ + LLM_USAGE_FILE="$WORK/usage.json" \ + "$LLM" --provider openai -m gpt-5.4-mini "$@" +} + +# Buffered output, endpoint, sidecar, reasoning, and usage. +reset +export CURL_MODE=buffered-completed +run_openai --no-stream --thinking medium "say hello" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +[[ "$rc" -eq 0 && "$(cat "$WORK/stdout")" == "hello" ]] \ + && ok "buffered Responses emits visible text" \ + || bad "buffered Responses emits visible text" "rc=$rc out=$(cat "$WORK/stdout")" +grep -q 'brief reasoning' "$WORK/stderr" \ + && ok "buffered Responses emits reasoning summary on stderr" \ + || bad "buffered Responses emits reasoning summary on stderr" +grep -q 'https://api.openai.com/v1/responses' "$CURL_ARGS" \ + && ok "native OpenAI selects /v1/responses" \ + || bad "native OpenAI selects /v1/responses" +jq -e '.id == "resp_buffered" and .output[1].phase == "final_answer"' "$WORK/response.json" >/dev/null \ + && ok "terminal Response sidecar preserves the full object" \ + || bad "terminal Response sidecar preserves the full object" +mode=$(stat -c %a "$WORK/response.json" 2>/dev/null || stat -f %Lp "$WORK/response.json") +[[ "$mode" == 600 ]] && ok "Response sidecar is mode 600" || bad "Response sidecar is mode 600" "mode=$mode" +jq -e '.in_tok == 21 and .out_tok == 8 and .think_tok == 3' "$WORK/usage.json" >/dev/null \ + && ok "Responses usage maps to the existing usage contract" \ + || bad "Responses usage maps to the existing usage contract" "$(cat "$WORK/usage.json")" +jq -e '(.include | index("reasoning.encrypted_content")) != null' "$CURL_PAYLOAD" >/dev/null \ + && ok "Responses requests include encrypted reasoning for stateless replay" \ + || bad "Responses requests include encrypted reasoning for stateless replay" "$(jq -c . "$CURL_PAYLOAD" 2>/dev/null)" + +# Buffered content parts concatenate byte-for-byte like SSE deltas; jq's +# default record newlines must not alter the model's text. +reset +export CURL_MODE=buffered-multipart +run_openai --no-stream "multiple parts" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -eq 0 && "$(cat "$WORK/stdout")" == $'alpha\nbetadenied' ]]; then + ok "buffered Responses concatenates content parts without invented newlines" +else + bad "buffered Responses concatenates content parts without invented newlines" "rc=$rc out=$(printf %q "$(cat "$WORK/stdout")")" +fi + +# Structured input is not flattened, extra fields pass through, and owned +# fields win over conflicting values from the extra-body file. +reset +cat > "$WORK/body.json" <<'JSON' +{ + "model": "wrong-model", + "input": "wrong-input", + "stream": true, + "max_output_tokens": 1, + "instructions": "wrong instructions", + "previous_response_id": "wrong-id", + "store": false, + "include": ["reasoning.encrypted_content"], + "tools": [{"type":"function","name":"weather","description":"Weather","parameters":{"type":"object"},"strict":true}], + "text": {"format":{"type":"json_schema","name":"answer","schema":{"type":"object"},"strict":true}} +} +JSON +cat > "$WORK/input.json" <<'JSON' +[ + {"role":"user","content":[{"type":"input_text","text":"describe"},{"type":"input_image","image_url":"data:image/png;base64,AAAA","detail":"low"}]}, + {"type":"reasoning","id":"rs_old","summary":[],"encrypted_content":"encrypted"}, + {"type":"function_call_output","call_id":"call_old","output":"sunny"} +] +JSON +export CURL_MODE=buffered-completed +LLM_API_FORMAT=responses \ +LLM_RESPONSES_BODY_FILE="$WORK/body.json" \ +LLM_PREVIOUS_RESPONSE_ID="resp_previous" \ +LLM_RESPONSE_FILE="$WORK/response.json" \ +LLM_PROVIDER=openai-compatible \ +LLM_API_URL="https://api.router.test/v1/responses" \ +LLM_API_KEY="test-compatible-key" \ + "$LLM" -m glm-test --no-stream --thinking high --system-prompt "real instructions" \ + --messages-file "$WORK/input.json" >/dev/null 2>"$WORK/stderr" +if jq -e ' + .model == "glm-test" and + .input[0].content[1].type == "input_image" and + .input[1].encrypted_content == "encrypted" and + .input[2].type == "function_call_output" and + .stream == false and + .max_output_tokens == 16384 and + .instructions == "real instructions" and + .previous_response_id == "resp_previous" and + .store == false and + (.include | index("reasoning.encrypted_content")) != null and + .tools[0].name == "weather" and + .text.format.type == "json_schema" and + .reasoning.effort == "high" and + .reasoning.summary == "auto" +' "$CURL_PAYLOAD" >/dev/null; then + ok "Responses request preserves typed input and merges the create body" +else + bad "Responses request preserves typed input and merges the create body" "$(jq -c . "$CURL_PAYLOAD" 2>/dev/null)" +fi +grep -q 'https://api.router.test/v1/responses' "$CURL_ARGS" \ + && ok "openai-compatible uses the exact configured Responses URL" \ + || bad "openai-compatible uses the exact configured Responses URL" + +# Function-only responses are protocol success, not an empty-response error. +reset +export CURL_MODE=buffered-function +run_openai --no-stream "call the function" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -eq 0 && ! -s "$WORK/stdout" ]] \ + && jq -e '.output[0].type == "function_call" and .output[0].call_id == "call_1"' "$WORK/response.json" >/dev/null; then + ok "buffered function-only Response succeeds through the sidecar" +else + bad "buffered function-only Response succeeds through the sidecar" "rc=$rc out=$(cat "$WORK/stdout")" +fi + +# Incomplete responses keep their partial output and warn. +reset +export CURL_MODE=buffered-incomplete +run_openai --no-stream "be long" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -eq 0 && "$(cat "$WORK/stdout")" == "partial" ]] && grep -q 'output truncated' "$WORK/stderr"; then + ok "incomplete max-output Response returns partial text with warning" +else + bad "incomplete max-output Response returns partial text with warning" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +# A 200 Response with status=failed is not successful output. +reset +export CURL_MODE=buffered-failed +LLM_RETRIES=2 run_openai --no-stream "fail" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'generation failed' "$WORK/stderr" \ + && jq -e '.status == "failed"' "$WORK/response.json" >/dev/null \ + && [[ "$(cat "$CURL_CALLS")" -eq 1 ]]; then + ok "buffered failed Response fails once and preserves terminal state" +else + bad "buffered failed Response fails once and preserves terminal state" "rc=$rc calls=$(cat "$CURL_CALLS" 2>/dev/null) stderr=$(cat "$WORK/stderr")" +fi + +# Non-2xx error envelopes are available to machine callers for safe fallback. +reset +export CURL_MODE=http-400-previous +LLM_PREVIOUS_RESPONSE_ID=resp_stale run_openai --no-stream "continue" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] \ + && jq -e '.error.param == "previous_response_id" and .error.code == "unsupported_parameter"' "$WORK/response.json" >/dev/null; then + ok "HTTP error envelope is written to the Response sidecar" +else + bad "HTTP error envelope is written to the Response sidecar" "rc=$rc response=$(cat "$WORK/response.json" 2>/dev/null)" +fi + +# SSE text/reasoning/terminal state and usage. +reset +export CURL_MODE=stream-completed +run_openai --thinking medium "stream" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -eq 0 && "$(cat "$WORK/stdout")" == $'line one\nline two' ]]; then + ok "Responses SSE preserves text delta newlines" +else + bad "Responses SSE preserves text delta newlines" "rc=$rc out=$(printf %q "$(cat "$WORK/stdout")")" +fi +grep -q 'stream reasoning' "$WORK/stderr" \ + && ok "Responses SSE emits reasoning summary deltas on stderr" \ + || bad "Responses SSE emits reasoning summary deltas on stderr" +if jq -e '.id == "resp_stream" and .status == "completed"' "$WORK/response.json" >/dev/null \ + && jq -e '.in_tok == 30 and .out_tok == 12 and .think_tok == 4' "$WORK/usage.json" >/dev/null; then + ok "Responses SSE records terminal Response and usage" +else + bad "Responses SSE records terminal Response and usage" +fi + +# Function-only streams count the terminal item as protocol output. +reset +export CURL_MODE=stream-function +run_openai "stream a function" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -eq 0 && ! -s "$WORK/stdout" ]] \ + && jq -e '.output[0].type == "function_call"' "$WORK/response.json" >/dev/null; then + ok "function-only Responses SSE succeeds without visible stdout" +else + bad "function-only Responses SSE succeeds without visible stdout" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +# Stream terminal failures and pre-SSE error bodies remain failures and leave +# structured state for the caller. +reset +export CURL_MODE=stream-failed +run_openai "stream failure" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'stream generation failed' "$WORK/stderr" \ + && jq -e '.status == "failed"' "$WORK/response.json" >/dev/null; then + ok "terminal response.failed fails with structured state" +else + bad "terminal response.failed fails with structured state" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +reset +export CURL_MODE=stream-http-error +LLM_RETRIES=2 LLM_PREVIOUS_RESPONSE_ID=resp_missing \ + run_openai "continue" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] \ + && jq -e '.error.param == "previous_response_id"' "$WORK/response.json" >/dev/null \ + && [[ "$(cat "$CURL_CALLS")" -eq 1 ]]; then + ok "pre-SSE continuation rejection is preserved without internal retries" +else + bad "pre-SSE continuation rejection is preserved without internal retries" "rc=$rc calls=$(cat "$CURL_CALLS" 2>/dev/null) response=$(cat "$WORK/response.json" 2>/dev/null)" +fi + +reset +export CURL_MODE=stream-flat-error +LLM_RETRIES=2 LLM_PREVIOUS_RESPONSE_ID=resp_missing \ + run_openai "continue" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] \ + && jq -e '.param == "previous_response_id" and .code == "previous_response_not_found"' "$WORK/response.json" >/dev/null \ + && [[ "$(cat "$CURL_CALLS")" -eq 1 ]]; then + ok "flat SSE continuation rejection is preserved without internal retries" +else + bad "flat SSE continuation rejection is preserved without internal retries" "rc=$rc calls=$(cat "$CURL_CALLS" 2>/dev/null) response=$(cat "$WORK/response.json" 2>/dev/null)" +fi + +reset +export CURL_MODE=stream-no-terminal +LLM_RETRIES=2 run_openai "truncated stream" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 && "$(cat "$CURL_CALLS")" -eq 1 && ! -e "$WORK/response.json" ]] \ + && grep -q 'without a terminal response' "$WORK/stderr"; then + ok "Responses SSE with output but no terminal event fails without retry" +else + bad "Responses SSE with output but no terminal event fails without retry" "rc=$rc calls=$(cat "$CURL_CALLS" 2>/dev/null) stderr=$(cat "$WORK/stderr")" +fi + +# Provider and body validation fail before curl. +reset +export CURL_MODE=buffered-completed +LLM_API_FORMAT=responses "$LLM" --provider anthropic -m claude-sonnet-4-5 "no" \ + >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'Responses format is not supported for provider anthropic' "$WORK/stderr"; then + ok "unsupported provider fails explicitly" +else + bad "unsupported provider fails explicitly" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +printf '[]' > "$WORK/body.json" +LLM_API_FORMAT=responses LLM_RESPONSES_BODY_FILE="$WORK/body.json" \ + "$LLM" --provider openai -m gpt-5.4-mini "no" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'must contain a JSON object' "$WORK/stderr"; then + ok "Responses extra body must be an object" +else + bad "Responses extra body must be an object" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +printf '{"background":true}' > "$WORK/body.json" +LLM_API_FORMAT=responses LLM_RESPONSES_BODY_FILE="$WORK/body.json" \ + "$LLM" --provider openai -m gpt-5.4-mini "no" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'background responses require lifecycle operations' "$WORK/stderr"; then + ok "background Responses are rejected until lifecycle support exists" +else + bad "background Responses are rejected until lifecycle support exists" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +printf '{"conversation":"conv_123"}' > "$WORK/body.json" +LLM_API_FORMAT=responses LLM_RESPONSES_BODY_FILE="$WORK/body.json" \ + "$LLM" --provider openai -m gpt-5.4-mini "no" >"$WORK/stdout" 2>"$WORK/stderr" +rc=$? +if [[ "$rc" -ne 0 ]] && grep -q 'conversation state cannot be combined' "$WORK/stderr"; then + ok "conversation state is rejected before continuation can conflict" +else + bad "conversation state is rejected before continuation can conflict" "rc=$rc stderr=$(cat "$WORK/stderr")" +fi + +# OpenRouter has its own Responses endpoint but remains a separate stateless +# service from generic openai-compatible endpoints. +reset +export CURL_MODE=buffered-completed +LLM_API_FORMAT=responses LLM_RESPONSE_FILE="$WORK/response.json" \ + "$LLM" --provider openrouter -m openai/gpt-5 --no-stream "hello" \ + >"$WORK/stdout" 2>"$WORK/stderr" +grep -q 'https://openrouter.ai/api/v1/responses' "$CURL_ARGS" \ + && ok "OpenRouter selects its documented Responses endpoint" \ + || bad "OpenRouter selects its documented Responses endpoint" +jq -e '(.include | index("reasoning.encrypted_content")) != null' "$CURL_PAYLOAD" >/dev/null \ + && ok "OpenRouter requests encrypted reasoning for exact replay" \ + || bad "OpenRouter requests encrypted reasoning for exact replay" "$(jq -c . "$CURL_PAYLOAD" 2>/dev/null)" + +printf '\n%d passed, %d failed\n' "$pass" "$fail" +[[ "$fail" -eq 0 ]] diff --git a/tests/test_shellm_responses_continuation.sh b/tests/test_shellm_responses_continuation.sh new file mode 100644 index 0000000..a6e1767 --- /dev/null +++ b/tests/test_shellm_responses_continuation.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash +# test_shellm_responses_continuation.sh — persisted Responses continuation and +# one-time full-context fallback in shellm. + +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(dirname "$HERE")" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +pass=0 +fail=0 +ok() { pass=$((pass+1)); printf 'ok %s\n' "$1"; } +bad() { fail=$((fail+1)); printf 'FAIL %s%s\n' "$1" "${2:+ — $2}"; } + +mkdir -p "$WORK/home" "$WORK/wd" +cp -R "$REPO/bin" "$WORK/toolbin" +cat > "$WORK/toolbin/llm" <<'STUB' +#!/usr/bin/env bash +main_loop=0 +messages_file="" +prev="" +for arg in "$@"; do + [[ "$arg" == --thinking ]] && main_loop=1 + [[ "$prev" == --messages-file ]] && messages_file="$arg" + prev="$arg" +done +if [[ "$main_loop" -ne 1 ]]; then + printf '{}\n' + exit 0 +fi + +n=0 +[[ -f "$LLM_STUB_DIR/calls" ]] && read -r n < "$LLM_STUB_DIR/calls" +n=$((n + 1)) +printf '%s\n' "$n" > "$LLM_STUB_DIR/calls" +printf '%s\n' "${LLM_API_FORMAT:-}" > "$LLM_STUB_DIR/format-$n" +printf '%s\n' "${LLM_PREVIOUS_RESPONSE_ID:-}" > "$LLM_STUB_DIR/previous-$n" +printf '%s\n' "${LLM_RESPONSE_FILE:-}" > "$LLM_STUB_DIR/response-file-$n" +[[ -n "$messages_file" ]] && cp "$messages_file" "$LLM_STUB_DIR/messages-$n.json" +if [[ -n "${LLM_RESPONSE_FILE:-}" ]]; then + state_dir=$(dirname "$LLM_RESPONSE_FILE") + if [[ -f "$state_dir/.response-id" ]]; then + cp "$state_dir/.response-id" "$LLM_STUB_DIR/id-value-$n" + (stat -c %a "$state_dir/.response-id" 2>/dev/null \ + || stat -f %Lp "$state_dir/.response-id" 2>/dev/null) \ + > "$LLM_STUB_DIR/id-mode-$n" + fi + [[ -e "$state_dir/.continuation-disabled" ]] \ + && printf 'yes\n' > "$LLM_STUB_DIR/disabled-$n" \ + || printf 'no\n' > "$LLM_STUB_DIR/disabled-$n" +fi + +write_response() { + local id="$1" + [[ -n "${LLM_RESPONSE_FILE:-}" ]] || return 0 + ( umask 077; jq -nc --arg id "$id" '{ + id: $id, + object: "response", + status: "completed", + output: [ + {id:("rs_" + $id), type:"reasoning", summary:[], encrypted_content:("enc_" + $id)}, + {id:("msg_" + $id), type:"message", role:"assistant", status:"completed", phase:"final_answer", content:[{type:"output_text", text:("text_" + $id)}]} + ] + }' > "$LLM_RESPONSE_FILE" ) +} + +case "$LLM_STUB_MODE:$n" in + continue:1|fallback:1|stateless:1) + write_response resp_1 + printf '%s\n' '```bash' 'printf "first output\n"' '```' + ;; + continue:2|stateless:2) + write_response resp_2 + printf '%s\n' '```bash' 'FINAL=done' '```' + ;; + fallback:2) + ( umask 077; printf '%s' '{"error":{"message":"previous response is unavailable","param":"previous_response_id","code":"previous_response_not_found"}}' > "$LLM_RESPONSE_FILE" ) + printf '%s\n' 'llm: error: API error (HTTP 400): previous response is unavailable' >&2 + exit 1 + ;; + fallback:3) + write_response resp_3 + printf '%s\n' '```bash' 'FINAL=done-after-fallback' '```' + ;; + empty:1) + ( umask 077; jq -nc '{ + id: "resp_incomplete", + object: "response", + status: "incomplete", + incomplete_details: {reason: "max_output_tokens"}, + output: [{id:"rs_incomplete", type:"reasoning", summary:[], encrypted_content:"enc_incomplete"}] + }' > "$LLM_RESPONSE_FILE" ) + printf '%s\n' 'llm: warning: output truncated (max_output_tokens)' >&2 + ;; + empty:2) + write_response resp_after_incomplete + printf '%s\n' '```bash' 'FINAL=done-after-incomplete' '```' + ;; + function:1) + ( umask 077; jq -nc '{ + id: "resp_function", + object: "response", + status: "completed", + output: [{id:"fc_1", type:"function_call", call_id:"call_1", name:"weather", arguments:"{}", status:"completed"}] + }' > "$LLM_RESPONSE_FILE" ) + ;; + no-terminal:1) + printf '%s\n' '```bash' "touch '$LLM_STUB_DIR/executed'" '```' + ;; + chat:1) + [[ -z "${LLM_RESPONSE_FILE:-}" ]] || { echo "chat unexpectedly received LLM_RESPONSE_FILE" >&2; exit 2; } + printf '%s\n' '```bash' 'FINAL=chat-done' '```' + ;; + *) + echo "unexpected stub call $LLM_STUB_MODE:$n" >&2 + exit 2 + ;; +esac +STUB +chmod +x "$WORK/toolbin/llm" + +export PATH="$WORK/toolbin:$PATH" +export HOME="$WORK/home" +export HEADLONG_HOME="$WORK/home/.headlong" +export OPENAI_API_KEY="test-key" +export OPENROUTER_API_KEY="test-router-key" +export SHELLM_MODEL="gpt-5-test" +export SHELLM_ENV=local +export SHELLM_NO_BANNER=1 + +run_shellm() { + local mode="$1" format="$2" provider="${3:-}" model="${4:-gpt-5-test}" + rm -rf "$WORK/stub" "$HEADLONG_HOME" "$WORK/wd"/* + mkdir -p "$WORK/stub" "$WORK/wd" + LLM_STUB_DIR="$WORK/stub" LLM_STUB_MODE="$mode" SHELLM_API_FORMAT="$format" \ + LLM_PROVIDER="$provider" SHELLM_MODEL="$model" \ + "$WORK/toolbin/shellm" --workdir "$WORK/wd" --max-iterations 3 "do the task" \ + > "$WORK/out" 2> "$WORK/err" < /dev/null +} + +main_calls() { cat "$WORK/stub/calls" 2>/dev/null || echo 0; } + +# A successful terminal response becomes the next request's continuation ID; +# only the messages after the last assistant turn are sent as new input. +run_shellm continue responses +rc=$? +if [[ "$rc" -eq 0 && "$(main_calls)" -eq 2 && "$(cat "$WORK/stub/previous-2")" == resp_1 ]]; then + ok "later shellm iterations use the persisted previous_response_id" +else + bad "later shellm iterations use the persisted previous_response_id" "rc=$rc calls=$(main_calls) previous=$(cat "$WORK/stub/previous-2" 2>/dev/null)" +fi + +if [[ "$(cat "$WORK/stub/format-1")" == responses && -n "$(cat "$WORK/stub/response-file-1")" ]]; then + ok "shellm opts llm into Responses and requests a terminal sidecar" +else + bad "shellm opts llm into Responses and requests a terminal sidecar" +fi + +if jq -e 'length == 1 and .[0].role == "user" and (. [0].content | contains("first output"))' \ + "$WORK/stub/messages-2.json" >/dev/null 2>&1; then + ok "continuation sends only input after the last assistant turn" +else + bad "continuation sends only input after the last assistant turn" "$(jq -c . "$WORK/stub/messages-2.json" 2>/dev/null)" +fi + +if [[ "$(cat "$WORK/stub/id-value-2" 2>/dev/null)" == resp_1 \ + && "$(cat "$WORK/stub/id-mode-2" 2>/dev/null)" == 600 ]]; then + ok "successful Response ID persists in mode-600 process state" +else + bad "successful Response ID persists in mode-600 process state" "value=$(cat "$WORK/stub/id-value-2" 2>/dev/null) mode=$(cat "$WORK/stub/id-mode-2" 2>/dev/null)" +fi + +# A provider rejection tied to previous_response_id clears continuation and +# retries once with the exact replay chain. It then stays disabled so the next +# iteration cannot enter a fallback loop. +run_shellm fallback responses +rc=$? +if [[ "$rc" -eq 0 && "$(main_calls)" -eq 3 \ + && "$(cat "$WORK/stub/previous-2")" == resp_1 \ + && -z "$(cat "$WORK/stub/previous-3")" ]]; then + ok "rejected continuation retries once without previous_response_id" +else + bad "rejected continuation retries once without previous_response_id" "rc=$rc calls=$(main_calls) prev2=$(cat "$WORK/stub/previous-2" 2>/dev/null) prev3=$(cat "$WORK/stub/previous-3" 2>/dev/null)" +fi + +if jq -e ' + length >= 4 and + any(.role == "user" and (.content | contains("do the task"))) and + any(.type == "reasoning" and .encrypted_content == "enc_resp_1") and + any(.type == "message" and .role == "assistant" and .phase == "final_answer") and + any(.role == "user" and (.content | contains("first output"))) +' "$WORK/stub/messages-3.json" >/dev/null 2>&1; then + ok "continuation fallback replays exact typed Responses items" +else + bad "continuation fallback replays exact typed Responses items" "$(jq -c . "$WORK/stub/messages-3.json" 2>/dev/null)" +fi + +if [[ ! -e "$WORK/stub/id-value-3" \ + && "$(cat "$WORK/stub/disabled-3" 2>/dev/null)" == yes ]] \ + && grep -q 'retrying once with exact replay context' "$WORK/err"; then + ok "fallback clears persisted continuation state and disables reuse" +else + bad "fallback clears persisted continuation state and disables reuse" "id=$(cat "$WORK/stub/id-value-3" 2>/dev/null) disabled=$(cat "$WORK/stub/disabled-3" 2>/dev/null) stderr=$(tail -3 "$WORK/err" | tr '\n' ' ')" +fi + +# OpenRouter documents its Responses endpoint as stateless. It must replay +# exact output items from the first turn without first paying for a rejected +# previous_response_id request. +run_shellm stateless responses openrouter openai/o4-mini +rc=$? +if [[ "$rc" -eq 0 && "$(main_calls)" -eq 2 \ + && -z "$(cat "$WORK/stub/previous-1")" \ + && -z "$(cat "$WORK/stub/previous-2")" ]]; then + ok "OpenRouter Responses starts in stateless replay mode" +else + bad "OpenRouter Responses starts in stateless replay mode" "rc=$rc calls=$(main_calls) prev1=$(cat "$WORK/stub/previous-1" 2>/dev/null) prev2=$(cat "$WORK/stub/previous-2" 2>/dev/null)" +fi + +if jq -e ' + length >= 4 and + any(.type == "reasoning" and .encrypted_content == "enc_resp_1") and + any(.type == "message" and .phase == "final_answer") and + any(.role == "user" and (.content | contains("first output"))) +' "$WORK/stub/messages-2.json" >/dev/null 2>&1; then + ok "OpenRouter replay preserves reasoning and assistant phase items" +else + bad "OpenRouter replay preserves reasoning and assistant phase items" "$(jq -c . "$WORK/stub/messages-2.json" 2>/dev/null)" +fi + +# A reasoning-only incomplete Response continues from the terminal Response +# state. Its reasoning summary is never fabricated as an assistant message. +run_shellm empty responses +rc=$? +if [[ "$rc" -eq 0 && "$(main_calls)" -eq 2 \ + && "$(cat "$WORK/stub/previous-2")" == resp_incomplete ]]; then + ok "reasoning-only incomplete Response continues by response ID" +else + bad "reasoning-only incomplete Response continues by response ID" "rc=$rc calls=$(main_calls) previous=$(cat "$WORK/stub/previous-2" 2>/dev/null)" +fi + +if jq -e ' + length == 1 and + .[0].role == "user" and + (. [0].content | contains("Continue from the incomplete response")) and + all(.[]; .role != "assistant") +' "$WORK/stub/messages-2.json" >/dev/null 2>&1; then + ok "Responses retry does not fabricate assistant reasoning context" +else + bad "Responses retry does not fabricate assistant reasoning context" "$(jq -c . "$WORK/stub/messages-2.json" 2>/dev/null)" +fi + +# shellm cannot execute Responses-native function calls. It fails closed +# instead of treating empty stdout as another model turn. +run_shellm function responses +rc=$? +if [[ "$rc" -ne 0 && "$(main_calls)" -eq 1 ]] \ + && grep -q 'function calls without visible shellm output' "$WORK/err"; then + ok "function-only Response fails closed without an empty-output retry" +else + bad "function-only Response fails closed without an empty-output retry" "rc=$rc calls=$(main_calls) stderr=$(tail -5 "$WORK/err" | tr '\n' ' ')" +fi + +# Defense in depth: even a malformed/custom llm that exits successfully after +# visible output cannot make shellm execute without terminal Responses state. +run_shellm no-terminal responses +rc=$? +if [[ "$rc" -ne 0 && "$(main_calls)" -eq 1 && ! -e "$WORK/stub/executed" ]] \ + && grep -q 'without a terminal response' "$WORK/err"; then + ok "shellm rejects Responses output without terminal state before execution" +else + bad "shellm rejects Responses output without terminal state before execution" "rc=$rc calls=$(main_calls) stderr=$(tail -5 "$WORK/err" | tr '\n' ' ')" +fi + +# A bounded run context must never resend the pinned original prompt against an +# existing continuation when its assistant boundary has fallen out of view. +SHELLM_CONTEXT_SCOPE=run SHELLM_CONTEXT_RUN_TAIL=1 SHELLM_CONTEXT_RUN_TAIL_BLOCK=1 \ + run_shellm continue responses +rc=$? +if [[ "$rc" -ne 0 && "$(main_calls)" -eq 1 ]] \ + && grep -q 'continuation boundary fell outside' "$WORK/err"; then + ok "bounded context fails closed when its continuation boundary is absent" +else + bad "bounded context fails closed when its continuation boundary is absent" "rc=$rc calls=$(main_calls) stderr=$(tail -5 "$WORK/err" | tr '\n' ' ')" +fi + +# Invalid protocol configuration fails through shellm's normal error contract, +# rather than calling the error helper before it has been defined. +SHELLM_API_FORMAT=invalid "$WORK/toolbin/shellm" --help \ + > "$WORK/out" 2> "$WORK/err" +rc=$? +if [[ "$rc" -ne 0 ]] \ + && grep -q 'Invalid SHELLM_API_FORMAT: invalid' "$WORK/err" \ + && ! grep -q 'command not found' "$WORK/err"; then + ok "invalid Responses format fails through shellm's error contract" +else + bad "invalid Responses format fails through shellm's error contract" "rc=$rc stderr=$(cat "$WORK/err")" +fi + +# Default chat mode does not create or pass Responses state. +run_shellm chat chat +rc=$? +if [[ "$rc" -eq 0 && "$(cat "$WORK/stub/format-1")" == chat \ + && -z "$(cat "$WORK/stub/previous-1")" \ + && -z "$(rg --files "$HEADLONG_HOME/trajectories" 2>/dev/null | rg '/responses/' | head -1)" ]]; then + ok "default chat mode remains stateless" +else + bad "default chat mode remains stateless" "rc=$rc format=$(cat "$WORK/stub/format-1")" +fi + +printf '\n%d passed, %d failed\n' "$pass" "$fail" +[[ "$fail" -eq 0 ]]