From a198962dfddc149f3010ab42ba69130230c91fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Sat, 15 Aug 2026 15:11:03 +0200 Subject: [PATCH 1/6] feat(filter): resume IRR iterations within streaming responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the downstream response open while the iterative request router runs successive model and tool steps. Stream each step incrementally, then use completion transitions to resume the next step without committing a new response. Add bounded iteration state, logical stream limits, deadline handling, header-safe failover, and typed termination handling. Propagate runtime resources and listener limits into nested step pipelines, and preserve correct lifecycle ordering across normal completion, errors, and client disconnects. Cover the behavior with unit and end-to-end tests for streaming order, multi-step resumption, suppression, limits, failover, and cleanup. Fixes https://github.com/praxis-proxy/praxis/issues/984 Signed-off-by: Sébastien Han --- .../iterative_request_router.md | 5 +- filter/src/actions.rs | 11 +- .../iterative_request_router/config.rs | 21 +- .../iterative_request_router/mod.rs | 1025 ++++------- .../iterative_request_router/runner.rs | 592 +++++++ .../iterative_request_router/streaming.rs | 693 +++++++- .../iterative_request_router/tests.rs | 247 ++- filter/src/context.rs | 206 ++- filter/src/filter.rs | 9 + filter/src/lib.rs | 3 +- filter/src/pipeline/build.rs | 13 +- filter/src/pipeline/mod.rs | 38 + .../pingora/handler/request_filter/mod.rs | 32 +- .../tests/suite/iterative_request_router.rs | 1517 ++++++++++++++++- .../suite/streaming_terminal_response.rs | 32 +- 15 files changed, 3718 insertions(+), 726 deletions(-) create mode 100644 filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs diff --git a/docs/filters/http/traffic_management/iterative_request_router.md b/docs/filters/http/traffic_management/iterative_request_router.md index ba8ecad2a..7451ee5e8 100644 --- a/docs/filters/http/traffic_management/iterative_request_router.md +++ b/docs/filters/http/traffic_management/iterative_request_router.md @@ -9,6 +9,8 @@ Framework-level filter for iterative sub-request execution. Holds named steps, each backed by a pre-built sub-pipeline. During request processing, runs an iteration loop: execute each step's request filters, make the HTTP call via Pingora's `Connector`, execute its response filters, evaluate transition rules, and continue or return the final response. +Streaming steps remain pull-based. Header-safe failover rules run before any bytes are exposed; all other `on_result` rules run after clean EOF and may resume another step inside the same committed downstream response. + ## Configuration | Field | Type | Required | Description | @@ -16,6 +18,7 @@ Holds named steps, each backed by a pre-built sub-pipeline. During request proce | `initial_step` | string | yes | Name of the first step to execute. | | `max_iterations` | integer | no | Maximum iterations before aborting (default 10, max 100). | | `max_response_bytes` | integer | no | Maximum response body bytes per sub-request. | +| `max_stream_response_bytes` | integer | no | Optional cumulative byte ceiling for one logical streamed response. This is intentionally distinct from buffered per-step response limits. | | `max_state_bytes` | integer | no | Maximum accumulated iteration state bytes. | | `step_timeout_ms` | integer | no | Per-step timeout in milliseconds. Defaults to `timeout_ms`. | | `steps` | StepConfig[] | yes | Named steps, each with filters and transition rules. | @@ -37,7 +40,7 @@ Holds named steps, each backed by a pre-built sub-pipeline. During request proce | `steps[].filters[].name` | string | no | Optional user-assigned name for this filter entry. Used as a rejoin target by branch chains. | | `steps[].filters[].response_conditions` | ResponseCondition[] | no | Ordered conditions that gate whether this filter runs on responses. Evaluated against the upstream response (status, headers). Empty means the filter always runs on responses. | | `steps[].filters[].failure_mode` | `closed` \| `open` | no | Per-filter failure behaviour (`open` or `closed`). | -| `steps[].on_result` | StepTransition[] | no | Transition rules evaluated after the sub-request response. First match wins. | +| `steps[].on_result` | StepTransition[] | no | Transition rules evaluated in order. Streaming header-safe failovers run before body exposure; remaining rules run after step completion. | | `steps[].on_result[].default` | bool | no | If true, this is the default (always-match) rule. | | `steps[].on_result[].filter` | string | no | Filter name whose results to check. | | `steps[].on_result[].key` | string | no | Result key to match. | diff --git a/filter/src/actions.rs b/filter/src/actions.rs index 9ac3fb4a7..93296cfdb 100644 --- a/filter/src/actions.rs +++ b/filter/src/actions.rs @@ -8,7 +8,7 @@ use std::fmt; use async_trait::async_trait; use bytes::Bytes; -use crate::FilterError; +use crate::{FilterError, RequestExtensions}; // ----------------------------------------------------------------------------- // Streaming terminal response @@ -37,6 +37,15 @@ pub trait StreamingResponseBody: Send + 'static { /// /// This operation must be idempotent. async fn cancel(&mut self); + + /// Exchange request extensions with the protocol lifecycle owner. + /// + /// Most streaming bodies do not own filter extensions and use this + /// default no-op. Iterative sessions override it so the same extension + /// set can move between step filters and parent response filters without + /// cloning type-erased values. + #[doc(hidden)] + fn swap_extensions(&mut self, _extensions: &mut RequestExtensions) {} } /// A terminal response whose body is delivered incrementally. diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs index b963b73bb..bce3e7bc9 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs @@ -75,6 +75,11 @@ pub(crate) struct IterativeRequestRouterConfig { #[serde(default = "default_max_response_bytes")] pub(crate) max_response_bytes: usize, + /// Optional cumulative byte ceiling for one logical streamed response. + /// This is intentionally distinct from buffered per-step response limits. + #[serde(default)] + pub(crate) max_stream_response_bytes: Option, + /// Maximum accumulated iteration state bytes. #[serde(default = "default_max_state_bytes")] pub(crate) max_state_bytes: usize, @@ -92,7 +97,7 @@ pub(crate) struct IterativeRequestRouterConfig { } /// A named step within the iterative router. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct StepConfig { /// Step name (must be unique within the router). @@ -101,14 +106,14 @@ pub(crate) struct StepConfig { /// Filters to execute for this step's sub-request. pub(crate) filters: Vec, - /// Transition rules evaluated after the sub-request - /// response. First match wins. + /// Transition rules evaluated in order. Streaming header-safe failovers + /// run before body exposure; remaining rules run after step completion. #[serde(default)] pub(crate) on_result: Vec, } /// A transition rule evaluated after a step completes. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct StepTransition { /// If true, this is the default (always-match) rule. @@ -224,6 +229,14 @@ pub(crate) fn validate(cfg: &IterativeRequestRouterConfig) -> Result<(), FilterE .into()); } + if cfg.max_stream_response_bytes == Some(0) { + return Err( + "iterative_request_router: max_stream_response_bytes must be > 0 when configured" + .to_owned() + .into(), + ); + } + if cfg.steps.is_empty() { return Err("iterative_request_router: at least one step required".to_owned().into()); } diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs index be66f35df..b9d5a8e8a 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs @@ -18,15 +18,15 @@ //! //! When a step's filters select [`SubRequestResponseMode::Streaming`], //! IRR dispatches via `send_streaming()` instead of the buffered -//! `execute()` path. Response-header filters run before transition -//! evaluation; only status, origin, transport error, and default -//! predicates are available. Once a streaming response is committed -//! downstream, no failover or replacement response can occur. +//! `execute()` path. Header-safe failover transitions run before that +//! step exposes bytes. After clean EOF and response-body completion, +//! ordinary `on_result` rules may resume another step inside the same +//! committed downstream response. //! -//! Transitions that depend on filter results (filter/key/value -//! predicates) or `BodyMode::StreamBuffer` response mode are -//! rejected at config validation time for streaming-capable pipelines, -//! with a runtime guard for dynamically registered filters. +//! Header-safe failovers must precede completion-dependent transitions. +//! `BodyMode::StreamBuffer` remains incompatible with streaming-capable +//! step pipelines. Once the logical response is committed, failures can +//! only terminate its stream; they cannot replace its HTTP response. //! //! # Position requirement //! @@ -38,6 +38,7 @@ //! See proposal 00786 for the full design rationale. mod config; +mod runner; mod streaming; #[cfg(test)] #[expect(clippy::allow_attributes, reason = "blanket test suppressions")] @@ -45,11 +46,8 @@ mod streaming; mod tests; use std::{ - collections::HashMap, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, + collections::{HashMap, VecDeque}, + sync::Arc, time::{Duration, Instant}, }; @@ -57,21 +55,20 @@ use async_trait::async_trait; use bytes::Bytes; use http::HeaderMap; use pingora_core::upstreams::peer::HttpPeer; -use praxis_core::subrequest::{FrameworkHeaders, StreamLimits}; use tracing::{debug, info, warn}; use self::{ config::IterativeRequestRouterConfig, - streaming::{IrrStreamingBody, StepResponseContinuation}, + runner::{IrrStepRunner, OpenedStepKind, StepRuntime}, + streaming::{IrrStreamingBody, IrrStreamingSession, ensure_combined_retained_limit}, }; use crate::{ - FilterEntry, FilterError, FilterPipeline, FilterRegistry, IterationState, NextIterationBody, SubRequest, + FilterEntry, FilterError, FilterPipeline, FilterRegistry, IterationState, StreamTermination, SubRequest, SubRequestResponseMode, SubResponse, - actions::{FilterAction, Rejection, StreamingTerminalResponse, TerminalResponse}, + actions::{FilterAction, Rejection, StreamingResponseBody as _, StreamingTerminalResponse, TerminalResponse}, factory::parse_filter_config, filter::{HttpFilter, HttpFilterContext}, pipeline::subrequest::DEPTH_HEADER, - results::RetainedFilterResults, }; // --------------------------------------------------------------------------- @@ -87,12 +84,35 @@ const STREAMING_IDLE_TIMEOUT: Duration = Duration::from_secs(30); /// Whether transitions depend on response body content (filter result /// predicates or body-dependent metadata). +#[cfg(test)] pub(super) fn has_body_dependent_transitions(transitions: &[config::StepTransition]) -> bool { transitions .iter() .any(|t| t.filter.is_some() || t.key.is_some() || t.value.is_some()) } +/// Whether a transition can safely fail over before exposing a streamed body. +fn is_header_safe_failover(transition: &config::StepTransition) -> bool { + transition.next.is_some() + && !transition.default + && transition.filter.is_none() + && transition.key.is_none() + && transition.value.is_none() +} + +/// Whether the header-safe failover prefix is followed only by completion rules. +pub(super) fn streaming_transition_order_is_valid(transitions: &[config::StepTransition]) -> bool { + let mut completion_seen = false; + transitions.iter().all(|transition| { + if is_header_safe_failover(transition) { + !completion_seen + } else { + completion_seen = true; + true + } + }) +} + // --------------------------------------------------------------------------- // IterativeRequestRouterFilter // --------------------------------------------------------------------------- @@ -105,6 +125,10 @@ pub(super) fn has_body_dependent_transitions(transitions: &[config::StepTransiti /// `Connector`, execute its response filters, evaluate transition /// rules, and continue or return the final response. /// +/// Streaming steps remain pull-based. Header-safe failover rules run before +/// any bytes are exposed; all other `on_result` rules run after clean EOF and +/// may resume another step inside the same committed downstream response. +/// /// # YAML configuration /// /// ```yaml @@ -134,6 +158,9 @@ pub struct IterativeRequestRouterFilter { /// Maximum response body bytes per sub-request. max_response_bytes: usize, + /// Optional cumulative byte ceiling for a logical streamed response. + max_stream_response_bytes: Option, + /// Maximum accumulated state bytes. max_state_bytes: usize, @@ -216,13 +243,21 @@ impl IterativeRequestRouterFilter { continue; } if let Some(transitions) = step_transitions.get(name) { - for (i, t) in transitions.iter().enumerate() { - if t.filter.is_some() || t.key.is_some() || t.value.is_some() { + for (i, transition) in transitions.iter().enumerate() { + if is_header_safe_failover(transition) && !streaming_transition_order_is_valid(transitions) { return Err(format!( "iterative_request_router: step '{name}' transition {i}: \ - filter/key/value predicates are incompatible with \ - streaming-capable pipelines (response body is not \ - available during transition evaluation)" + header-safe streaming failover rules must precede completion rules" + ) + .into()); + } + if transition.next.is_some() + && transition.filter.is_none() + && (transition.key.is_some() || transition.value.is_some()) + { + return Err(format!( + "iterative_request_router: step '{name}' transition {i}: \ + ambiguous streaming transition predicates" ) .into()); } @@ -248,6 +283,7 @@ impl IterativeRequestRouterFilter { initial_step: Arc::from(cfg.initial_step.as_str()), max_iterations: cfg.max_iterations, max_response_bytes: cfg.max_response_bytes, + max_stream_response_bytes: cfg.max_stream_response_bytes, max_state_bytes: cfg.max_state_bytes, step_pipelines, step_timeout, @@ -273,6 +309,22 @@ impl HttpFilter for IterativeRequestRouterFilter { } } + fn visit_nested_pipelines(&mut self, visitor: &mut dyn FnMut(&mut FilterPipeline)) { + for pipeline in self.step_pipelines.values_mut() { + if let Some(pipeline) = Arc::get_mut(pipeline) { + visitor(pipeline); + } else { + debug_assert!(false, "IRR step pipelines must be uniquely owned during configuration"); + } + } + } + + fn apply_insecure_options(&self, options: &praxis_core::config::InsecureOptions) { + for pipeline in self.step_pipelines.values() { + pipeline.apply_insecure_options(options); + } + } + /// Validate the request, then run the iteration at the router's normal /// request-header position after preceding filters have completed. async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { @@ -299,7 +351,7 @@ impl HttpFilter for IterativeRequestRouterFilter { .into() })?; - Box::pin(self.run_iterations(ctx, request_body)).await + Box::pin(self.run_iterations_with_runner(ctx, request_body)).await } } @@ -308,15 +360,17 @@ impl HttpFilter for IterativeRequestRouterFilter { reason = "lifecycle implementation is kept separate from construction" )] impl IterativeRequestRouterFilter { - /// Run the complete iterative subrequest lifecycle. - #[expect(clippy::too_many_lines, reason = "iteration loop is inherently sequential")] - #[expect(clippy::large_stack_frames, reason = "sub-pipeline execution needs stack space")] - #[expect(clippy::large_futures, reason = "streaming and buffered paths require large future")] + /// Run the logical request through the reusable one-step executor. + #[expect(clippy::too_many_lines, reason = "the loop owns explicit state transitions")] + #[expect( + clippy::large_stack_frames, + reason = "opening a step reconstructs a full filter context" + )] #[expect( clippy::significant_drop_tightening, - reason = "step execution temporaries span iteration loop" + reason = "opened streaming steps are consumed by their selected lifecycle" )] - async fn run_iterations( + async fn run_iterations_with_runner( &self, ctx: &mut HttpFilterContext<'_>, request_body: Bytes, @@ -327,14 +381,12 @@ impl IterativeRequestRouterFilter { .ok_or_else(|| -> FilterError { "iterative_request_router: no sub-request client".to_owned().into() })? .clone(); let max_response_bytes = effective_response_limit(self.max_response_bytes, ctx.response_body_mode); - let original_request = SubRequest { method: ctx.request.method.clone(), uri: ctx.request.uri.clone(), headers: ctx.request.headers.clone(), body: request_body, }; - let mut state = IterationState { original_request: original_request.clone(), previous_response: None, @@ -353,25 +405,43 @@ impl IterativeRequestRouterFilter { return Ok(FilterAction::Reject(Rejection::status(413))); } + let runner = IrrStepRunner::new( + client, + depth, + max_response_bytes, + self.max_state_bytes, + StepRuntime { + client_addr: ctx.client_addr, + downstream_tls: ctx.downstream_tls, + peer_identity: ctx.peer_identity.clone(), + request_start: ctx.request_start, + }, + self.step_pipelines.clone(), + self.step_timeout, + ); let mut current_step = Arc::clone(&self.initial_step); let mut current_request = original_request; + let mut extensions = std::mem::take(&mut ctx.extensions); + let mut pending_chunks = VecDeque::new(); + let mut pending_bytes = 0_usize; loop { if state.iteration >= self.max_iterations { + ctx.extensions = extensions; warn!( iterations = state.iteration, max = self.max_iterations, - "iterative_request_router: max iterations \ - exhausted" + "iterative_request_router: max iterations exhausted" ); return Ok(FilterAction::Reject(Rejection::status(508))); } - - let remaining = state + if state .deadline .checked_duration_since(Instant::now()) - .unwrap_or(Duration::ZERO); - if remaining.is_zero() { + .unwrap_or(Duration::ZERO) + .is_zero() + { + ctx.extensions = extensions; warn!( iterations = state.iteration, "iterative_request_router: deadline exceeded" @@ -379,656 +449,252 @@ impl IterativeRequestRouterFilter { return Ok(FilterAction::Reject(Rejection::status(504))); } - let pipeline = self.step_pipelines.get(¤t_step).ok_or_else(|| -> FilterError { - format!( - "iterative_request_router: step \ - '{current_step}' not found" - ) - .into() - })?; - - let step_span = tracing::info_span!( - "iterative_subrequest", - step = current_step.as_ref(), - iteration = state.iteration, - ); - let _enter = step_span.enter(); - - debug!( - step = current_step.as_ref(), - iteration = state.iteration, - "executing step" - ); - - let mut sub_headers = current_request.headers.clone(); - strip_reserved_headers(&mut sub_headers); - - let sub_req = crate::Request { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: sub_headers.clone(), - }; - let mut routed_req = sub_req.clone(); - - // Keep the response metadata alive for the full step - // context so response-header and response-body hooks share - // the same lifecycle state. - let mut response_header = crate::Response { - headers: HeaderMap::new(), - status: http::StatusCode::OK, - }; - let runtime_resources = SubPipelineRuntimeResources { - client_addr: ctx.client_addr, - downstream_tls: ctx.downstream_tls, - health_registry: ctx.health_registry, - id_generator: ctx.id_generator, - kv_stores: ctx.kv_stores, - peer_identity: ctx.peer_identity.as_ref(), - request_start: ctx.request_start, - subrequest_client: Some(&client), - time_source: ctx.time_source, + let opened = match Box::pin(runner.open_step(¤t_step, ¤t_request, &state, extensions)).await { + Ok(opened) => opened, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, }; - let mut filter_ctx = build_sub_filter_context(pipeline, &sub_req, runtime_resources); - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - filter_ctx.extensions.insert(state.clone()); - filter_ctx.extensions.insert(RetainedFilterResults::default()); - let step_timeout = remaining.min(self.step_timeout); - let step_start = Instant::now(); - let in_transport = Arc::new(AtomicBool::new(false)); - let in_transport_inner = Arc::clone(&in_transport); - let step_result: Result = match tokio::time::timeout(step_timeout, async { - let mut request_body = Some(current_request.body.clone()); - if body_exceeds_limit( - pipeline.body_capabilities().request_body_mode, - request_body.as_ref().map_or(0, Bytes::len), - ) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - let pre_read_body = matches!( - pipeline.body_capabilities().request_body_mode, - crate::body::BodyMode::StreamBuffer { .. } - ); - if pre_read_body { - let action = pipeline - .execute_http_request_body(&mut filter_ctx, &mut request_body, true) - .await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - apply_pre_read_header_mutations(&mut routed_req.headers, &filter_ctx); - filter_ctx.extra_request_headers.clear(); - filter_ctx.request_headers_to_remove.clear(); - filter_ctx.request_headers_to_set.clear(); - filter_ctx.pre_read_mutations.clear(); - sub_headers.clone_from(&routed_req.headers); - filter_ctx.request = &routed_req; - } - - let action = pipeline.execute_http_request(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - if !pre_read_body { - let action = pipeline - .execute_http_request_body(&mut filter_ctx, &mut request_body, true) - .await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); + let runner::OpenedStep { continuation, kind } = opened; + + match kind { + OpenedStepKind::Streaming { body, outcome } => { + let transitions = self.step_transitions.get(¤t_step).map_or(&[][..], Vec::as_slice); + if !streaming_transition_order_is_valid(transitions) { + (*body).cancel().await; + ctx.extensions = continuation.into_parent_extensions(); + return Err(format!( + "iterative_request_router: step '{current_step}' selected streaming with interleaved transition phases" + ) + .into()); } - } - - let upstream = filter_ctx.upstream.as_ref().ok_or_else(|| -> FilterError { - format!( - "iterative_request_router: step \ - '{current_step}' did not resolve an upstream" - ) - .into() - })?; - in_transport_inner.store(true, Ordering::Release); - let peer = build_peer(upstream).await; - - apply_request_header_mutations(&mut sub_headers, &filter_ctx); - ensure_destination_host(&mut sub_headers, &upstream.address)?; - sanitize_subrequest_headers(&mut sub_headers); - let sub_request_for_exec = SubRequest { - method: current_request.method.clone(), - uri: filter_ctx.rewritten_path.as_ref().map_or_else( - || current_request.uri.clone(), - |p| http::Uri::try_from(p.as_str()).unwrap_or_else(|_| current_request.uri.clone()), - ), - headers: sub_headers, - body: request_body.unwrap_or_default(), - }; - - let mut fw_headers = FrameworkHeaders::new(); - fw_headers.set_depth(depth + 1); - - let per_request_timeout = step_timeout.checked_sub(step_start.elapsed()).unwrap_or(Duration::ZERO); - if per_request_timeout.is_zero() { - return Ok(StepExecution::Rejected(Rejection::status(504))); - } - - match filter_ctx.subrequest_response_mode { - SubRequestResponseMode::Streaming => { - // Runtime guard: reject if transitions depend on response body - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - if has_body_dependent_transitions(transitions) { - return Err(format!( - "iterative_request_router: step '{current_step}' selected \ - streaming but has body-dependent transitions" + match evaluate_header_transitions(transitions, &outcome) { + TransitionResult::Next(next_step) => { + debug!( + from = current_step.as_ref(), + to = next_step.as_ref(), + "streaming header failover before response commitment" + ); + let mut skipped = IrrStreamingBody::new(body, continuation); + if let Err(error) = skipped.suppress().await { + ctx.extensions = skipped.into_continuation().into_parent_extensions(); + return Err(error); + } + let mut completion = skipped.into_continuation().into_completion()?; + completion.state.previous_response = None; + completion.state.iteration += 1; + if completion.state.retained_bytes() > self.max_state_bytes { + ctx.extensions = completion.extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } + let next_body = completion + .next_iteration_body + .unwrap_or_else(|| current_request.body.clone()); + state = completion.state; + extensions = completion.extensions; + current_request = SubRequest { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: HeaderMap::new(), + body: next_body, + }; + current_step = next_step; + }, + TransitionResult::Done | TransitionResult::NoMatch => { + let Some(active_state) = continuation.extensions.get::() else { + (*body).cancel().await; + ctx.extensions = continuation.into_parent_extensions(); + return Err( + "iterative_request_router: iteration state missing before stream handoff" + .to_owned() + .into(), + ); + }; + if ensure_combined_retained_limit( + active_state.retained_bytes(), + pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, ) - .into()); - } - if matches!( - pipeline.body_capabilities().response_body_mode, - crate::body::BodyMode::StreamBuffer { .. } - ) { - return Err(format!( - "iterative_request_router: step '{current_step}' selected \ - streaming but has StreamBuffer response body mode" + .is_err() + { + (*body).cancel().await; + let completion = continuation.into_completion()?; + ctx.extensions = completion.extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } + let status = normalize_response_status(outcome.response.status); + let headers = outcome.response.headers.clone(); + let terminal = StreamingTerminalResponse::new( + status, + Box::new(IrrStreamingSession::new( + runner, + Arc::clone(¤t_step), + current_request, + outcome, + body, + continuation, + pending_chunks, + self.step_transitions.clone(), + self.max_state_bytes, + self.max_stream_response_bytes, + )), ) - .into()); - } - - let mut step_origin = config::ResponseOrigin::Upstream; - let mut step_transport_error = None; - - let streaming_result = match peer { - Ok(peer) => { - let limits = StreamLimits { - idle_timeout: STREAMING_IDLE_TIMEOUT, - max_stream_duration: None, - max_total_bytes: None, - }; - Box::pin(client.send_streaming( - &peer, - &sub_request_for_exec, - per_request_timeout, - limits, - Some(&fw_headers), - )) - .await - }, - Err(error) => Err(praxis_core::subrequest::SubRequestError::Connect(error.to_string())), - }; - in_transport_inner.store(false, Ordering::Release); - - match streaming_result { - Ok(streaming_response) => { - let status = streaming_response.status; - let mut response_headers = streaming_response.headers; - sanitize_subresponse_headers(&mut response_headers); - - response_header.status = - http::StatusCode::from_u16(status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid upstream status: {e}").into() - })?; - response_header.headers.clone_from(&response_headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - streaming_response.body.cancel().await; - return Ok(StepExecution::Rejected(rejection)); - } - - // Extract final response metadata from filter context - let (final_status, mut final_headers) = - if let Some(meta) = filter_ctx.response_header.as_deref() { - (meta.status, meta.headers.clone()) - } else { - ( - http::StatusCode::from_u16(status).unwrap_or(http::StatusCode::BAD_GATEWAY), - response_headers, - ) - }; - sanitize_subresponse_headers(&mut final_headers); - - let outcome = StepOutcome { - response: SubResponse { - status: final_status.as_u16(), - headers: final_headers.clone(), - body: Bytes::new(), - }, - origin: step_origin, - transport_error: step_transport_error, - }; + .with_headers(headers); + return Ok(FilterAction::StreamingTerminalResponse(Box::new(terminal))); + }, + } + }, + OpenedStepKind::Complete(mut outcome) => { + let completion = continuation.into_completion()?; + let abnormal_stream_completion = completion.termination.is_some(); + let handled_abnormal_stream_completion = completion + .termination + .as_ref() + .is_some_and(StreamTermination::is_handled); + let completed_pending_chunks = completion.pending_chunks; + state = completion.state; + state.previous_response = Some(outcome.response.clone()); + state.iteration += 1; + let next_iteration_body = completion.next_iteration_body; + let filter_results = completion.filter_results; + extensions = completion.extensions; + if state.retained_bytes() > self.max_state_bytes { + ctx.extensions = extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } - Ok(StepExecution::Streaming { - outcome, - body: Box::new(streaming_response.body), - response_snapshot: crate::Response { - status: final_status, - headers: final_headers, - }, - }) - }, - Err(error) => { - let (status, kind) = classify_transport_failure(&error); - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(kind); - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: streaming sub-request transport failure" + info!( + step = current_step.as_ref(), + iteration = state.iteration - 1, + status = outcome.response.status, + body_bytes = outcome.response.body.len(), + "sub-request complete" + ); + let transitions = self.step_transitions.get(¤t_step).map_or(&[][..], Vec::as_slice); + match evaluate_transitions(transitions, &outcome, &filter_results) { + TransitionResult::Next(next_step) => { + if !abnormal_stream_completion { + let appended = append_pending_chunks( + &mut pending_chunks, + completed_pending_chunks, + pending_bytes, + self.max_state_bytes, + state.retained_bytes(), ); - let response = SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), + let Ok(updated_pending_bytes) = appended else { + ctx.extensions = extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); }; - // Fall through to buffered transition path with synthetic response - // (transport failures don't have a body to stream) - - response_header.status = - http::StatusCode::from_u16(status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid status: {e}").into() + pending_bytes = updated_pending_bytes; + } + let next_body = next_iteration_body.unwrap_or_else(|| current_request.body.clone()); + current_request = SubRequest { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: HeaderMap::new(), + body: next_body, + }; + current_step = next_step; + }, + TransitionResult::Done | TransitionResult::NoMatch => { + if handled_abnormal_stream_completion { + let combined_bytes = pending_chunks + .iter() + .chain(completed_pending_chunks.iter()) + .chain(std::iter::once(&outcome.response.body)) + .try_fold(0_usize, |total, chunk| total.checked_add(chunk.len())) + .ok_or_else(|| -> FilterError { + "iterative_request_router: completion body byte count overflow".into() })?; - response_header.headers.clone_from(&response.headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - - Ok(StepExecution::Complete { - response, - origin: step_origin, - transport_error: step_transport_error, - }) - }, - } - }, - SubRequestResponseMode::Buffered => { - let mut step_origin = config::ResponseOrigin::Upstream; - let mut step_transport_error = None; - let mut response = match peer { - Ok(peer) => match Box::pin(client.execute( - &peer, - &sub_request_for_exec, - max_response_bytes, - per_request_timeout, - Some(&fw_headers), - )) - .await - { - Ok(response) => response, - Err(error) => { - let (status, kind) = classify_transport_failure(&error); - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(kind); - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: sub-request transport failure" + if combined_bytes > max_response_bytes { + ctx.extensions = extensions; + return Err( + "iterative_request_router: abnormal completion exceeds response body limit" + .to_owned() + .into(), ); - SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), - } - }, - }, - Err(error) => { - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(config::TransportErrorKind::Connect); - let status = 502; - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: sub-request transport failure" - ); - SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), } - }, - }; - in_transport_inner.store(false, Ordering::Release); - sanitize_subresponse_headers(&mut response.headers); - - response_header.status = - http::StatusCode::from_u16(response.status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid upstream status: {e}").into() - })?; - response_header.headers.clone_from(&response.headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - let mut response_body = Some(std::mem::take(&mut response.body)); - if response_body_exceeds_limits( - pipeline.body_capabilities().response_body_mode, - max_response_bytes, - response_body.as_ref().map_or(0, Bytes::len), - ) { - return Err("iterative_request_router: step response exceeds configured body limit" - .to_owned() - .into()); - } - let action = pipeline.execute_http_response_body(&mut filter_ctx, &mut response_body, true)?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - if response_body_exceeds_limits( - pipeline.body_capabilities().response_body_mode, - max_response_bytes, - response_body.as_ref().map_or(0, Bytes::len), - ) { - return Err( - "iterative_request_router: transformed step response exceeds configured body limit" - .to_owned() - .into(), - ); - } - - if let Some(meta) = filter_ctx.response_header.as_deref() { - response.status = meta.status.as_u16(); - response.headers.clone_from(&meta.headers); - } - response.body = response_body.unwrap_or_default(); - sanitize_subresponse_headers(&mut response.headers); - - Ok(StepExecution::Complete { - response, - origin: step_origin, - transport_error: step_transport_error, - }) - }, - } - }) - .await - { - Ok(result) => result, - Err(_elapsed) => { - if in_transport.load(Ordering::Acquire) { - Ok(StepExecution::Complete { - response: SubResponse { - status: 504, - headers: HeaderMap::new(), - body: Bytes::new(), - }, - origin: config::ResponseOrigin::Transport, - transport_error: Some(config::TransportErrorKind::DeadlineExceeded), - }) - } else { - Ok(StepExecution::Rejected(Rejection::status(504))) + let mut combined = Vec::with_capacity(combined_bytes); + for chunk in pending_chunks.drain(..) { + combined.extend_from_slice(&chunk); + } + for chunk in completed_pending_chunks { + combined.extend_from_slice(&chunk); + } + combined.extend_from_slice(&outcome.response.body); + outcome.response.body = Bytes::from(combined); + } else if abnormal_stream_completion { + pending_chunks.clear(); + outcome.response.body = Bytes::new(); + } else if !pending_chunks.is_empty() || !completed_pending_chunks.is_empty() { + ctx.extensions = extensions; + return Err( + "iterative_request_router: stream chunks were emitted without a streaming response" + .to_owned() + .into(), + ); + } + ctx.extensions = extensions; + return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( + &outcome.response, + current_request.method == http::Method::HEAD, + )))); + }, } }, - }; - - // Handle streaming step results before normal post-step processing. - // The streaming terminal path takes ownership of filter_ctx.extensions - // (which include the parent's extensions swapped in at step start). - if let Ok(StepExecution::Streaming { - outcome, - body: upstream_body, - response_snapshot, - }) = step_result - { - // Extract iteration state and filter results for transition evaluation - if let Some(updated_state) = filter_ctx.extensions.remove::() { - state = updated_state; - } - let next_iteration_body = filter_ctx.extensions.remove::(); - let mut step_filter_results = filter_ctx - .extensions - .remove::() - .unwrap_or_default() - .0; - step_filter_results.extend( - filter_ctx - .filter_results - .iter() - .map(|(name, results)| (*name, results.clone())), - ); - - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - match evaluate_transitions(transitions, &outcome, &step_filter_results) { - TransitionResult::Done | TransitionResult::NoMatch => { - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - let continuation = StepResponseContinuation { - pipeline: Arc::clone(self.step_pipelines.get(¤t_step).ok_or_else( - || -> FilterError { - format!("iterative_request_router: step '{current_step}' not found").into() - }, - )?), - request_snapshot: crate::Request { - method: filter_ctx.request.method.clone(), - uri: filter_ctx.request.uri.clone(), - headers: filter_ctx.request.headers.clone(), - }, - response_snapshot, - extensions: std::mem::take(&mut filter_ctx.extensions), - filter_state: std::mem::take(&mut filter_ctx.filter_state), - filter_results: std::mem::take(&mut filter_ctx.filter_results), - filter_metadata: std::mem::take(&mut filter_ctx.filter_metadata), - structured_metadata: std::mem::take(&mut filter_ctx.structured_metadata), - executed_filter_indices: std::mem::take(&mut filter_ctx.executed_filter_indices), - body_done_indices: std::mem::take(&mut filter_ctx.body_done_indices), - response_body_bytes: filter_ctx.response_body_bytes, - response_body_mode: filter_ctx.response_body_mode, - completed: false, - client_addr: filter_ctx.client_addr, - downstream_tls: filter_ctx.downstream_tls, - request_start: filter_ctx.request_start, - peer_identity: filter_ctx.peer_identity.clone(), - }; - let status = normalize_response_status(outcome.response.status); - let terminal = StreamingTerminalResponse::new( - status, - Box::new(IrrStreamingBody::new(*upstream_body, continuation)), - ) - .with_headers(outcome.response.headers); - return Ok(FilterAction::StreamingTerminalResponse(Box::new(terminal))); - }, - TransitionResult::Next(next_step) => { - debug!( - from = current_step.as_ref(), - to = next_step.as_ref(), - "streaming failover: cancelling unread body" - ); - (*upstream_body).cancel().await; - // Swap extensions back to parent and continue the loop - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - state.previous_response = None; - state.iteration += 1; - let next_body = next_iteration_body.map_or_else(|| current_request.body.clone(), |b| b.0); - current_request = SubRequest { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: HeaderMap::new(), - body: next_body, - }; - current_step = next_step; - continue; - }, - } - } - - // Existing post-step processing continues unchanged for - // Complete/Rejected variants... - if let Some(updated_state) = filter_ctx.extensions.remove::() { - state = updated_state; - } - if state.retained_bytes() > self.max_state_bytes { - return Ok(FilterAction::Reject(Rejection::status(413))); - } - let next_iteration_body = filter_ctx.extensions.remove::(); - let mut step_filter_results = filter_ctx - .extensions - .remove::() - .unwrap_or_default() - .0; - step_filter_results.extend( - filter_ctx - .filter_results - .iter() - .map(|(name, results)| (*name, results.clone())), - ); - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - - let step_result_value = step_result?; - let (mut response, origin, transport_error) = match step_result_value { - StepExecution::Complete { - response, - origin, - transport_error, - } => (response, origin, transport_error), - StepExecution::Rejected(rejection) => { - debug!( - step = current_step.as_ref(), - status = rejection.status, - "step pipeline produced a local response" - ); - ( - subresponse_from_rejection(rejection), - config::ResponseOrigin::Local, - None, - ) - }, - StepExecution::Streaming { .. } => { - unreachable!("streaming step results are handled before this point") - }, - }; - sanitize_subresponse_headers(&mut response.headers); - - info!( - step = current_step.as_ref(), - iteration = state.iteration, - status = response.status, - body_bytes = response.body.len(), - "sub-request complete" - ); - - state.previous_response = Some(response.clone()); - state.iteration += 1; - if state.retained_bytes() > self.max_state_bytes { - return Ok(FilterAction::Reject(Rejection::status(413))); - } - - let outcome = StepOutcome { - response, - origin, - transport_error, - }; - - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - - match evaluate_transitions(transitions, &outcome, &step_filter_results) { - TransitionResult::Done => { - debug!(step = current_step.as_ref(), "iteration complete, returning response"); - return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( - &outcome.response, - current_request.method == http::Method::HEAD, - )))); - }, - TransitionResult::Next(next_step) => { - debug!( - from = current_step.as_ref(), - to = next_step.as_ref(), - "transitioning to next step" - ); - let next_body = next_iteration_body.map_or_else(|| current_request.body.clone(), |b| b.0); - current_request = SubRequest { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: HeaderMap::new(), - body: next_body, - }; - current_step = next_step; - }, - TransitionResult::NoMatch => { - debug!( - step = current_step.as_ref(), - "no transition matched, returning response" - ); - return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( - &outcome.response, - current_request.method == http::Method::HEAD, - )))); - }, } } } } +/// Append locally emitted chunks while preserving the shared retained-state bound. +fn append_pending_chunks( + target: &mut VecDeque, + chunks: VecDeque, + current_bytes: usize, + max_state_bytes: usize, + retained_bytes: usize, +) -> Result { + let added_bytes = chunks.iter().try_fold(0_usize, |total, chunk| { + total + .checked_add(chunk.len()) + .ok_or_else(|| -> FilterError { "iterative_request_router: pending stream byte count overflow".into() }) + })?; + let pending_bytes = current_bytes + .checked_add(added_bytes) + .ok_or_else(|| -> FilterError { "iterative_request_router: pending stream byte count overflow".into() })?; + if retained_bytes + .checked_add(pending_bytes) + .is_none_or(|total| total > max_state_bytes) + { + return Err( + "iterative_request_router: retained state and pending stream output exceed configured limit" + .to_owned() + .into(), + ); + } + target.extend(chunks); + Ok(pending_bytes) +} + // --------------------------------------------------------------------------- // Transition Evaluation // --------------------------------------------------------------------------- -/// Result of executing one step's complete filter and HTTP lifecycle. -enum StepExecution { - /// A step filter rejected the request or response. - Rejected(Rejection), - - /// The upstream exchange completed after all response hooks. - Complete { - /// The sub-request response. - response: SubResponse, - /// Where the response originated. - origin: config::ResponseOrigin, - /// Transport error classification, if any. - transport_error: Option, - }, - - /// The step uses streaming mode; response-header filters ran. - Streaming { - /// Step outcome with header-only response snapshot. - outcome: StepOutcome, - /// Upstream streaming body handle. - body: Box, - /// Snapshot of response headers for continuation. - response_snapshot: crate::Response, - }, -} - /// A step's response together with metadata about where it came from. -struct StepOutcome { +pub(super) struct StepOutcome { /// The sub-request response. - response: SubResponse, + pub(super) response: SubResponse, /// Where the response originated. - origin: config::ResponseOrigin, + pub(super) origin: config::ResponseOrigin, /// Transport error classification, if any. - transport_error: Option, + pub(super) transport_error: Option, } /// Result of evaluating step transition rules. -enum TransitionResult { +pub(super) enum TransitionResult { /// Return the current response to the client. Done, @@ -1040,7 +706,9 @@ enum TransitionResult { } /// Convert transport failures into a gateway status and error classification. -fn classify_transport_failure(error: &praxis_core::subrequest::SubRequestError) -> (u16, config::TransportErrorKind) { +pub(super) fn classify_transport_failure( + error: &praxis_core::subrequest::SubRequestError, +) -> (u16, config::TransportErrorKind) { use praxis_core::subrequest::SubRequestError; match error { SubRequestError::AdmissionTimeout { .. } => (503, config::TransportErrorKind::AdmissionTimeout), @@ -1086,7 +754,7 @@ fn normalize_response_status(status: u16) -> u16 { } /// Evaluate transition rules against a step outcome. -fn evaluate_transitions( +pub(super) fn evaluate_transitions( transitions: &[config::StepTransition], outcome: &StepOutcome, filter_results: &HashMap<&str, crate::results::FilterResultSet>, @@ -1106,6 +774,24 @@ fn evaluate_transitions( TransitionResult::NoMatch } +/// Evaluate only the ordered header-safe failover prefix. +pub(super) fn evaluate_header_transitions( + transitions: &[config::StepTransition], + outcome: &StepOutcome, +) -> TransitionResult { + for transition in transitions + .iter() + .take_while(|transition| is_header_safe_failover(transition)) + { + if matches_transition(transition, outcome, &HashMap::new()) + && let Some(next) = &transition.next + { + return TransitionResult::Next(Arc::from(next.as_str())); + } + } + TransitionResult::NoMatch +} + /// Check if a transition matches the outcome and/or filter results. fn matches_transition( transition: &config::StepTransition, @@ -1153,7 +839,8 @@ fn parse_depth(request: &crate::Request) -> u8 { } /// Strip all reserved internal headers from sub-request headers -/// so the core executor re-injects depth via [`FrameworkHeaders`]. +/// so the core executor re-injects depth via +/// [`FrameworkHeaders`](praxis_core::subrequest::FrameworkHeaders). /// /// The depth header uses a reserved `x-praxis-*` prefix, so it /// is covered by the [`is_reserved`] check. @@ -1312,6 +999,18 @@ fn effective_response_limit(configured: usize, parent_mode: crate::body::BodyMod } } +/// Extract only the listener-level streaming ceiling from a nested pipeline. +/// +/// The IRR `max_response_bytes` setting is intentionally buffered-only. A +/// nested `SizeLimit` is produced by listener body-limit propagation and must +/// still constrain the live transport. +fn streaming_transport_limit(mode: crate::body::BodyMode) -> Option { + match mode { + crate::body::BodyMode::SizeLimit { max_bytes } => Some(max_bytes), + crate::body::BodyMode::Stream | crate::body::BodyMode::StreamBuffer { .. } => None, + } +} + /// Whether a step filter grew the shared iteration state past its ceiling. fn iteration_state_exceeds_limit(ctx: &HttpFilterContext<'_>, max_state_bytes: usize) -> bool { ctx.extensions diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs new file mode 100644 index 000000000..959a0e8dc --- /dev/null +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Praxis Contributors + +//! One-step execution for the iterative request router. + +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use bytes::Bytes; +use http::HeaderMap; +use praxis_core::subrequest::{FrameworkHeaders, StreamLimits}; +use tracing::warn; + +use super::{ + StepOutcome, SubPipelineRuntimeResources, apply_pre_read_header_mutations, apply_request_header_mutations, + body_exceeds_limit, build_peer, build_sub_filter_context, classify_transport_failure, config, + ensure_destination_host, iteration_state_exceeds_limit, response_body_exceeds_limits, sanitize_subrequest_headers, + sanitize_subresponse_headers, streaming::StepResponseContinuation, streaming_transport_limit, + strip_reserved_headers, subresponse_from_rejection, +}; +use crate::{ + FilterAction, FilterError, FilterPipeline, IterationState, NextIterationBody, RequestExtensions, StreamTermination, + StreamTerminationCause, SubRequest, SubRequestResponseMode, SubResponse, actions::Rejection, + context::PendingStreamChunks, results::RetainedFilterResults, +}; + +/// Owned request attributes needed after the outer request hook returns. +#[derive(Clone)] +pub(super) struct StepRuntime { + /// Original downstream client address. + pub(super) client_addr: Option, + /// Whether the original downstream uses TLS. + pub(super) downstream_tls: bool, + /// Verified downstream peer identity. + pub(super) peer_identity: Option, + /// Start time of the logical client request. + pub(super) request_start: Instant, +} + +/// Executes exactly one IRR step and returns owned continuation state. +pub(super) struct IrrStepRunner { + /// Shared transport client. + client: praxis_core::subrequest::SubRequestClient, + /// Nested IRR depth forwarded to subrequests. + depth: u8, + /// Per-step buffered response ceiling. + max_response_bytes: usize, + /// Retained iteration-state and pending-output ceiling. + max_state_bytes: usize, + /// Owned downstream request attributes. + runtime: StepRuntime, + /// Named, pre-built step pipelines. + step_pipelines: HashMap, Arc>, + /// Per-step duration ceiling. + step_timeout: Duration, +} + +/// One opened step, including state needed for body/completion processing. +pub(super) struct OpenedStep { + /// Owned filter lifecycle state. + pub(super) continuation: StepResponseContinuation, + /// Buffered or pull-based response source. + pub(super) kind: OpenedStepKind, +} + +/// A step error together with the parent request extensions it borrowed. +pub(super) struct OpenStepError { + /// Underlying filter or lifecycle error. + error: FilterError, + /// Parent-owned extensions recovered from the nested filter context. + extensions: RequestExtensions, +} + +impl OpenStepError { + /// Build an error before a nested filter context exists. + fn new(error: FilterError, extensions: RequestExtensions) -> Self { + Self { error, extensions } + } + + /// Recover only the parent-owned extensions from a failed nested context. + fn capture(error: FilterError, ctx: &mut crate::HttpFilterContext<'_>) -> Self { + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + Self::new(error, std::mem::take(&mut ctx.extensions)) + } + + /// Split the error from the extensions its caller must restore. + pub(super) fn into_parts(self) -> (FilterError, RequestExtensions) { + (self.error, self.extensions) + } +} + +/// Transport/body shape selected by the step filters. +pub(super) enum OpenedStepKind { + /// The complete response was collected and filtered. + Complete(StepOutcome), + /// Response headers are filtered; body remains pull-based. + Streaming { + /// Live upstream response body. + body: Box, + /// Header-time transition metadata. + outcome: StepOutcome, + }, +} + +/// Internal result before owned continuation state is captured. +enum RawStepKind { + /// Complete buffered or synthetic response. + Complete(StepOutcome), + /// Local filter rejection. + Rejected(Rejection), + /// Open pull-based upstream response. + Streaming { + /// Live upstream response body. + body: Box, + /// Header-time transition metadata. + outcome: StepOutcome, + }, +} + +impl IrrStepRunner { + /// Build an owned runner for one logical IRR request. + #[expect(clippy::too_many_arguments, reason = "runner owns explicit IRR limits and resources")] + pub(super) fn new( + client: praxis_core::subrequest::SubRequestClient, + depth: u8, + max_response_bytes: usize, + max_state_bytes: usize, + runtime: StepRuntime, + step_pipelines: HashMap, Arc>, + step_timeout: Duration, + ) -> Self { + Self { + client, + depth, + max_response_bytes, + max_state_bytes, + runtime, + step_pipelines, + step_timeout, + } + } + + /// Open one named step under the remaining overall deadline. + #[expect( + clippy::too_many_lines, + reason = "one step owns the complete filter and transport lifecycle" + )] + #[expect(clippy::large_futures, reason = "step execution spans filter and transport futures")] + #[expect( + clippy::large_stack_frames, + reason = "step execution reconstructs a full filter context" + )] + pub(super) async fn open_step( + &self, + current_step: &Arc, + current_request: &SubRequest, + state: &IterationState, + mut extensions: RequestExtensions, + ) -> Result { + let remaining = state + .deadline() + .checked_duration_since(Instant::now()) + .unwrap_or(Duration::ZERO); + if remaining.is_zero() { + return Err(OpenStepError::new( + "iterative_request_router: overall deadline exceeded".to_owned().into(), + extensions, + )); + } + let Some(pipeline) = self.step_pipelines.get(current_step) else { + return Err(OpenStepError::new( + format!("iterative_request_router: step '{current_step}' not found").into(), + extensions, + )); + }; + + let mut sub_headers = current_request.headers.clone(); + strip_reserved_headers(&mut sub_headers); + let sub_req = crate::Request { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: sub_headers.clone(), + }; + let mut routed_req = sub_req.clone(); + let mut response_header = crate::Response { + headers: HeaderMap::new(), + status: http::StatusCode::OK, + }; + let resources = SubPipelineRuntimeResources { + client_addr: self.runtime.client_addr, + downstream_tls: self.runtime.downstream_tls, + health_registry: pipeline.health_registry(), + id_generator: pipeline.id_generator(), + kv_stores: pipeline.kv_stores(), + peer_identity: self.runtime.peer_identity.as_ref(), + request_start: self.runtime.request_start, + subrequest_client: Some(&self.client), + time_source: pipeline.time_source(), + }; + let mut filter_ctx = build_sub_filter_context(pipeline, &sub_req, resources); + filter_ctx.extensions = std::mem::take(&mut extensions); + filter_ctx.extensions.insert(state.clone()); + filter_ctx.extensions.insert(RetainedFilterResults::default()); + filter_ctx.enable_stream_chunk_emission(self.max_state_bytes); + + let step_budget = remaining.min(self.step_timeout); + let step_started = Instant::now(); + let step_deadline = step_started + .checked_add(step_budget) + .unwrap_or_else(|| state.deadline()); + let in_transport = Arc::new(AtomicBool::new(false)); + let in_transport_inner = Arc::clone(&in_transport); + + let timed: Result, tokio::time::error::Elapsed> = + tokio::time::timeout(step_budget, async { + let mut request_body = Some(current_request.body.clone()); + if body_exceeds_limit( + pipeline.body_capabilities().request_body_mode, + request_body.as_ref().map_or(0, Bytes::len), + ) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + + let pre_read_body = matches!( + pipeline.body_capabilities().request_body_mode, + crate::BodyMode::StreamBuffer { .. } + ); + if pre_read_body { + let action = pipeline + .execute_http_request_body(&mut filter_ctx, &mut request_body, true) + .await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + apply_pre_read_header_mutations(&mut routed_req.headers, &filter_ctx); + filter_ctx.extra_request_headers.clear(); + filter_ctx.request_headers_to_remove.clear(); + filter_ctx.request_headers_to_set.clear(); + filter_ctx.pre_read_mutations.clear(); + sub_headers.clone_from(&routed_req.headers); + filter_ctx.request = &routed_req; + } + + let action = pipeline.execute_http_request(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + if !pre_read_body { + let action = pipeline + .execute_http_request_body(&mut filter_ctx, &mut request_body, true) + .await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + } + + let upstream = filter_ctx.upstream.as_ref().ok_or_else(|| -> FilterError { + format!("iterative_request_router: step '{current_step}' did not resolve an upstream").into() + })?; + in_transport_inner.store(true, Ordering::Release); + let peer = build_peer(upstream).await; + apply_request_header_mutations(&mut sub_headers, &filter_ctx); + ensure_destination_host(&mut sub_headers, &upstream.address)?; + sanitize_subrequest_headers(&mut sub_headers); + let request = SubRequest { + method: current_request.method.clone(), + uri: filter_ctx.rewritten_path.as_ref().map_or_else( + || current_request.uri.clone(), + |path| http::Uri::try_from(path.as_str()).unwrap_or_else(|_| current_request.uri.clone()), + ), + headers: sub_headers, + body: request_body.unwrap_or_default(), + }; + let mut framework_headers = FrameworkHeaders::new(); + framework_headers.set_depth(self.depth + 1); + let transport_budget = step_budget + .checked_sub(step_started.elapsed()) + .unwrap_or(Duration::ZERO); + if transport_budget.is_zero() { + return Ok(RawStepKind::Rejected(Rejection::status(504))); + } + + match filter_ctx.subrequest_response_mode { + SubRequestResponseMode::Streaming => { + if matches!( + pipeline.body_capabilities().response_body_mode, + crate::BodyMode::StreamBuffer { .. } + ) { + return Err(format!( + "iterative_request_router: step '{current_step}' selected streaming with StreamBuffer response mode" + ) + .into()); + } + let limits = StreamLimits { + idle_timeout: super::STREAMING_IDLE_TIMEOUT, + // IrrStreamingBody enforces the original absolute step + // deadline so header time cannot be granted twice. + max_stream_duration: None, + max_total_bytes: streaming_transport_limit( + pipeline.body_capabilities().response_body_mode, + ), + }; + let response = match peer { + Ok(peer) => self + .client + .send_streaming(&peer, &request, transport_budget, limits, Some(&framework_headers)) + .await, + Err(error) => Err(praxis_core::subrequest::SubRequestError::Connect(error.to_string())), + }; + in_transport_inner.store(false, Ordering::Release); + match response { + Ok(response) => { + let status = response.status; + let mut headers = response.headers; + sanitize_subresponse_headers(&mut headers); + response_header.status = http::StatusCode::from_u16(status) + .map_err(|error| -> FilterError { format!("invalid upstream status: {error}").into() })?; + response_header.headers.clone_from(&headers); + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + response.body.cancel().await; + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + response.body.cancel().await; + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let metadata = filter_ctx.response_header.as_deref().ok_or_else(|| -> FilterError { + "iterative_request_router: response metadata missing after header filters" + .to_owned() + .into() + })?; + let status = metadata.status; + let mut headers = metadata.headers.clone(); + sanitize_subresponse_headers(&mut headers); + Ok(RawStepKind::Streaming { + body: Box::new(response.body), + outcome: StepOutcome { + response: SubResponse { status: status.as_u16(), headers, body: Bytes::new() }, + origin: config::ResponseOrigin::Upstream, + transport_error: None, + }, + }) + }, + Err(error) => { + let (status, kind) = classify_transport_failure(&error); + warn!(step = current_step.as_ref(), %error, status, "IRR streaming transport failure"); + let response = SubResponse { status, headers: HeaderMap::new(), body: Bytes::new() }; + response_header.status = http::StatusCode::from_u16(status) + .map_err(|source| -> FilterError { source.into() })?; + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let metadata = filter_ctx.response_header.as_deref().ok_or_else(|| -> FilterError { + "iterative_request_router: response metadata missing after header filters" + .to_owned() + .into() + })?; + let mut headers = metadata.headers.clone(); + sanitize_subresponse_headers(&mut headers); + Ok(RawStepKind::Complete(StepOutcome { + response: SubResponse { + status: metadata.status.as_u16(), + headers, + body: response.body, + }, + origin: config::ResponseOrigin::Transport, + transport_error: Some(kind), + })) + }, + } + }, + SubRequestResponseMode::Buffered => { + let (mut response, origin, transport_error) = match peer { + Ok(peer) => match self + .client + .execute(&peer, &request, self.max_response_bytes, transport_budget, Some(&framework_headers)) + .await + { + Ok(response) => (response, config::ResponseOrigin::Upstream, None), + Err(error) => { + let (status, kind) = classify_transport_failure(&error); + ( + SubResponse { status, headers: HeaderMap::new(), body: Bytes::new() }, + config::ResponseOrigin::Transport, + Some(kind), + ) + }, + }, + Err(_) => ( + SubResponse { status: 502, headers: HeaderMap::new(), body: Bytes::new() }, + config::ResponseOrigin::Transport, + Some(config::TransportErrorKind::Connect), + ), + }; + in_transport_inner.store(false, Ordering::Release); + sanitize_subresponse_headers(&mut response.headers); + response_header.status = http::StatusCode::from_u16(response.status) + .map_err(|error| -> FilterError { error.into() })?; + response_header.headers.clone_from(&response.headers); + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let mut body = Some(std::mem::take(&mut response.body)); + if response_body_exceeds_limits( + pipeline.body_capabilities().response_body_mode, + self.max_response_bytes, + body.as_ref().map_or(0, Bytes::len), + ) { + return Err("iterative_request_router: step response exceeds configured body limit".into()); + } + let body_action = pipeline.execute_http_response_body(&mut filter_ctx, &mut body, true)?; + if let FilterAction::Reject(rejection) = body_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + if response_body_exceeds_limits( + pipeline.body_capabilities().response_body_mode, + self.max_response_bytes, + body.as_ref().map_or(0, Bytes::len), + ) { + return Err( + "iterative_request_router: transformed step response exceeds configured body limit" + .into(), + ); + } + response.body = body.unwrap_or_default(); + if let Some(metadata) = filter_ctx.response_header.as_deref() { + response.status = metadata.status.as_u16(); + response.headers.clone_from(&metadata.headers); + } + sanitize_subresponse_headers(&mut response.headers); + Ok(RawStepKind::Complete(StepOutcome { response, origin, transport_error })) + }, + } + }) + .await; + + let mut raw = match timed { + Ok(Ok(raw)) => raw, + Ok(Err(error)) => return Err(OpenStepError::capture(error, &mut filter_ctx)), + Err(_) if in_transport.load(Ordering::Acquire) => RawStepKind::Complete(StepOutcome { + response: SubResponse { + status: 504, + headers: HeaderMap::new(), + body: Bytes::new(), + }, + origin: config::ResponseOrigin::Transport, + transport_error: Some(config::TransportErrorKind::DeadlineExceeded), + }), + Err(_) => RawStepKind::Rejected(Rejection::status(504)), + }; + + filter_ctx.response_header = None; + if filter_ctx.subrequest_response_mode == SubRequestResponseMode::Streaming + && let RawStepKind::Complete(outcome) = &mut raw + && outcome.origin == config::ResponseOrigin::Transport + { + let cause = outcome + .transport_error + .map_or(StreamTerminationCause::Io, stream_termination_cause); + filter_ctx.extensions.insert(StreamTermination::new(cause)); + let response_snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status).unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + let mut completion_body = None; + let completion_action = pipeline + .execute_http_response_body_with_response_header( + &mut filter_ctx, + &mut completion_body, + true, + Some(&response_snapshot), + ) + .map_err(|error| OpenStepError::capture(error, &mut filter_ctx))?; + if let FilterAction::Reject(_) = completion_action { + let error = "iterative_request_router: step completion filter rejected an abnormal stream" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + let error = "iterative_request_router: retained state limit exceeded during stream completion" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + if completion_body + .as_ref() + .is_some_and(|body| body.len() > self.max_response_bytes) + { + let error = "iterative_request_router: abnormal completion exceeds response body limit" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + outcome.response.body = completion_body.unwrap_or_default(); + } + let (kind, response_snapshot, completed) = match raw { + RawStepKind::Complete(outcome) => { + let snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status) + .unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + (OpenedStepKind::Complete(outcome), snapshot, true) + }, + RawStepKind::Rejected(rejection) => { + let response = subresponse_from_rejection(rejection); + let snapshot = crate::Response { + status: http::StatusCode::from_u16(response.status).unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: response.headers.clone(), + }; + ( + OpenedStepKind::Complete(StepOutcome { + response, + origin: config::ResponseOrigin::Local, + transport_error: None, + }), + snapshot, + true, + ) + }, + RawStepKind::Streaming { body, outcome } => { + let snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status) + .unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + (OpenedStepKind::Streaming { body, outcome }, snapshot, false) + }, + }; + let request_snapshot = crate::Request { + method: filter_ctx.request.method.clone(), + uri: filter_ctx.request.uri.clone(), + headers: filter_ctx.request.headers.clone(), + }; + let continuation = StepResponseContinuation::capture( + Arc::clone(pipeline), + request_snapshot, + response_snapshot, + &mut filter_ctx, + completed, + step_deadline, + ); + Ok(OpenedStep { continuation, kind }) + } +} + +/// Convert transition-level transport metadata into completion-hook metadata. +fn stream_termination_cause(kind: config::TransportErrorKind) -> StreamTerminationCause { + match kind { + config::TransportErrorKind::AdmissionTimeout => StreamTerminationCause::AdmissionTimeout, + config::TransportErrorKind::CircuitOpen => StreamTerminationCause::CircuitOpen, + config::TransportErrorKind::Connect => StreamTerminationCause::Connect, + config::TransportErrorKind::Io => StreamTerminationCause::Io, + config::TransportErrorKind::DeadlineExceeded => StreamTerminationCause::DeadlineExceeded, + config::TransportErrorKind::ResponseTooLarge => StreamTerminationCause::ResponseTooLarge, + } +} diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs index 11042bbce..1aeb7c20d 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs @@ -6,11 +6,13 @@ //! The iterative request router runs step pipelines across multiple //! sub-requests. When a step's pipeline selects streaming mode, //! `on_request()` returns a terminal streaming response whose body -//! is owned by [`IrrStreamingBody`]. +//! is owned by [`IrrStreamingSession`]. //! -//! [`IrrStreamingBody`] pulls upstream chunks through the step's +//! [`IrrStreamingSession`] pulls upstream chunks through each step's //! response-body filters until the stream ends, then runs the step's -//! completion lifecycle exactly once (owned `end_of_stream` hook). +//! completion lifecycle exactly once and evaluates the next transition. +//! A matching `next` opens another step without recommitting downstream +//! response headers. //! //! [`StepResponseContinuation`] holds all state needed to run body //! filters after `on_request()` returns: the step pipeline, request @@ -18,17 +20,40 @@ //! and a completion guard. The continuation owns an `Arc` //! so the pipeline outlives the router filter. -use std::{any::Any, collections::HashMap}; +use std::{ + any::Any, + collections::{HashMap, VecDeque}, + sync::Arc, +}; use async_trait::async_trait; use bytes::Bytes; use praxis_core::subrequest::SubResponseBody; use crate::{ - FilterError, FilterPipeline, actions::StreamingResponseBody, extensions::RequestExtensions, - results::FilterResultSet, + FilterError, FilterPipeline, IterationState, NextIterationBody, StreamTermination, StreamTerminationCause, + actions::StreamingResponseBody, + context::PendingStreamChunks, + extensions::RequestExtensions, + results::{FilterResultSet, RetainedFilterResults}, }; +/// State made available after a step's completion hook has run. +pub(super) struct StepCompletion { + /// Shared request extensions after step completion. + pub(super) extensions: RequestExtensions, + /// Results retained across all step phases. + pub(super) filter_results: HashMap<&'static str, FilterResultSet>, + /// Optional request body for the next step. + pub(super) next_iteration_body: Option, + /// Bounded locally emitted chunks. + pub(super) pending_chunks: VecDeque, + /// Updated iteration state. + pub(super) state: IterationState, + /// Typed abnormal source termination, when present. + pub(super) termination: Option, +} + /// State continuation for streaming a step's response body. /// /// Owns the step pipeline, request/response snapshots, and all @@ -38,7 +63,7 @@ use crate::{ /// router's `on_request` has already completed. pub(super) struct StepResponseContinuation { /// Arc-wrapped step pipeline for executing response-body filters. - pub(super) pipeline: std::sync::Arc, + pub(super) pipeline: Arc, /// Snapshot of the step's request for filter context reconstruction. pub(super) request_snapshot: crate::Request, /// Snapshot of the step's response headers for filter context reconstruction. @@ -69,10 +94,87 @@ pub(super) struct StepResponseContinuation { pub(super) downstream_tls: bool, /// Start time of the containing client request. pub(super) request_start: std::time::Instant, + /// Absolute deadline shared by header and body processing for this step. + pub(super) step_deadline: std::time::Instant, /// Verified downstream mTLS identity. pub(super) peer_identity: Option, } +impl StepResponseContinuation { + /// Capture all owned step context after response headers have run. + #[expect( + clippy::too_many_arguments, + reason = "capture owns the complete response continuation boundary" + )] + pub(super) fn capture( + pipeline: Arc, + request_snapshot: crate::Request, + response_snapshot: crate::Response, + ctx: &mut crate::filter::HttpFilterContext<'_>, + completed: bool, + step_deadline: std::time::Instant, + ) -> Self { + Self { + pipeline, + request_snapshot, + response_snapshot, + extensions: std::mem::take(&mut ctx.extensions), + filter_state: std::mem::take(&mut ctx.filter_state), + filter_results: std::mem::take(&mut ctx.filter_results), + filter_metadata: std::mem::take(&mut ctx.filter_metadata), + structured_metadata: std::mem::take(&mut ctx.structured_metadata), + executed_filter_indices: std::mem::take(&mut ctx.executed_filter_indices), + body_done_indices: std::mem::take(&mut ctx.body_done_indices), + response_body_bytes: ctx.response_body_bytes, + response_body_mode: ctx.response_body_mode, + completed, + client_addr: ctx.client_addr, + downstream_tls: ctx.downstream_tls, + request_start: ctx.request_start, + step_deadline, + peer_identity: ctx.peer_identity.clone(), + } + } + + /// Recover only caller-owned extensions when an internal invariant fails. + pub(super) fn into_parent_extensions(mut self) -> RequestExtensions { + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions + } + + /// Consume the completed continuation into transition inputs. + pub(super) fn into_completion(mut self) -> Result { + let state = self + .extensions + .remove::() + .ok_or_else(|| -> FilterError { + "iterative_request_router: iteration state missing after step completion" + .to_owned() + .into() + })?; + let next_iteration_body = self.extensions.remove::().map(|body| body.0); + let pending_chunks = self + .extensions + .remove::() + .map_or_else(VecDeque::new, PendingStreamChunks::into_chunks); + let termination = self.extensions.remove::(); + let mut filter_results = self.extensions.remove::().unwrap_or_default().0; + filter_results.extend(self.filter_results); + Ok(StepCompletion { + extensions: self.extensions, + filter_results, + next_iteration_body, + pending_chunks, + state, + termination, + }) + } +} + /// Streaming body implementation for iterative request router steps. /// /// Pulls upstream chunks through the step's response-body filters, @@ -85,23 +187,44 @@ pub(super) struct StepResponseContinuation { /// pattern: `.take()` to move it out for cancellation. pub(super) struct IrrStreamingBody { /// Upstream streaming body handle. `None` after cancellation. - upstream: Option, + upstream: Option>, /// Owned state for running step response-body filters. continuation: StepResponseContinuation, /// Whether the stream has finished (EOF or error). finished: bool, + /// Completion-hook body output held until IRR selects a transition. + deferred_completion_output: Option, + /// Per-callback local output waiting to be pulled downstream. + pending_chunks: VecDeque, } impl IrrStreamingBody { /// Create a new streaming body for a step's response. - pub(super) fn new(upstream: SubResponseBody, continuation: StepResponseContinuation) -> Self { + pub(super) fn new(upstream: Box, continuation: StepResponseContinuation) -> Self { Self { upstream: Some(upstream), continuation, finished: false, + deferred_completion_output: None, + pending_chunks: VecDeque::new(), } } + /// Consume the body wrapper after EOF and recover its owned step state. + pub(super) fn into_continuation(self) -> StepResponseContinuation { + self.continuation + } + + /// Consume a finished body into its continuation and deferred output. + fn into_finished_parts(self) -> (StepResponseContinuation, Option) { + (self.continuation, self.deferred_completion_output) + } + + /// Exchange extensions with the outer protocol lifecycle. + fn exchange_extensions(&mut self, extensions: &mut RequestExtensions) { + std::mem::swap(&mut self.continuation.extensions, extensions); + } + /// Run the step's response-body filters on a single chunk. /// /// Reconstructs a temporary `HttpFilterContext` from the @@ -125,9 +248,9 @@ impl IrrStreamingBody { filter_metadata: std::mem::take(&mut cont.filter_metadata), filter_results: std::mem::take(&mut cont.filter_results), filter_state: std::mem::take(&mut cont.filter_state), - health_registry: None, + health_registry: cont.pipeline.health_registry(), id_generator: cont.pipeline.id_generator(), - kv_stores: None, + kv_stores: cont.pipeline.kv_stores(), metrics_route: None, peer_identity: cont.peer_identity.clone(), pre_read_mutations: Vec::new(), @@ -144,8 +267,8 @@ impl IrrStreamingBody { rewritten_path: None, selected_endpoint_index: None, structured_metadata: std::mem::take(&mut cont.structured_metadata), - subrequest_client: None, - subrequest_response_mode: crate::context::SubRequestResponseMode::Buffered, + subrequest_client: cont.pipeline.subrequest_client(), + subrequest_response_mode: crate::context::SubRequestResponseMode::Streaming, time_source: cont.pipeline.time_source(), upstream: None, }; @@ -196,18 +319,41 @@ impl IrrStreamingBody { fn handle_upstream_chunk(&mut self, chunk: Bytes) -> Result, FilterError> { let mut body = Some(chunk); self.run_step_body_filters(&mut body, false)?; - Ok(body) + let emitted = self + .continuation + .extensions + .get_mut::() + .map_or_else(VecDeque::new, PendingStreamChunks::drain_chunks); + self.pending_chunks.extend(emitted); + self.pending_chunks.extend(body.filter(|bytes| !bytes.is_empty())); + Ok(self.pending_chunks.pop_front()) + } + + /// Complete the step after a response-body filter failure. + async fn handle_filter_error(&mut self, error: FilterError) -> Result, FilterError> { + if let Some(upstream_body) = self.upstream.take() { + (*upstream_body).cancel().await; + } + self.continuation + .extensions + .insert(StreamTermination::new(StreamTerminationCause::Filter)); + let completion = self.complete_step().map_err(|completion_error| -> FilterError { + format!( + "iterative_request_router: response filter failed ({error}); completion also failed ({completion_error})" + ) + .into() + })?; + self.finished = true; + self.deferred_completion_output = self.handled_completion_output(completion); + Ok(None) } /// Handle upstream EOF. fn handle_upstream_eof(&mut self) -> Result, FilterError> { let completion = self.complete_step()?; self.finished = true; - if completion.as_ref().is_some_and(|b| !b.is_empty()) { - Ok(completion) - } else { - Ok(None) - } + self.deferred_completion_output = completion.filter(|bytes| !bytes.is_empty()); + Ok(None) } /// Handle an upstream error. @@ -215,17 +361,52 @@ impl IrrStreamingBody { &mut self, e: praxis_core::subrequest::SubRequestError, ) -> Result, FilterError> { - self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; } - Err(format!("iterative_request_router: upstream stream error: {e}").into()) + self.continuation + .extensions + .insert(StreamTermination::new(termination_cause(&e))); + let completion = self.complete_step()?; + self.finished = true; + self.deferred_completion_output = self.handled_completion_output(completion); + Ok(None) + } + + /// Expose an abnormal completion body only when a filter explicitly + /// converted the failure into a valid terminal sequence. + fn handled_completion_output(&self, completion: Option) -> Option { + self.continuation + .extensions + .get::() + .is_some_and(StreamTermination::is_handled) + .then_some(completion) + .flatten() + .filter(|bytes| !bytes.is_empty()) + } +} + +/// Map transport detail to the provider-neutral completion classification. +fn termination_cause(error: &praxis_core::subrequest::SubRequestError) -> StreamTerminationCause { + use praxis_core::subrequest::SubRequestError; + match error { + SubRequestError::AdmissionTimeout { .. } => StreamTerminationCause::AdmissionTimeout, + SubRequestError::CircuitOpen { .. } => StreamTerminationCause::CircuitOpen, + SubRequestError::Connect(_) => StreamTerminationCause::Connect, + SubRequestError::DeadlineExceeded => StreamTerminationCause::DeadlineExceeded, + SubRequestError::StreamIdleTimeout { .. } => StreamTerminationCause::IdleTimeout, + SubRequestError::ResponseTooLarge { .. } => StreamTerminationCause::ResponseTooLarge, + _ => StreamTerminationCause::Io, } } #[async_trait] impl StreamingResponseBody for IrrStreamingBody { + #[expect(clippy::too_many_lines, reason = "pull loop applies deadlines and completion state")] async fn next_chunk(&mut self) -> Result, FilterError> { + if let Some(chunk) = self.pending_chunks.pop_front() { + return Ok(Some(chunk)); + } if self.finished { return Ok(None); } @@ -235,11 +416,24 @@ impl StreamingResponseBody for IrrStreamingBody { "iterative_request_router: upstream already consumed".to_owned().into() })?; - match upstream.next_chunk().await { - Ok(Some(chunk)) => { - if let Some(bytes) = self.handle_upstream_chunk(chunk)? { - return Ok(Some(bytes)); - } + let remaining = self + .continuation + .step_deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_default(); + let next = if remaining.is_zero() { + Err(praxis_core::subrequest::SubRequestError::DeadlineExceeded) + } else { + tokio::time::timeout(remaining, upstream.next_chunk()) + .await + .unwrap_or(Err(praxis_core::subrequest::SubRequestError::DeadlineExceeded)) + }; + + match next { + Ok(Some(chunk)) => match self.handle_upstream_chunk(chunk) { + Ok(Some(bytes)) => return Ok(Some(bytes)), + Ok(None) => {}, + Err(error) => return Box::pin(self.handle_filter_error(error)).await, }, Ok(None) => return self.handle_upstream_eof(), Err(e) => return Box::pin(self.handle_upstream_error(e)).await, @@ -251,7 +445,7 @@ impl StreamingResponseBody for IrrStreamingBody { if !self.finished { self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; } self.complete_step()?; } @@ -262,8 +456,449 @@ impl StreamingResponseBody for IrrStreamingBody { if !self.finished { self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; } } } + + fn swap_extensions(&mut self, extensions: &mut RequestExtensions) { + self.exchange_extensions(extensions); + } +} + +/// A committed logical response that can span multiple IRR steps. +pub(super) struct IrrStreamingSession { + /// Active step body, when a streamed step is being consumed. + current: Option, + /// Outcome corresponding to `current`. + current_outcome: Option, + /// Request inherited or replaced for the current step. + current_request: crate::SubRequest, + /// Current configured step name. + current_step: Arc, + /// Whether the logical response reached a terminal state. + done: bool, + /// Unhandled failure returned after final pending chunks drain. + deferred_error: Option, + /// Cumulative downstream payload bytes produced by this session. + emitted_bytes: usize, + /// Extensions held between active steps. + extensions: Option, + /// Whether EOF follows the pending local output queue. + finish_after_pending: bool, + /// Retained-state and pending-output ceiling. + max_state_bytes: usize, + /// Optional logical streamed-response byte ceiling. + max_stream_response_bytes: Option, + /// Step selected after the current completion. + next_step: Option>, + /// Locally emitted or terminal buffered chunks awaiting delivery. + pending_chunks: VecDeque, + /// Reusable one-step executor. + runner: super::runner::IrrStepRunner, + /// Iteration state held between active steps. + state: Option, + /// Ordered transition rules for every named step. + step_transitions: HashMap, Vec>, +} + +impl IrrStreamingSession { + /// Create a logical session from the already opened first streamed step. + #[expect( + clippy::too_many_arguments, + reason = "session owns the complete logical response state" + )] + pub(super) fn new( + runner: super::runner::IrrStepRunner, + current_step: Arc, + current_request: crate::SubRequest, + outcome: super::StepOutcome, + body: Box, + continuation: StepResponseContinuation, + pending_chunks: VecDeque, + step_transitions: HashMap, Vec>, + max_state_bytes: usize, + max_stream_response_bytes: Option, + ) -> Self { + Self { + current: Some(IrrStreamingBody::new(body, continuation)), + current_outcome: Some(outcome), + current_request, + current_step, + done: false, + deferred_error: None, + emitted_bytes: 0, + extensions: None, + finish_after_pending: false, + max_state_bytes, + max_stream_response_bytes, + next_step: None, + pending_chunks, + runner, + state: None, + step_transitions, + } + } + + /// Account for one outgoing chunk against the logical byte ceiling. + fn checked_chunk(&mut self, chunk: Bytes) -> Result, FilterError> { + let total = self + .emitted_bytes + .checked_add(chunk.len()) + .ok_or_else(|| -> FilterError { "iterative_request_router: stream byte count overflow".into() })?; + if self.max_stream_response_bytes.is_some_and(|limit| total > limit) { + return Err("iterative_request_router: logical stream byte limit exceeded" + .to_owned() + .into()); + } + self.emitted_bytes = total; + Ok(Some(chunk)) + } + + /// Transition rules for the current step. + fn transitions(&self) -> &[super::config::StepTransition] { + self.step_transitions.get(&self.current_step).map_or(&[], Vec::as_slice) + } + + /// Persist a completed step and select the next session phase. + #[expect( + clippy::too_many_lines, + reason = "completion handles limits, failures, and transitions" + )] + fn apply_completion( + &mut self, + mut completion: StepCompletion, + outcome: &super::StepOutcome, + completion_output: Option, + terminal_body: Option, + ) -> Result<(), FilterError> { + completion.state.previous_response = Some(outcome.response.clone()); + completion.state.iteration += 1; + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + let transition = super::evaluate_transitions(self.transitions(), outcome, &completion.filter_results); + let abnormal_completion = completion.termination.is_some(); + let unhandled_termination = completion + .termination + .as_ref() + .is_some_and(|termination| !termination.is_handled()); + if unhandled_termination + && matches!( + transition, + super::TransitionResult::Done | super::TransitionResult::NoMatch + ) + { + let cause = completion.termination.as_ref().map(StreamTermination::cause); + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + self.deferred_error = + Some(format!("iterative_request_router: unhandled upstream stream termination: {cause:?}").into()); + return Ok(()); + } + let completion_output = (!abnormal_completion || !matches!(transition, super::TransitionResult::Next(_))) + .then_some(completion_output) + .flatten() + .filter(|body| !body.is_empty()); + if abnormal_completion && matches!(transition, super::TransitionResult::Next(_)) { + completion.pending_chunks.clear(); + } + let terminal_body = matches!( + transition, + super::TransitionResult::Done | super::TransitionResult::NoMatch + ) + .then_some(terminal_body) + .flatten() + .filter(|body| !body.is_empty()); + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks + .iter() + .chain(completion.pending_chunks.iter()) + .chain(completion_output.iter()) + .chain(terminal_body.iter()) + .map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + self.pending_chunks.extend(completion.pending_chunks); + self.pending_chunks.extend(completion_output); + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + match transition { + super::TransitionResult::Next(next) => { + let next_body = completion + .next_iteration_body + .unwrap_or_else(|| self.current_request.body.clone()); + self.current_request = crate::SubRequest { + method: self.current_request.method.clone(), + uri: self.current_request.uri.clone(), + headers: http::HeaderMap::new(), + body: next_body, + }; + self.next_step = Some(next); + }, + super::TransitionResult::Done | super::TransitionResult::NoMatch => { + self.pending_chunks.extend(terminal_body); + self.finish_after_pending = true; + }, + } + Ok(()) + } + + /// Consume the current body after clean or typed completion. + #[expect( + clippy::significant_drop_tightening, + reason = "consuming the body also consumes its continuation" + )] + fn finish_current(&mut self) -> Result<(), FilterError> { + let current = self + .current + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: current stream missing at EOF".into() })?; + let (continuation, completion_output) = current.into_finished_parts(); + let completion = continuation.into_completion()?; + let outcome = self + .current_outcome + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: current step outcome missing at EOF".into() })?; + self.apply_completion(completion, &outcome, completion_output, None) + } + + /// Open and classify the next selected step. + #[expect(clippy::too_many_lines, reason = "next step may fail over, buffer, or stream")] + #[expect( + clippy::significant_drop_tightening, + reason = "opened step is destructured across match arms" + )] + #[expect( + clippy::large_stack_frames, + reason = "opening a step reconstructs its filter context" + )] + async fn open_next(&mut self) -> Result<(), FilterError> { + let next = self + .next_step + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: next step missing".into() })?; + self.current_step = next; + let state = self.state.take().ok_or_else(|| -> FilterError { + "iterative_request_router: iteration state missing between steps".into() + })?; + if state.iteration >= state.max_iterations { + return Err("iterative_request_router: max iterations exhausted".to_owned().into()); + } + let extensions = self.extensions.take().unwrap_or_default(); + let opened = + match Box::pin( + self.runner + .open_step(&self.current_step, &self.current_request, &state, extensions), + ) + .await + { + Ok(opened) => opened, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; + let super::runner::OpenedStep { continuation, kind } = opened; + match kind { + super::runner::OpenedStepKind::Streaming { body, outcome } => { + if !super::streaming_transition_order_is_valid(self.transitions()) { + (*body).cancel().await; + self.extensions = Some(continuation.into_parent_extensions()); + self.state = Some(state); + return Err(format!( + "iterative_request_router: step '{}' selected streaming with interleaved transition phases", + self.current_step + ) + .into()); + } + match super::evaluate_header_transitions(self.transitions(), &outcome) { + super::TransitionResult::Next(next) => { + let mut skipped = IrrStreamingBody::new(body, continuation); + if let Err(error) = skipped.suppress().await { + self.extensions = Some(skipped.into_continuation().into_parent_extensions()); + self.state = Some(state); + return Err(error); + } + let mut completion = skipped.into_continuation().into_completion()?; + completion.state.previous_response = None; + completion.state.iteration += 1; + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + self.current_request = crate::SubRequest { + method: self.current_request.method.clone(), + uri: self.current_request.uri.clone(), + headers: http::HeaderMap::new(), + body: completion + .next_iteration_body + .unwrap_or_else(|| self.current_request.body.clone()), + }; + self.next_step = Some(next); + }, + super::TransitionResult::Done | super::TransitionResult::NoMatch => { + self.current = Some(IrrStreamingBody::new(body, continuation)); + self.current_outcome = Some(outcome); + }, + } + }, + super::runner::OpenedStepKind::Complete(outcome) => { + let completion = continuation.into_completion()?; + let abnormal_completion = completion.termination.is_some(); + let completion_output = abnormal_completion.then(|| outcome.response.body.clone()); + let terminal_body = (!abnormal_completion).then(|| outcome.response.body.clone()); + self.apply_completion(completion, &outcome, completion_output, terminal_body)?; + }, + } + Ok(()) + } +} + +/// Enforce the shared retained-state and pending-output ceiling using the +/// final state produced by the complete response-filter lifecycle. +pub(super) fn ensure_combined_retained_limit( + state_bytes: usize, + mut chunk_lengths: impl Iterator, + limit: usize, +) -> Result<(), FilterError> { + let retained = chunk_lengths.try_fold(state_bytes, |retained, chunk_len| { + retained + .checked_add(chunk_len) + .ok_or_else(|| -> FilterError { "iterative_request_router: retained state size overflow".into() }) + })?; + if retained > limit { + return Err("iterative_request_router: retained state limit exceeded" + .to_owned() + .into()); + } + Ok(()) +} + +#[async_trait] +impl StreamingResponseBody for IrrStreamingSession { + async fn next_chunk(&mut self) -> Result, FilterError> { + loop { + if let Some(chunk) = self.pending_chunks.pop_front() { + return self.checked_chunk(chunk); + } + if let Some(error) = self.deferred_error.take() { + self.done = true; + return Err(error); + } + if self.finish_after_pending || self.done { + self.done = true; + return Ok(None); + } + if let Some(current) = self.current.as_mut() { + if let Some(chunk) = current.next_chunk().await? { + return self.checked_chunk(chunk); + } + self.finish_current()?; + continue; + } + if self.next_step.is_some() { + Box::pin(self.open_next()).await?; + continue; + } + return Err("iterative_request_router: streaming session has no runnable phase" + .to_owned() + .into()); + } + } + + async fn suppress(&mut self) -> Result<(), FilterError> { + loop { + self.pending_chunks.clear(); + if let Some(error) = self.deferred_error.take() { + self.done = true; + return Err(error); + } + if self.finish_after_pending || self.done { + self.done = true; + return Ok(()); + } + if let Some(current) = self.current.as_mut() { + current.suppress().await?; + self.finish_current()?; + continue; + } + if self.next_step.is_some() { + Box::pin(self.open_next()).await?; + continue; + } + return Err( + "iterative_request_router: suppressed streaming session has no runnable phase" + .to_owned() + .into(), + ); + } + } + + async fn cancel(&mut self) { + if let Some(current) = self.current.as_mut() { + current.cancel().await; + } + self.current = None; + self.pending_chunks.clear(); + self.next_step = None; + self.done = true; + } + + fn swap_extensions(&mut self, extensions: &mut RequestExtensions) { + if let Some(current) = self.current.as_mut() { + current.swap_extensions(extensions); + } else if let Some(owned) = self.extensions.as_mut() { + std::mem::swap(owned, extensions); + } + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + + use super::ensure_combined_retained_limit; + + #[test] + fn completion_rechecks_final_state_with_pending_output() { + let chunks = [Bytes::from_static(b"123")]; + let result = ensure_combined_retained_limit(8, chunks.iter().map(Bytes::len), 10); + assert!( + result + .as_ref() + .is_err_and(|error| error.to_string().contains("retained state limit")), + "combined limit failure should identify retained state: {result:?}" + ); + } + + #[test] + fn completion_accepts_exact_combined_retained_limit() { + let chunks = [Bytes::from_static(b"12")]; + let result = ensure_combined_retained_limit(8, chunks.iter().map(Bytes::len), 10); + assert!( + result.is_ok(), + "final state and pending chunks should be accepted at the exact limit: {result:?}" + ); + } } diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs index 739743203..f4c179db0 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs @@ -422,6 +422,28 @@ steps: assert!(err.to_string().contains("max_state_bytes")); } +#[test] +fn rejects_zero_max_stream_response_bytes() { + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: step1 +max_stream_response_bytes: 0 +steps: + - name: step1 + filters: + - filter: static_response + status: 200 +", + ) + .unwrap(); + let cfg: IterativeRequestRouterConfig = parse_filter_config("iterative_request_router", &yaml).unwrap(); + let error = config::validate(&cfg).unwrap_err(); + assert!( + error.to_string().contains("max_stream_response_bytes"), + "zero logical stream limit should be rejected: {error}" + ); +} + #[test] fn accepts_multi_step_config() { let yaml: serde_yaml::Value = serde_yaml::from_str( @@ -784,7 +806,7 @@ fn classify_invalid_request_falls_through_to_io() { #[test] #[expect(clippy::too_many_lines, reason = "YAML config literal")] -fn rejects_filter_result_transition_on_streaming_capable_step() { +fn accepts_completion_result_transition_on_streaming_capable_step() { let mut registry = crate::FilterRegistry::with_builtins(); registry .register( @@ -814,16 +836,48 @@ steps: .unwrap(); let result = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry); assert!( - result.is_err(), - "filter/key/value transition should be rejected for streaming step" + result.is_ok(), + "filter/key/value transition should be evaluated after streaming EOF" + ); +} + +#[test] +#[expect(clippy::too_many_lines, reason = "YAML transition-order fixture")] +fn rejects_header_failover_after_stream_completion_rule() { + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StreamingSelectorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: s +steps: + - name: s + filters: + - filter: test_streaming_selector + - filter: static_response + status: 200 + on_result: + - filter: test_streaming_selector + key: action + value: loop + next: s + - status: [503] + next: s + - default: true + done: true +", + ) + .unwrap(); + let result = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry); + let error = result.err().expect("interleaved streaming transitions must fail"); + assert!( + error.to_string().contains("must precede completion rules"), + "validation error should explain phase ordering: {error}" ); - if let Err(err) = result { - let msg = err.to_string(); - assert!( - msg.contains("streaming") && msg.contains("filter"), - "error should mention streaming and filter: {msg}" - ); - } } #[test] @@ -907,6 +961,27 @@ fn streaming_runtime_guard_accepts_default_transition() { struct StreamingSelectorFilter; +struct UndeclaredStreamingSelectorFilter; + +#[derive(Debug, PartialEq, Eq)] +struct ParentExtension(&'static str); + +struct StepErrorFilter; + +#[async_trait::async_trait] +impl crate::HttpFilter for StepErrorFilter { + fn name(&self) -> &'static str { + "test_step_error" + } + + async fn on_request( + &self, + _ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + Err("nested step failure".to_owned().into()) + } +} + #[async_trait::async_trait] impl crate::HttpFilter for StreamingSelectorFilter { fn name(&self) -> &'static str { @@ -926,6 +1001,21 @@ impl crate::HttpFilter for StreamingSelectorFilter { } } +#[async_trait::async_trait] +impl crate::HttpFilter for UndeclaredStreamingSelectorFilter { + fn name(&self) -> &'static str { + "test_undeclared_streaming_selector" + } + + async fn on_request( + &self, + ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + ctx.set_subrequest_response_mode(crate::SubRequestResponseMode::Streaming); + Ok(crate::FilterAction::Continue) + } +} + /// Build a default-done transition. fn make_default_done() -> config::StepTransition { config::StepTransition { @@ -1598,6 +1688,134 @@ async fn on_request_no_connector() { assert!(result.is_err(), "no connector should return error"); } +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "nested pipeline setup and ownership assertions")] +async fn step_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_step_error", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StepErrorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: failing +steps: + - name: failing + filters: + - filter: test_step_error + on_result: + - default: true + done: true +", + ) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!(result.is_err(), "the nested step error should propagate"); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a nested step error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "runtime guard setup and ownership assertions")] +async fn streaming_runtime_guard_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_undeclared_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(UndeclaredStreamingSelectorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_undeclared_streaming_selector + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true + - status: [502] + next: stream +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!( + result + .as_ref() + .is_err_and(|error| error.to_string().contains("interleaved transition phases")), + "the runtime guard should reject a misdeclared streaming selector: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the runtime guard must restore outer request extensions" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +fn start_unit_stream_backend() -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + let _sent = stream.write_all( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n1\r\nx\r\n0\r\n\r\n", + ); + }); + port +} + // --------------------------------------------------------------------------- // on_request_body - not end_of_stream // --------------------------------------------------------------------------- @@ -2073,6 +2291,15 @@ fn listener_response_limit_clamps_router_limit() { assert_eq!(super::effective_response_limit(4, crate::BodyMode::Stream), 4); } +#[test] +fn streaming_transport_uses_only_listener_limit() { + assert_eq!( + super::streaming_transport_limit(crate::BodyMode::SizeLimit { max_bytes: 4 }), + Some(4) + ); + assert_eq!(super::streaming_transport_limit(crate::BodyMode::Stream), None); +} + #[test] fn strip_request_framing_headers_removes_stale_lengths() { let mut headers = HeaderMap::new(); diff --git a/filter/src/context.rs b/filter/src/context.rs index fd5bcf367..9190e6ce6 100644 --- a/filter/src/context.rs +++ b/filter/src/context.rs @@ -3,7 +3,14 @@ //! Transport-agnostic HTTP request/response metadata and per-request filter context. -use std::{any::Any, borrow::Cow, collections::HashMap, net::IpAddr, sync::Arc, time::Instant}; +use std::{ + any::Any, + borrow::Cow, + collections::{HashMap, VecDeque}, + net::IpAddr, + sync::Arc, + time::Instant, +}; use http::{HeaderMap, Method, StatusCode, Uri, header::HeaderName}; use praxis_core::{ @@ -11,7 +18,42 @@ use praxis_core::{ }; use praxis_tls::TlsPeerIdentity; -use crate::{body::BodyMode, extensions::RequestExtensions, pipeline::body::merge_body_mode, results::FilterResultSet}; +use crate::{ + FilterError, IterationState, body::BodyMode, extensions::RequestExtensions, pipeline::body::merge_body_mode, + results::FilterResultSet, +}; + +/// Bounded opaque chunks emitted by filters while IRR owns a logical stream. +pub(crate) struct PendingStreamChunks { + /// FIFO ordering of locally emitted opaque chunks. + chunks: VecDeque, + /// Combined iteration-state and pending-output ceiling. + max_retained_bytes: usize, + /// Bytes currently retained in `chunks`. + retained_bytes: usize, +} + +impl PendingStreamChunks { + /// Create an empty bounded pending-output queue. + pub(crate) fn new(max_retained_bytes: usize) -> Self { + Self { + chunks: VecDeque::new(), + max_retained_bytes, + retained_bytes: 0, + } + } + + /// Consume the accounting wrapper and return its FIFO queue. + pub(crate) fn into_chunks(self) -> VecDeque { + self.chunks + } + + /// Drain queued chunks and reset their retained-byte accounting. + pub(crate) fn drain_chunks(&mut self) -> VecDeque { + self.retained_bytes = 0; + std::mem::take(&mut self.chunks) + } +} // ----------------------------------------------------------------------------- // Constants @@ -100,6 +142,53 @@ pub enum SubRequestResponseMode { Streaming, } +/// Provider-neutral reason an owned streaming source terminated abnormally. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StreamTerminationCause { + /// Admission capacity was not acquired in time. + AdmissionTimeout, + /// The peer circuit breaker rejected the exchange. + CircuitOpen, + /// Connection establishment failed. + Connect, + /// The overall or per-step deadline expired. + DeadlineExceeded, + /// The upstream produced no bytes within its idle budget. + IdleTimeout, + /// Transport I/O failed after connection establishment. + Io, + /// A response-body filter failed after commitment. + Filter, + /// A configured response byte ceiling was exceeded. + ResponseTooLarge, +} + +/// Typed abnormal termination exposed to streaming completion filters. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StreamTermination { + /// Framework-level failure classification. + cause: StreamTerminationCause, + /// Set by a completion filter that produced a valid terminal sequence. + handled: bool, +} + +impl StreamTermination { + /// Create an unhandled termination value. + pub(crate) fn new(cause: StreamTerminationCause) -> Self { + Self { cause, handled: false } + } + + /// The provider-neutral termination classification. + pub fn cause(&self) -> StreamTerminationCause { + self.cause + } + + /// Whether a completion filter converted the failure into final bytes. + pub fn is_handled(&self) -> bool { + self.handled + } +} + // ----------------------------------------------------------------------------- // HttpFilterContext // ----------------------------------------------------------------------------- @@ -358,6 +447,72 @@ impl HttpFilterContext<'_> { self.subrequest_response_mode = mode; } + /// Emit an opaque response chunk into the IRR-owned logical stream. + /// + /// Emission is available only inside an iterative session. A buffered + /// intermediate step may retain chunks for a later streaming step; if the + /// iteration instead terminates with a buffered response, IRR rejects the + /// pending chunks. Chunks emitted by a streaming response-body callback are + /// delivered in FIFO order before that callback's body output. Pending + /// chunks are bounded together with retained [`IterationState`]; exceeding + /// that bound returns an error and enqueues nothing. Praxis does not inspect + /// or reinterpret the bytes. + /// + /// # Errors + /// + /// Returns an error outside an IRR step or when the retained-state limit + /// would be exceeded. + pub fn emit_stream_chunk(&mut self, bytes: bytes::Bytes) -> Result<(), FilterError> { + let state_bytes = self + .extensions + .get::() + .map_or(0, IterationState::retained_bytes); + let pending = self + .extensions + .get_mut::() + .ok_or_else(|| -> FilterError { + "stream chunk emission is only available inside iterative_request_router" + .to_owned() + .into() + })?; + let retained = state_bytes + .checked_add(pending.retained_bytes) + .and_then(|value| value.checked_add(bytes.len())) + .ok_or_else(|| -> FilterError { "stream chunk retained-state size overflow".to_owned().into() })?; + if retained > pending.max_retained_bytes { + return Err(format!( + "stream chunk emission exceeds retained-state limit ({} > {})", + retained, pending.max_retained_bytes + ) + .into()); + } + pending.retained_bytes += bytes.len(); + pending.chunks.push_back(bytes); + Ok(()) + } + + /// Abnormal source termination visible during a streaming completion hook. + pub fn stream_termination(&self) -> Option<&StreamTermination> { + self.extensions.get::() + } + + /// Mark the current abnormal stream termination as converted to a valid + /// provider-specific terminal sequence by this filter. + /// + /// Returns `false` when the step is completing normally. + pub fn mark_stream_termination_handled(&mut self) -> bool { + let Some(termination) = self.extensions.get_mut::() else { + return false; + }; + termination.handled = true; + true + } + + /// Enable bounded local stream emission for an IRR step. + pub(crate) fn enable_stream_chunk_emission(&mut self, max_retained_bytes: usize) { + self.extensions.insert(PendingStreamChunks::new(max_retained_bytes)); + } + /// Read a durable metadata value by key. pub fn get_metadata(&self, key: &str) -> Option<&str> { self.filter_metadata.get(key).map(String::as_str) @@ -1656,4 +1811,51 @@ mod tests { "merge should drop new key past limit" ); } + + #[test] + fn stream_chunk_emission_is_bounded() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + ctx.enable_stream_chunk_emission(5); + ctx.emit_stream_chunk(bytes::Bytes::from_static(b"12345")).unwrap(); + let error = ctx.emit_stream_chunk(bytes::Bytes::from_static(b"6")).unwrap_err(); + assert!( + error.to_string().contains("retained-state limit"), + "overflow should report the retained-state limit: {error}" + ); + } + + #[test] + fn stream_chunk_emission_requires_irr_session() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + let error = ctx.emit_stream_chunk(bytes::Bytes::from_static(b"event")).unwrap_err(); + assert!( + error + .to_string() + .contains("only available inside iterative_request_router"), + "out-of-session emission should be rejected: {error}" + ); + } + + #[test] + fn stream_termination_requires_explicit_handling() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + ctx.extensions + .insert(StreamTermination::new(StreamTerminationCause::IdleTimeout)); + assert_eq!( + ctx.stream_termination().map(StreamTermination::cause), + Some(StreamTerminationCause::IdleTimeout), + "completion filters should see the typed cause" + ); + assert!( + ctx.mark_stream_termination_handled(), + "an abnormal completion should be markable as handled" + ); + assert!( + ctx.stream_termination().is_some_and(StreamTermination::is_handled), + "handled state should persist for the session" + ); + } } diff --git a/filter/src/filter.rs b/filter/src/filter.rs index a8672b20d..a57b27b14 100644 --- a/filter/src/filter.rs +++ b/filter/src/filter.rs @@ -14,6 +14,7 @@ use crate::{ actions::FilterAction, body::{BodyAccess, BodyMode}, builtins::http::payload_processing::compression_config::CompressionConfig, + pipeline::FilterPipeline, }; // ----------------------------------------------------------------------------- @@ -118,6 +119,14 @@ pub trait HttpFilter: Send + Sync { false } + /// Visit pipelines owned by this filter. + /// + /// Framework filters that embed nested pipelines override this hook so + /// server-provided runtime resources are propagated into those pipelines. + /// Ordinary filters should use the default no-op implementation. + #[doc(hidden)] + fn visit_nested_pipelines(&mut self, _visitor: &mut dyn FnMut(&mut FilterPipeline)) {} + /// Called for each response, in reverse pipeline order. /// /// Default: [`FilterAction::Continue`] diff --git a/filter/src/lib.rs b/filter/src/lib.rs index 8896a6d7b..59bcf6f54 100644 --- a/filter/src/lib.rs +++ b/filter/src/lib.rs @@ -38,7 +38,8 @@ pub use builtins::{ }; pub use condition::{should_execute, should_execute_response, should_execute_response_ref}; pub use context::{ - HttpFilterContext, PendingHeaderResult, Request, Response, SubRequestResponseMode, TrustedHeaderMutation, + HttpFilterContext, PendingHeaderResult, Request, Response, StreamTermination, StreamTerminationCause, + SubRequestResponseMode, TrustedHeaderMutation, }; pub use error_response::{ ErrorResponseContext, ErrorResponseFormatter, ErrorResponseFormatterHandle, FormattedErrorResponse, diff --git a/filter/src/pipeline/build.rs b/filter/src/pipeline/build.rs index 409368d46..a600a3509 100644 --- a/filter/src/pipeline/build.rs +++ b/filter/src/pipeline/build.rs @@ -95,19 +95,24 @@ impl FilterPipeline { let body_capabilities = compute_body_capabilities(&filters); let compression = extract_compression_config(&filters); let may_select_streaming_subrequest_response = filters_may_select_streaming_subrequest_response(&filters); - Self { + let id_generator = Arc::new(IdGenerator::new()); + let time_source: Arc = Arc::new(SystemTimeSource); + let mut pipeline = Self { body_capabilities, compression, filters, health_registry: None, - id_generator: Arc::new(IdGenerator::new()), + id_generator: Arc::clone(&id_generator), kv_stores: None, pipeline_extensions: Vec::new(), record_filter_duration_metrics: false, subrequest_client: None, may_select_streaming_subrequest_response, - time_source: Arc::new(SystemTimeSource), - } + time_source: Arc::clone(&time_source), + }; + pipeline.set_id_generator(id_generator); + pipeline.set_time_source(time_source); + pipeline } /// Validate the pipeline for structural misconfigurations that diff --git a/filter/src/pipeline/mod.rs b/filter/src/pipeline/mod.rs index 44a3ea0d5..1b9c00321 100644 --- a/filter/src/pipeline/mod.rs +++ b/filter/src/pipeline/mod.rs @@ -161,6 +161,8 @@ impl FilterPipeline { max_response: Option, allow_unbounded: bool, ) -> Result<(), FilterError> { + self.apply_nested_body_limits(max_request, max_response, allow_unbounded)?; + if let Some(ceiling) = max_request { self.body_capabilities.request_body_mode = clamp_body_mode( self.body_capabilities.request_body_mode, @@ -193,6 +195,27 @@ impl FilterPipeline { Ok(()) } + /// Apply the listener ceilings recursively to nested pipelines. + fn apply_nested_body_limits( + &mut self, + max_request: Option, + max_response: Option, + allow_unbounded: bool, + ) -> Result<(), FilterError> { + let mut nested_error = None; + self.visit_nested_pipelines(&mut |pipeline| { + if nested_error.is_none() + && let Err(error) = pipeline.apply_body_limits(max_request, max_response, allow_unbounded) + { + nested_error = Some(error); + } + }); + if let Some(error) = nested_error { + return Err(error); + } + Ok(()) + } + /// Pre-computed body processing capabilities for this pipeline. pub fn body_capabilities(&self) -> &BodyCapabilities { &self.body_capabilities @@ -220,11 +243,13 @@ impl FilterPipeline { /// Set the shared [`HealthRegistry`] for this pipeline. pub fn set_health_registry(&mut self, registry: HealthRegistry) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_health_registry(Arc::clone(®istry))); self.health_registry = Some(registry); } /// Enable or disable recording of per-filter duration metrics. pub fn set_record_filter_duration_metrics(&mut self, enabled: bool) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_record_filter_duration_metrics(enabled)); self.record_filter_duration_metrics = enabled; } @@ -245,6 +270,7 @@ impl FilterPipeline { /// Override the [`IdGenerator`] for this pipeline. pub fn set_id_generator(&mut self, generator: Arc) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_id_generator(Arc::clone(&generator))); self.id_generator = generator; } @@ -255,6 +281,7 @@ impl FilterPipeline { /// Set the shared [`KvStoreRegistry`] for this pipeline. pub fn set_kv_stores(&mut self, stores: KvStoreRegistry) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_kv_stores(stores.clone())); self.kv_stores = Some(stores); } @@ -273,6 +300,7 @@ impl FilterPipeline { /// /// [`SubRequestClient`]: praxis_core::subrequest::SubRequestClient pub fn set_subrequest_client(&mut self, client: praxis_core::subrequest::SubRequestClient) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_subrequest_client(client.clone())); self.subrequest_client = Some(client); } @@ -306,6 +334,7 @@ impl FilterPipeline { /// Override the [`TimeSource`] for this pipeline. pub fn set_time_source(&mut self, source: Arc) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_time_source(Arc::clone(&source))); self.time_source = source; } @@ -324,6 +353,15 @@ impl FilterPipeline { } } } + + /// Apply a mutation to every pipeline directly embedded by a filter. + fn visit_nested_pipelines(&mut self, visitor: &mut dyn FnMut(&mut FilterPipeline)) { + for pf in &mut self.filters { + if let crate::any_filter::AnyFilter::Http(filter) = &mut pf.filter { + filter.visit_nested_pipelines(visitor); + } + } + } } // ----------------------------------------------------------------------------- diff --git a/protocol/src/http/pingora/handler/request_filter/mod.rs b/protocol/src/http/pingora/handler/request_filter/mod.rs index 0ea05c4df..b37a3a3ec 100644 --- a/protocol/src/http/pingora/handler/request_filter/mod.rs +++ b/protocol/src/http/pingora/handler/request_filter/mod.rs @@ -509,6 +509,7 @@ async fn run_streaming_terminal_response( headers, body: mut streaming_body, } = terminal; + streaming_body.swap_extensions(&mut ctx.extensions); let mut resp = match prepare_terminal_response(pipeline, ctx, status, headers).await { Ok(resp) => resp, Err(rejection) => { @@ -517,9 +518,11 @@ async fn run_streaming_terminal_response( return; }, }; + streaming_body.swap_extensions(&mut ctx.extensions); if matches!(ctx.response_body_mode, BodyMode::StreamBuffer { .. }) { error!("streaming terminal response is incompatible with StreamBuffer response mode"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; send_rejection(session, Rejection::status(500)).await; return; @@ -538,20 +541,40 @@ async fn run_streaming_terminal_response( let http_version = session.req_header().version; prepare_streaming_headers(&mut resp, false, false, http_version); let Some(header) = build_streaming_terminal_header(&resp) else { + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; send_rejection(session, Rejection::status(500)).await; return; }; if let Err(e) = session.write_response_header(Box::new(header), false).await { debug!(error = %e, "failed to write streaming terminal response header"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; return; } + // A client may validly half-close its HTTP/1 write side after sending the + // request while continuing to read the response. Keep FIN distinct from a + // real disconnect; resets and failed response writes still abort promptly. + session.as_downstream_mut().set_abort_on_close(false); loop { - match streaming_body.next_chunk().await { + let source_result = tokio::select! { + result = streaming_body.next_chunk() => Some(result), + downstream = session.as_downstream_mut().read_body_or_idle(true) => { + debug!(?downstream, "downstream disconnected while terminal stream source was pending"); + None + }, + }; + let Some(source_result) = source_result else { + streaming_body.swap_extensions(&mut ctx.extensions); + streaming_body.cancel().await; + session.as_downstream_mut().shutdown().await; + return; + }; + match source_result { Ok(Some(chunk)) => { + streaming_body.swap_extensions(&mut ctx.extensions); let mut body = Some(chunk); if run_parent_terminal_body_filters(pipeline, ctx, &resp, &mut body, false).is_err() || streaming_size_limit_exceeded(ctx) @@ -560,14 +583,19 @@ async fn run_streaming_terminal_response( session.as_downstream_mut().shutdown().await; return; } + streaming_body.swap_extensions(&mut ctx.extensions); if let Err(e) = session.write_response_body(body, false).await { debug!(error = %e, "failed to write streaming terminal response body"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; return; } }, Ok(None) => { + // Restore the default before a clean keep-alive session can be reused. + session.as_downstream_mut().set_abort_on_close(true); + streaming_body.swap_extensions(&mut ctx.extensions); let mut completion_body = None; if run_parent_terminal_body_filters(pipeline, ctx, &resp, &mut completion_body, true).is_err() { streaming_body.cancel().await; @@ -581,6 +609,7 @@ async fn run_streaming_terminal_response( return; }, Err(e) => { + streaming_body.swap_extensions(&mut ctx.extensions); warn!(error = %e, "streaming terminal response source failed after commitment"); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; @@ -609,6 +638,7 @@ async fn suppress_streaming_terminal_response( send_rejection(session, Rejection::status(500)).await; return; } + streaming_body.swap_extensions(&mut ctx.extensions); let mut completion_body = None; if let Err(rejection) = run_parent_terminal_body_filters(pipeline, ctx, resp, &mut completion_body, true) { streaming_body.cancel().await; diff --git a/tests/integration/tests/suite/iterative_request_router.rs b/tests/integration/tests/suite/iterative_request_router.rs index f0d04dea7..78327eef6 100644 --- a/tests/integration/tests/suite/iterative_request_router.rs +++ b/tests/integration/tests/suite/iterative_request_router.rs @@ -12,6 +12,7 @@ use bytes::Bytes; use praxis_core::config::Config; use praxis_filter::{ BodyAccess, BodyMode, FilterAction, FilterError, FilterFactory, FilterRegistry, HttpFilter, HttpFilterContext, + StreamTerminationCause, }; use praxis_test_utils::{ Backend, free_port, http_get, http_post, http_send, json_post, parse_body, parse_header, parse_status, @@ -1271,6 +1272,45 @@ steps: assert_eq!(status, 413, "iteration state above max_state_bytes should be rejected"); } +#[test] +fn terminal_response_rechecks_retained_state_limit() { + let backend = Backend::fixed(&"x".repeat(512)).start_with_shutdown(); + let proxy_port = free_port(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: terminal +max_state_bytes: 256 +steps: + - name: terminal + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy(&config); + + let (status, _) = http_get(proxy.addr(), "/", None); + + assert_eq!( + status, 413, + "the retained terminal response must remain inside max_state_bytes" + ); +} + #[test] fn combined_status_and_filter_result_transition() { let model = start_stateful_backend(vec![(200, "final-answer".to_owned())]); @@ -1636,6 +1676,95 @@ steps: ); } +#[test] +fn nested_response_filter_reserved_header_is_stripped() { + let backend = Backend::fixed("ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = nested_header_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: primary +steps: + - name: primary + filters: + - filter: test_nested_response_headers + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send(proxy.addr(), &http_get_raw("/")); + + assert_eq!(parse_status(&raw), 200, "buffered step should succeed"); + assert!( + raw.contains("x-step-visible: true"), + "ordinary nested response-header mutations should reach the client" + ); + assert!( + !raw.contains("x-praxis-step-private"), + "reserved nested response headers must be stripped after filters" + ); +} + +#[test] +fn nested_response_filter_reserved_header_is_stripped_on_transport_error() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = nested_header_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: primary +steps: + - name: primary + filters: + - filter: test_nested_response_headers + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send(proxy.addr(), &http_get_raw("/")); + + assert_eq!(parse_status(&raw), 502, "transport failure should remain synthetic 502"); + assert!( + raw.contains("x-step-visible: true"), + "ordinary synthetic-response mutations should reach the client" + ); + assert!( + !raw.contains("x-praxis-step-private"), + "reserved synthetic-response headers must be stripped after filters" + ); +} + #[test] fn response_hooks_execute_once() { let backend_port = start_backend("once"); @@ -1928,6 +2057,195 @@ steps: assert!(body.contains("data: chunk3"), "last chunk should arrive: {body}"); } +#[test] +fn streaming_body_callback_emission_precedes_its_body_output() { + let backend = Backend::chunked(vec!["upstream".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: test_per_chunk_emission + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "streaming response should succeed"); + assert_eq!( + parse_body(&raw), + "local|upstream", + "a callback's local emissions must not be delayed until stream completion" + ); +} + +#[test] +fn streaming_completion_emission_precedes_completion_body() { + let backend = Backend::chunked(vec!["upstream|".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: test_completion_emission_and_body + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "streaming response should succeed"); + assert_eq!( + parse_body(&raw), + "upstream|emitted|completion-body", + "completion emissions must precede the same callback's body output" + ); +} + +#[test] +fn handled_initial_transport_completion_preserves_emission_order() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: unavailable +steps: + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_handled_termination_order + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 502, + "the handled connect failure should retain its status" + ); + assert_eq!( + parse_body(&raw), + "emitted|completion-body", + "pre-commit abnormal completion must preserve callback emission order" + ); +} + +#[test] +fn streaming_step_deadline_includes_header_time() { + let backend_port = start_split_delay_chunked_backend( + std::time::Duration::from_millis(450), + std::time::Duration::from_millis(450), + ); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +timeout_ms: 800 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: slow + - filter: load_balancer + clusters: + - name: slow + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "headers should arrive before the deadline"); + assert_eq!( + parse_body(&raw), + "", + "body time must use only the deadline remaining after headers" + ); +} + #[test] fn streaming_header_failover_cancels_unread_body() { let failing = Backend::status(503, "fail").start_with_shutdown(); @@ -1943,6 +2261,7 @@ steps: - name: primary filters: - filter: test_streaming_selector + - filter: test_stream_completion - filter: router routes: - path_prefix: "/" @@ -2027,15 +2346,822 @@ steps: ); } -// --------------------------------------------------------------------------- -// Test-Only Streaming Selector Filter -// --------------------------------------------------------------------------- - -struct IntegrationStreamingSelectorFilter; - -#[async_trait::async_trait] -impl HttpFilter for IntegrationStreamingSelectorFilter { - fn name(&self) -> &'static str { +#[test] +fn streaming_completion_transition_resumes_second_step_in_same_response() { + let first = Backend::chunked(vec!["first-a|".to_owned(), "first-b|".to_owned()]).start_with_shutdown(); + let second = Backend::chunked(vec!["second-a|".to_owned(), "second-b".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_stream_response_bytes: 65536 +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - default: true + done: true + - name: second + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "logical stream should retain initial status"); + assert_eq!( + parse_body(&raw), + "first-a|first-b|between|second-a|second-b", + "both upstream streams and the local completion chunk should share one ordered response" + ); +} + +#[test] +fn streaming_handoff_checks_pending_chunks_with_updated_state() { + let first = Backend::fixed("first").start_with_shutdown(); + let second = Backend::chunked(vec!["second".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + registry + .register( + "test_fill_iteration_state", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(FillIterationStateFilter)))), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_state_bytes: 2048 +steps: + - name: first + filters: + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - name: second + filters: + - filter: test_always_streaming + - filter: test_fill_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 413, + "updated state and carried completion chunks must share one retained-byte ceiling" + ); +} + +#[test] +fn buffered_transition_pending_limit_returns_413() { + let first = Backend::fixed("first").start_with_shutdown(); + let second = Backend::fixed("second").start_with_shutdown(); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + registry + .register( + "test_fill_iteration_state", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(FillIterationStateFilter)))), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_state_bytes: 2048 +steps: + - name: first + filters: + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - name: second + filters: + - filter: test_fill_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + next: final + - name: final + filters: + - filter: static_response + status: 200 + body: unreachable + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 413, + "pre-commit pending-output overflow must remain a bounded-request rejection" + ); +} + +#[test] +fn streaming_head_suppression_still_runs_completion_transition() { + let first = Backend::chunked(vec!["first".to_owned()]).start_with_shutdown(); + let second = Backend::chunked(vec!["second".to_owned()]).start_with_shutdown(); + let second_step_calls = Arc::new(AtomicUsize::new(0)); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + let counter = Arc::clone(&second_step_calls); + registry + .register( + "test_request_counter", + FilterFactory::Http(Arc::new(move |_| { + Ok(Box::new(IntegrationRequestCounterFilter { + count: Arc::clone(&counter), + })) + })), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - default: true + done: true + - name: second + filters: + - filter: test_request_counter + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "HEAD / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "HEAD should retain the committed logical status" + ); + assert_eq!(parse_body(&raw), "", "HEAD must not expose logical stream bytes"); + assert_eq!( + second_step_calls.load(Ordering::SeqCst), + 1, + "suppression must still evaluate completion transitions and execute the next step" + ); +} + +#[test] +fn buffered_response_limit_does_not_cap_streaming_transport() { + let backend = Backend::chunked(vec!["12345".to_owned(), "67890".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +max_response_bytes: 5 +max_stream_response_bytes: 20 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: stream + - filter: load_balancer + clusters: + - name: stream + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "streaming response should commit normally"); + assert_eq!( + parse_body(&raw), + "1234567890", + "the buffered per-step limit must not truncate a live stream" + ); +} + +#[test] +fn streaming_logical_byte_limit_terminates_after_exact_boundary() { + let backend = + Backend::chunked(vec!["12345".to_owned(), "67890".to_owned(), "overflow".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +max_stream_response_bytes: 10 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: stream + - filter: load_balancer + clusters: + - name: stream + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "headers are committed before the late limit failure" + ); + assert_eq!( + parse_body(&raw), + "1234567890", + "the chunk crossing the logical byte ceiling must not be exposed" + ); +} + +#[test] +fn streaming_resumed_transport_failure_can_complete_with_typed_termination() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_termination + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"#, + first.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "the first step should commit the logical response" + ); + assert_eq!( + parse_body(&raw), + "first|between|handled-connect", + "the resumed step completion hook should classify and handle the connect failure" + ); +} + +#[test] +fn streaming_resumed_transport_failure_can_fail_over() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - origin: transport + transport_error: connect + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "the first step should commit the response"); + assert_eq!( + parse_body(&raw), + "first|between|fallback-ok", + "the failed resumed step should transition without leaking its completion output" + ); +} + +#[test] +fn streaming_mid_body_failure_defers_completion_output_until_transition() { + let incomplete_port = start_incomplete_chunked_backend(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: incomplete +steps: + - name: incomplete + filters: + - filter: test_always_streaming + - filter: test_stream_termination + - filter: router + routes: + - path_prefix: "/" + cluster: incomplete + - filter: load_balancer + clusters: + - name: incomplete + endpoints: ["127.0.0.1:{incomplete_port}"] + on_result: + - filter: test_stream_termination + key: action + value: next + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "initial headers should commit the response"); + assert_eq!( + parse_body(&raw), + "first|xfallback-ok", + "failure completion bytes must be suppressed when the transition selects a fallback" + ); +} + +#[test] +fn streaming_resumed_header_failover_clears_previous_response() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let failing = Backend::status(503, "skip-me").start_with_shutdown(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: failing + - default: true + done: true + - name: failing + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: failing + - filter: load_balancer + clusters: + - name: failing + endpoints: ["127.0.0.1:{}"] + on_result: + - status: [503] + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: test_previous_response_absent + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + failing.port(), + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "the first step should commit the response"); + assert_eq!( + parse_body(&raw), + "first|between|fallback-ok", + "the header-failover target must not inherit the previous completed response" + ); +} + +#[test] +fn streaming_unhandled_transport_failure_discards_completion_output() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"#, + first.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "the first step should commit the logical response" + ); + assert_eq!( + parse_body(&raw), + "first|between|", + "an unhandled failure must not expose completion chunks from the failed step" + ); +} + +#[test] +fn streaming_unhandled_initial_transport_failure_discards_completion_output() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: unavailable +steps: + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 502, + "an unhandled initial connect failure should remain a 502" + ); + assert_eq!( + parse_body(&raw), + "", + "an unhandled pre-commit failure must not expose completion output" + ); +} + +// --------------------------------------------------------------------------- +// Test-Only Streaming Selector Filter +// --------------------------------------------------------------------------- + +struct IntegrationStreamingSelectorFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamingSelectorFilter { + fn name(&self) -> &'static str { "test_streaming_selector" } @@ -2056,8 +3182,276 @@ impl HttpFilter for IntegrationStreamingSelectorFilter { } } +struct IntegrationAlwaysStreamingFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationAlwaysStreamingFilter { + fn name(&self) -> &'static str { + "test_always_streaming" + } + + fn may_select_streaming_subrequest_response(&self) -> bool { + true + } + + async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + ctx.set_subrequest_response_mode(praxis_filter::SubRequestResponseMode::Streaming); + Ok(FilterAction::Continue) + } +} + +struct IntegrationRequestCounterFilter { + count: Arc, +} + +struct FillIterationStateFilter; + +#[async_trait::async_trait] +impl HttpFilter for FillIterationStateFilter { + fn name(&self) -> &'static str { + "test_fill_iteration_state" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + async fn on_response(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + const LIMIT: usize = 2_048; + const KEY: &str = "fill"; + let state = ctx + .extensions + .get_mut::() + .ok_or_else(|| -> FilterError { "iteration state missing in test filter".into() })?; + let fill_bytes = LIMIT + .checked_sub(state.retained_bytes().saturating_add(KEY.len())) + .ok_or_else(|| -> FilterError { "iteration state already exceeds test limit".into() })?; + state + .accumulator + .insert(KEY.to_owned(), Bytes::from(vec![b'x'; fill_bytes])); + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationRequestCounterFilter { + fn name(&self) -> &'static str { + "test_request_counter" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + self.count.fetch_add(1, Ordering::SeqCst); + Ok(FilterAction::Continue) + } +} + +struct IntegrationStreamCompletionFilter; + +struct IntegrationPerChunkEmissionFilter; + +struct IntegrationCompletionEmissionAndBodyFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationCompletionEmissionAndBodyFilter { + fn name(&self) -> &'static str { + "test_completion_emission_and_body" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadWrite + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream { + ctx.emit_stream_chunk(Bytes::from_static(b"emitted|"))?; + *body = Some(Bytes::from_static(b"completion-body")); + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationHandledTerminationOrderFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationHandledTerminationOrderFilter { + fn name(&self) -> &'static str { + "test_handled_termination_order" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadWrite + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream && ctx.stream_termination().is_some() { + assert!( + ctx.mark_stream_termination_handled(), + "the termination should be available to the completion hook" + ); + ctx.emit_stream_chunk(Bytes::from_static(b"emitted|"))?; + *body = Some(Bytes::from_static(b"completion-body")); + } + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationPerChunkEmissionFilter { + fn name(&self) -> &'static str { + "test_per_chunk_emission" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if !end_of_stream { + ctx.emit_stream_chunk(Bytes::from_static(b"local|"))?; + } + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamCompletionFilter { + fn name(&self) -> &'static str { + "test_stream_completion" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream { + ctx.filter_results + .entry("test_stream_completion") + .or_default() + .set("action", "next")?; + ctx.emit_stream_chunk(Bytes::from_static(b"between|"))?; + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationStreamTerminationFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamTerminationFilter { + fn name(&self) -> &'static str { + "test_stream_termination" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream && let Some(cause) = ctx.stream_termination().map(|termination| termination.cause()) { + assert!( + ctx.mark_stream_termination_handled(), + "the abnormal completion should remain available while the hook runs" + ); + ctx.filter_results + .entry("test_stream_termination") + .or_default() + .set("action", "next")?; + let output = match cause { + StreamTerminationCause::Connect => Bytes::from_static(b"handled-connect"), + _ => Bytes::from_static(b"handled-stream-error|"), + }; + ctx.emit_stream_chunk(output)?; + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationPreviousResponseAbsentFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationPreviousResponseAbsentFilter { + fn name(&self) -> &'static str { + "test_previous_response_absent" + } + + async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + let previous_response_present = ctx + .extensions + .get::() + .is_some_and(|state| state.previous_response.is_some()); + if previous_response_present { + return Ok(FilterAction::Reject(praxis_filter::Rejection::status(500))); + } + Ok(FilterAction::Continue) + } +} + fn streaming_registry() -> FilterRegistry { let mut registry = FilterRegistry::with_builtins(); + registry + .register( + "test_completion_emission_and_body", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationCompletionEmissionAndBodyFilter)))), + ) + .unwrap(); + registry + .register( + "test_handled_termination_order", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationHandledTerminationOrderFilter)))), + ) + .unwrap(); + registry + .register( + "test_per_chunk_emission", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationPerChunkEmissionFilter)))), + ) + .unwrap(); registry .register( "test_streaming_selector", @@ -2065,6 +3459,84 @@ fn streaming_registry() -> FilterRegistry { ) .unwrap(); registry + .register( + "test_always_streaming", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationAlwaysStreamingFilter)))), + ) + .unwrap(); + registry + .register( + "test_stream_completion", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationStreamCompletionFilter)))), + ) + .unwrap(); + registry + .register( + "test_stream_termination", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationStreamTerminationFilter)))), + ) + .unwrap(); + registry + .register( + "test_previous_response_absent", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationPreviousResponseAbsentFilter)))), + ) + .unwrap(); + registry +} + +fn nested_header_registry() -> FilterRegistry { + let mut registry = FilterRegistry::with_builtins(); + registry + .register( + "test_nested_response_headers", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(NestedResponseHeadersFilter)))), + ) + .unwrap(); + registry +} + +#[expect( + clippy::disallowed_methods, + reason = "blocking backend runs on a dedicated test thread" +)] +fn start_split_delay_chunked_backend(header_delay: std::time::Duration, body_delay: std::time::Duration) -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + std::thread::sleep(header_delay); + let _sent = stream.write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"); + let _flushed = stream.flush(); + std::thread::sleep(body_delay); + let _sent = stream.write_all(b"4\r\nlate\r\n0\r\n\r\n"); + }); + port +} + +fn start_incomplete_chunked_backend() -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + let _sent = stream.write_all( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n6\r\nfirst|\r\n5\r\nx", + ); + let _flushed = stream.flush(); + }); + port } // --------------------------------------------------------------------------- @@ -2121,6 +3593,33 @@ impl HttpFilter for ResponseProbeFilter { struct ResponseTaggerFilter; +struct NestedResponseHeadersFilter; + +#[async_trait::async_trait] +impl HttpFilter for NestedResponseHeadersFilter { + fn name(&self) -> &'static str { + "test_nested_response_headers" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + async fn on_response(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + let response = ctx + .response_header + .as_mut() + .ok_or_else(|| -> FilterError { "response header missing in test filter".into() })?; + response + .headers + .insert("x-step-visible", http::HeaderValue::from_static("true")); + response + .headers + .insert("x-praxis-step-private", http::HeaderValue::from_static("secret")); + Ok(FilterAction::Continue) + } +} + #[async_trait::async_trait] impl HttpFilter for ResponseTaggerFilter { fn name(&self) -> &'static str { diff --git a/tests/integration/tests/suite/streaming_terminal_response.rs b/tests/integration/tests/suite/streaming_terminal_response.rs index 3666800c1..259c65f12 100644 --- a/tests/integration/tests/suite/streaming_terminal_response.rs +++ b/tests/integration/tests/suite/streaming_terminal_response.rs @@ -11,7 +11,7 @@ use std::{ io::{Read as _, Write as _}, - net::TcpStream, + net::{Shutdown, TcpStream}, sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -336,6 +336,36 @@ fn streaming_h11_chunked_framing() { ); } +#[test] +fn streaming_h11_write_half_close_still_receives_response() { + let backend = start_echo_backend(); + let proxy_port = free_port(); + let config = Config::from_yaml(&custom_filter_yaml(proxy_port, backend.port(), "multi_chunk_streaming")).unwrap(); + let registry = registry_with("multi_chunk_streaming", || Box::new(MultiChunkStreamingFilter)); + let proxy = start_proxy_with_registry(&config, ®istry); + + let mut stream = TcpStream::connect(proxy.addr()).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .unwrap(); + stream.shutdown(Shutdown::Write).unwrap(); + let mut raw = Vec::new(); + stream.read_to_end(&mut raw).unwrap(); + let raw = String::from_utf8(raw).unwrap(); + + assert_eq!( + parse_status(&raw), + 200, + "a valid write half-close must preserve the response" + ); + assert_eq!( + parse_body(&raw), + "chunk1chunk2chunk3", + "the full stream should remain readable after the client half-closes writes" + ); +} + #[test] fn streaming_h10_close_delimited() { let backend = start_echo_backend(); From 322c67e78cfd5bc20d631f7fc5d82b9924f00ea3 Mon Sep 17 00:00:00 2001 From: Shane Utt Date: Mon, 17 Aug 2026 15:14:21 -0400 Subject: [PATCH 2/6] fix(Tests): wait for metrics Fixes a testing flake when the metrics wouldn't respond quickly enough. Signed-off-by: Shane Utt --- tests/integration/tests/suite/prometheus_metrics.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/tests/suite/prometheus_metrics.rs b/tests/integration/tests/suite/prometheus_metrics.rs index 6f4256be2..3774078f5 100644 --- a/tests/integration/tests/suite/prometheus_metrics.rs +++ b/tests/integration/tests/suite/prometheus_metrics.rs @@ -132,7 +132,12 @@ fn metrics_overload_rejects_listener_connections() { held.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); held.write_all(b"GET /api/ HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") .unwrap(); - std::thread::sleep(Duration::from_millis(200)); + + wait_for_metric( + &admin, + "praxis_connections_active", + Duration::from_secs(2), + ); let raw = http_send( proxy.addr(), From 75ba179c8b21050d0969ce7baf117cf760174b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 17 Aug 2026 21:28:14 +0200 Subject: [PATCH 3/6] fix(filter): preserve IRR streaming lifecycle state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the per-step tracing span around nested filter and transport execution so logs retain their step and iteration context. Return completion failures with recoverable parent extensions and restore them across buffered and committed-streaming error paths. Add regression coverage for both lifecycle shapes. Document the RAII cleanup contract when downstream disconnect races an in-flight streaming step. Signed-off-by: Sébastien Han --- .../iterative_request_router/mod.rs | 27 ++- .../iterative_request_router/runner.rs | 11 +- .../iterative_request_router/streaming.rs | 60 ++++-- .../iterative_request_router/tests.rs | 179 ++++++++++++++++++ .../pingora/handler/request_filter/mod.rs | 5 + 5 files changed, 266 insertions(+), 16 deletions(-) diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs index b9d5a8e8a..ab192ecab 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs @@ -482,7 +482,14 @@ impl IterativeRequestRouterFilter { ctx.extensions = skipped.into_continuation().into_parent_extensions(); return Err(error); } - let mut completion = skipped.into_continuation().into_completion()?; + let mut completion = match skipped.into_continuation().into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, + }; completion.state.previous_response = None; completion.state.iteration += 1; if completion.state.retained_bytes() > self.max_state_bytes { @@ -520,7 +527,14 @@ impl IterativeRequestRouterFilter { .is_err() { (*body).cancel().await; - let completion = continuation.into_completion()?; + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, + }; ctx.extensions = completion.extensions; return Ok(FilterAction::Reject(Rejection::status(413))); } @@ -547,7 +561,14 @@ impl IterativeRequestRouterFilter { } }, OpenedStepKind::Complete(mut outcome) => { - let completion = continuation.into_completion()?; + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, + }; let abnormal_stream_completion = completion.termination.is_some(); let handled_abnormal_stream_completion = completion .termination diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs index 959a0e8dc..a6c450c6d 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs @@ -15,7 +15,7 @@ use std::{ use bytes::Bytes; use http::HeaderMap; use praxis_core::subrequest::{FrameworkHeaders, StreamLimits}; -use tracing::warn; +use tracing::{Instrument as _, warn}; use super::{ StepOutcome, SubPipelineRuntimeResources, apply_pre_read_header_mutations, apply_request_header_mutations, @@ -221,6 +221,12 @@ impl IrrStepRunner { let in_transport = Arc::new(AtomicBool::new(false)); let in_transport_inner = Arc::clone(&in_transport); + let step_span = tracing::info_span!( + "iterative_subrequest", + step = current_step.as_ref(), + iteration = state.iteration, + ); + let timed: Result, tokio::time::error::Elapsed> = tokio::time::timeout(step_budget, async { let mut request_body = Some(current_request.body.clone()); @@ -465,7 +471,8 @@ impl IrrStepRunner { Ok(RawStepKind::Complete(StepOutcome { response, origin, transport_error })) }, } - }) + } + .instrument(step_span)) .await; let mut raw = match timed { diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs index 1aeb7c20d..5aae3151e 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs @@ -54,6 +54,21 @@ pub(super) struct StepCompletion { pub(super) termination: Option, } +/// A completion conversion failure together with recoverable parent state. +pub(super) struct StepCompletionError { + /// Underlying lifecycle error. + error: FilterError, + /// Parent-owned extensions recovered from the failed continuation. + extensions: RequestExtensions, +} + +impl StepCompletionError { + /// Split the error from the extensions its caller must restore. + pub(super) fn into_parts(self) -> (FilterError, RequestExtensions) { + (self.error, self.extensions) + } +} + /// State continuation for streaming a step's response body. /// /// Owns the step pipeline, request/response snapshots, and all @@ -147,15 +162,15 @@ impl StepResponseContinuation { } /// Consume the completed continuation into transition inputs. - pub(super) fn into_completion(mut self) -> Result { - let state = self - .extensions - .remove::() - .ok_or_else(|| -> FilterError { - "iterative_request_router: iteration state missing after step completion" + pub(super) fn into_completion(mut self) -> Result { + let Some(state) = self.extensions.remove::() else { + return Err(StepCompletionError { + error: "iterative_request_router: iteration state missing after step completion" .to_owned() - .into() - })?; + .into(), + extensions: self.into_parent_extensions(), + }); + }; let next_iteration_body = self.extensions.remove::().map(|body| body.0); let pending_chunks = self .extensions @@ -666,7 +681,14 @@ impl IrrStreamingSession { .take() .ok_or_else(|| -> FilterError { "iterative_request_router: current stream missing at EOF".into() })?; let (continuation, completion_output) = current.into_finished_parts(); - let completion = continuation.into_completion()?; + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + return Err(error); + }, + }; let outcome = self .current_outcome .take() @@ -733,7 +755,15 @@ impl IrrStreamingSession { self.state = Some(state); return Err(error); } - let mut completion = skipped.into_continuation().into_completion()?; + let mut completion = match skipped.into_continuation().into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; completion.state.previous_response = None; completion.state.iteration += 1; if let Err(error) = ensure_combined_retained_limit( @@ -764,7 +794,15 @@ impl IrrStreamingSession { } }, super::runner::OpenedStepKind::Complete(outcome) => { - let completion = continuation.into_completion()?; + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; let abnormal_completion = completion.termination.is_some(); let completion_output = abnormal_completion.then(|| outcome.response.body.clone()); let terminal_body = (!abnormal_completion).then(|| outcome.response.body.clone()); diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs index f4c179db0..609386b40 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs @@ -968,6 +968,8 @@ struct ParentExtension(&'static str); struct StepErrorFilter; +struct RemoveIterationStateFilter; + #[async_trait::async_trait] impl crate::HttpFilter for StepErrorFilter { fn name(&self) -> &'static str { @@ -982,6 +984,34 @@ impl crate::HttpFilter for StepErrorFilter { } } +#[async_trait::async_trait] +impl crate::HttpFilter for RemoveIterationStateFilter { + fn name(&self) -> &'static str { + "test_remove_iteration_state" + } + + async fn on_request( + &self, + _ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + Ok(crate::FilterAction::Continue) + } + + fn response_body_access(&self) -> crate::BodyAccess { + crate::BodyAccess::ReadOnly + } + + fn on_response_body( + &self, + ctx: &mut crate::HttpFilterContext<'_>, + _body: &mut Option, + _end_of_stream: bool, + ) -> Result { + ctx.extensions.remove::(); + Ok(crate::FilterAction::Continue) + } +} + #[async_trait::async_trait] impl crate::HttpFilter for StreamingSelectorFilter { fn name(&self) -> &'static str { @@ -1735,6 +1765,155 @@ steps: ); } +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "nested pipeline setup and ownership assertions")] +async fn completion_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_remove_iteration_state", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(RemoveIterationStateFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: completion_error +steps: + - name: completion_error + filters: + - filter: test_remove_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!( + result.as_ref().is_err_and(|error| error + .to_string() + .contains("iteration state missing after step completion")), + "missing completion state should propagate an invariant error: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a completion conversion error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "streaming pipeline setup and ownership assertions")] +async fn streaming_completion_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StreamingSelectorFilter)))), + ) + .unwrap(); + registry + .register( + "test_remove_iteration_state", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(RemoveIterationStateFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: completion_error +steps: + - name: completion_error + filters: + - filter: test_streaming_selector + - filter: test_remove_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let action = filter.on_request(&mut ctx).await.unwrap(); + let action_debug = format!("{action:?}"); + let terminal = match action { + crate::FilterAction::StreamingTerminalResponse(terminal) => Some(terminal), + _ => None, + }; + assert!( + terminal.is_some(), + "streaming selector should return a terminal stream: {action_debug}" + ); + let Some(mut terminal) = terminal else { + return; + }; + let first = terminal.body.next_chunk().await.unwrap(); + assert_eq!( + first, + Some(bytes::Bytes::from_static(b"x")), + "upstream chunk should arrive first" + ); + let result = terminal.body.next_chunk().await; + terminal.body.swap_extensions(&mut ctx.extensions); + + assert!( + result.as_ref().is_err_and(|error| error + .to_string() + .contains("iteration state missing after step completion")), + "missing streaming completion state should propagate an invariant error: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a streaming completion error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + #[tokio::test] #[expect(clippy::too_many_lines, reason = "runtime guard setup and ownership assertions")] async fn streaming_runtime_guard_restores_parent_request_extensions() { diff --git a/protocol/src/http/pingora/handler/request_filter/mod.rs b/protocol/src/http/pingora/handler/request_filter/mod.rs index b37a3a3ec..4af403227 100644 --- a/protocol/src/http/pingora/handler/request_filter/mod.rs +++ b/protocol/src/http/pingora/handler/request_filter/mod.rs @@ -559,6 +559,11 @@ async fn run_streaming_terminal_response( session.as_downstream_mut().set_abort_on_close(false); loop { + // Cancellation contract: dropping an in-flight `next_chunk()` also + // drops any partially opened step resources. A constructed + // `SubResponseBody` schedules protocol-aware cleanup from `Drop`; + // earlier transport state releases its permit, circuit guard, and + // session through ordinary RAII ownership. let source_result = tokio::select! { result = streaming_body.next_chunk() => Some(result), downstream = session.as_downstream_mut().read_body_or_idle(true) => { From 8ab2e362e71d426160fb0d7c6213223cf1e9c0b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 17 Aug 2026 21:44:35 +0200 Subject: [PATCH 4/6] fix(tests): adopt upstream wait_for_metric removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge with upstream/main incorrectly kept the old multi-line wait_for_metric call that upstream replaced with thread::sleep, causing a nightly rustfmt check failure in CI. Signed-off-by: Sébastien Han --- tests/integration/tests/suite/prometheus_metrics.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/integration/tests/suite/prometheus_metrics.rs b/tests/integration/tests/suite/prometheus_metrics.rs index 3774078f5..6f4256be2 100644 --- a/tests/integration/tests/suite/prometheus_metrics.rs +++ b/tests/integration/tests/suite/prometheus_metrics.rs @@ -132,12 +132,7 @@ fn metrics_overload_rejects_listener_connections() { held.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); held.write_all(b"GET /api/ HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") .unwrap(); - - wait_for_metric( - &admin, - "praxis_connections_active", - Duration::from_secs(2), - ); + std::thread::sleep(Duration::from_millis(200)); let raw = http_send( proxy.addr(), From 1423d96665bf9103b864881f41d7355d2e44e03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 21 Aug 2026 16:45:28 +0200 Subject: [PATCH 5/6] fix(filter): log buffered transport failures in IRR step runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming transport path logged failures with warn! but the buffered path silently discarded both build_peer and execute errors. Add matching warn! calls so production debugging has parity across both transport modes. Signed-off-by: Sébastien Han --- .../iterative_request_router/runner.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs index 6436c1578..0ec7c1d77 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs @@ -411,6 +411,7 @@ impl IrrStepRunner { Ok(response) => (response, config::ResponseOrigin::Upstream, None), Err(error) => { let (status, kind) = classify_transport_failure(&error); + warn!(step = current_step.as_ref(), %error, status, "IRR buffered transport failure"); ( SubResponse { status, headers: HeaderMap::new(), body: Bytes::new() }, config::ResponseOrigin::Transport, @@ -418,11 +419,14 @@ impl IrrStepRunner { ) }, }, - Err(_) => ( - SubResponse { status: 502, headers: HeaderMap::new(), body: Bytes::new() }, - config::ResponseOrigin::Transport, - Some(config::TransportErrorKind::Connect), - ), + Err(error) => { + warn!(step = current_step.as_ref(), %error, status = 502_u16, "IRR buffered transport failure"); + ( + SubResponse { status: 502, headers: HeaderMap::new(), body: Bytes::new() }, + config::ResponseOrigin::Transport, + Some(config::TransportErrorKind::Connect), + ) + }, }; in_transport_inner.store(false, Ordering::Release); sanitize_subresponse_headers(&mut response.headers); From 185e71b4ad51c4e68184e2dc84cf3b1a13829df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 21 Aug 2026 17:04:35 +0200 Subject: [PATCH 6/6] fix(tests): adapt IRR streaming guards to upstream renames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream renamed `has_body_dependent_transitions` to `streaming_transition_order_is_valid` with new semantics (interleaved transition phases) and shortened the StreamBuffer error message. Update test names, YAML configs, and assert strings accordingly. Signed-off-by: Sébastien Han --- .../iterative_request_router/tests.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs index 9bd1c547e..54b8f0551 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs @@ -3736,7 +3736,7 @@ fn sneaky_registry() -> crate::FilterRegistry { } #[tokio::test] -async fn runtime_guard_rejects_streaming_with_body_dependent_transitions() { +async fn runtime_guard_rejects_streaming_with_interleaved_transition_phases() { let (addr, backend) = spawn_raw_backend("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n").await; let yaml = format!( " @@ -3747,12 +3747,10 @@ steps: - filter: test_sneaky_streaming {} on_result: - - filter: router - key: matched - value: \"true\" - done: true - default: true done: true + - status: [502] + next: s ", routed_step_yaml(addr) ); @@ -3766,8 +3764,8 @@ steps: backend.abort(); assert!( - err.to_string().contains("body-dependent transitions"), - "runtime streaming with body-dependent transitions must error: {err}" + err.to_string().contains("interleaved transition phases"), + "runtime streaming with interleaved transitions must error: {err}" ); } @@ -3799,7 +3797,7 @@ steps: backend.abort(); assert!( - err.to_string().contains("StreamBuffer response body mode"), + err.to_string().contains("StreamBuffer response mode"), "runtime streaming with StreamBuffer response mode must error: {err}" ); }