diff --git a/docs/filters/http/traffic_management/iterative_request_router.md b/docs/filters/http/traffic_management/iterative_request_router.md index c7326f97d..fb20b1244 100644 --- a/docs/filters/http/traffic_management/iterative_request_router.md +++ b/docs/filters/http/traffic_management/iterative_request_router.md @@ -9,6 +9,8 @@ Framework-level filter for iterative sub-request execution. Holds named steps, each backed by a pre-built sub-pipeline. During request processing, runs an iteration loop: execute each step's request filters, make the HTTP call via Pingora's `Connector`, execute its response filters, evaluate transition rules, and continue or return the final response. +Streaming steps remain pull-based. Header-safe failover rules run before any bytes are exposed; all other `on_result` rules run after clean EOF and may resume another step inside the same committed downstream response. + ## Configuration | Field | Type | Required | Description | @@ -16,6 +18,7 @@ Holds named steps, each backed by a pre-built sub-pipeline. During request proce | `initial_step` | string | yes | Name of the first step to execute. | | `max_iterations` | integer | no | Maximum iterations before aborting (default 10, max 100). | | `max_response_bytes` | integer | no | Maximum response body bytes per sub-request. | +| `max_stream_response_bytes` | integer | no | Optional cumulative byte ceiling for one logical streamed response. This is intentionally distinct from buffered per-step response limits. | | `max_state_bytes` | integer | no | Maximum accumulated iteration state bytes. | | `step_timeout_ms` | integer | no | Per-step timeout in milliseconds. Defaults to `timeout_ms`. | | `steps` | StepConfig[] | yes | Named steps, each with filters and transition rules. | @@ -37,7 +40,7 @@ Holds named steps, each backed by a pre-built sub-pipeline. During request proce | `steps[].filters[].name` | string | no | Optional user-assigned name for this filter entry. Used as a rejoin target by branch chains. | | `steps[].filters[].response_conditions` | ResponseCondition[] | no | Ordered conditions that gate whether this filter runs on responses. Evaluated against the upstream response (status, headers). Empty means the filter always runs on responses. | | `steps[].filters[].failure_mode` | `closed` \| `open` | no | Per-filter failure behaviour (`open` or `closed`). | -| `steps[].on_result` | StepTransition[] | no | Transition rules evaluated after the sub-request response. First match wins. | +| `steps[].on_result` | StepTransition[] | no | Transition rules evaluated in order. Streaming header-safe failovers run before body exposure; remaining rules run after step completion. | | `steps[].on_result[].default` | bool | no | If true, this is the default (always-match) rule. | | `steps[].on_result[].filter` | string | no | Filter name whose results to check. | | `steps[].on_result[].key` | string | no | Result key to match. | diff --git a/filter/src/actions.rs b/filter/src/actions.rs index 9ac3fb4a7..93296cfdb 100644 --- a/filter/src/actions.rs +++ b/filter/src/actions.rs @@ -8,7 +8,7 @@ use std::fmt; use async_trait::async_trait; use bytes::Bytes; -use crate::FilterError; +use crate::{FilterError, RequestExtensions}; // ----------------------------------------------------------------------------- // Streaming terminal response @@ -37,6 +37,15 @@ pub trait StreamingResponseBody: Send + 'static { /// /// This operation must be idempotent. async fn cancel(&mut self); + + /// Exchange request extensions with the protocol lifecycle owner. + /// + /// Most streaming bodies do not own filter extensions and use this + /// default no-op. Iterative sessions override it so the same extension + /// set can move between step filters and parent response filters without + /// cloning type-erased values. + #[doc(hidden)] + fn swap_extensions(&mut self, _extensions: &mut RequestExtensions) {} } /// A terminal response whose body is delivered incrementally. diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs index 553035809..d946a4023 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/config.rs @@ -74,6 +74,11 @@ pub(crate) struct IterativeRequestRouterConfig { #[serde(default = "default_max_response_bytes")] pub(crate) max_response_bytes: usize, + /// Optional cumulative byte ceiling for one logical streamed response. + /// This is intentionally distinct from buffered per-step response limits. + #[serde(default)] + pub(crate) max_stream_response_bytes: Option, + /// Maximum accumulated iteration state bytes. #[serde(default = "default_max_state_bytes")] pub(crate) max_state_bytes: usize, @@ -95,7 +100,7 @@ pub(crate) struct IterativeRequestRouterConfig { } /// A named step within the iterative router. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct StepConfig { /// Step name (must be unique within the router). @@ -104,14 +109,14 @@ pub(crate) struct StepConfig { /// Filters to execute for this step's sub-request. pub(crate) filters: Vec, - /// Transition rules evaluated after the sub-request - /// response. First match wins. + /// Transition rules evaluated in order. Streaming header-safe failovers + /// run before body exposure; remaining rules run after step completion. #[serde(default)] pub(crate) on_result: Vec, } /// A transition rule evaluated after a step completes. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct StepTransition { /// If true, this is the default (always-match) rule. @@ -227,6 +232,14 @@ pub(crate) fn validate(cfg: &IterativeRequestRouterConfig) -> Result<(), FilterE .into()); } + if cfg.max_stream_response_bytes == Some(0) { + return Err( + "iterative_request_router: max_stream_response_bytes must be > 0 when configured" + .to_owned() + .into(), + ); + } + if cfg.max_response_bytes == 0 { return Err("iterative_request_router: max_response_bytes must be > 0" .to_owned() diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs index eb9225a01..77669ce09 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/mod.rs @@ -18,15 +18,15 @@ //! //! When a step's filters select [`SubRequestResponseMode::Streaming`], //! IRR dispatches via `send_streaming()` instead of the buffered -//! `execute()` path. Response-header filters run before transition -//! evaluation; only status, origin, transport error, and default -//! predicates are available. Once a streaming response is committed -//! downstream, no failover or replacement response can occur. +//! `execute()` path. Header-safe failover transitions run before that +//! step exposes bytes. After clean EOF and response-body completion, +//! ordinary `on_result` rules may resume another step inside the same +//! committed downstream response. //! -//! Transitions that depend on filter results (filter/key/value -//! predicates) or `BodyMode::StreamBuffer` response mode are -//! rejected at config validation time for streaming-capable pipelines, -//! with a runtime guard for dynamically registered filters. +//! Header-safe failovers must precede completion-dependent transitions. +//! `BodyMode::StreamBuffer` remains incompatible with streaming-capable +//! step pipelines. Once the logical response is committed, failures can +//! only terminate its stream; they cannot replace its HTTP response. //! //! # Position requirement //! @@ -38,6 +38,7 @@ //! See proposal 00786 for the full design rationale. mod config; +mod runner; mod streaming; #[cfg(test)] #[expect(clippy::allow_attributes, reason = "blanket test suppressions")] @@ -51,11 +52,8 @@ mod streaming; mod tests; use std::{ - collections::HashMap, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, + collections::{HashMap, VecDeque}, + sync::Arc, time::{Duration, Instant}, }; @@ -63,21 +61,20 @@ use async_trait::async_trait; use bytes::Bytes; use http::HeaderMap; use pingora_core::upstreams::peer::HttpPeer; -use praxis_core::subrequest::{FrameworkHeaders, StreamLimits}; use tracing::{debug, info, warn}; use self::{ config::IterativeRequestRouterConfig, - streaming::{IrrStreamingBody, StepResponseContinuation}, + runner::{IrrStepRunner, OpenedStepKind, StepRuntime}, + streaming::{IrrStreamingBody, IrrStreamingSession, ensure_combined_retained_limit}, }; use crate::{ - FilterEntry, FilterError, FilterPipeline, FilterRegistry, IterationState, NextIterationBody, SubRequest, + FilterEntry, FilterError, FilterPipeline, FilterRegistry, IterationState, StreamTermination, SubRequest, SubRequestResponseMode, SubResponse, - actions::{FilterAction, Rejection, StreamingTerminalResponse, TerminalResponse}, + actions::{FilterAction, Rejection, StreamingResponseBody as _, StreamingTerminalResponse, TerminalResponse}, factory::parse_filter_config, filter::{HttpFilter, HttpFilterContext}, pipeline::subrequest::DEPTH_HEADER, - results::RetainedFilterResults, }; // --------------------------------------------------------------------------- @@ -93,12 +90,35 @@ const STREAMING_IDLE_TIMEOUT: Duration = Duration::from_secs(30); /// Whether transitions depend on response body content (filter result /// predicates or body-dependent metadata). +#[cfg(test)] pub(super) fn has_body_dependent_transitions(transitions: &[config::StepTransition]) -> bool { transitions .iter() .any(|t| t.filter.is_some() || t.key.is_some() || t.value.is_some()) } +/// Whether a transition can safely fail over before exposing a streamed body. +fn is_header_safe_failover(transition: &config::StepTransition) -> bool { + transition.next.is_some() + && !transition.default + && transition.filter.is_none() + && transition.key.is_none() + && transition.value.is_none() +} + +/// Whether the header-safe failover prefix is followed only by completion rules. +pub(super) fn streaming_transition_order_is_valid(transitions: &[config::StepTransition]) -> bool { + let mut completion_seen = false; + transitions.iter().all(|transition| { + if is_header_safe_failover(transition) { + !completion_seen + } else { + completion_seen = true; + true + } + }) +} + // --------------------------------------------------------------------------- // IterativeRequestRouterFilter // --------------------------------------------------------------------------- @@ -111,6 +131,10 @@ pub(super) fn has_body_dependent_transitions(transitions: &[config::StepTransiti /// `Connector`, execute its response filters, evaluate transition /// rules, and continue or return the final response. /// +/// Streaming steps remain pull-based. Header-safe failover rules run before +/// any bytes are exposed; all other `on_result` rules run after clean EOF and +/// may resume another step inside the same committed downstream response. +/// /// # YAML configuration /// /// ```yaml @@ -140,6 +164,9 @@ pub struct IterativeRequestRouterFilter { /// Maximum response body bytes per sub-request. max_response_bytes: usize, + /// Optional cumulative byte ceiling for a logical streamed response. + max_stream_response_bytes: Option, + /// Maximum accumulated state bytes. max_state_bytes: usize, @@ -222,13 +249,21 @@ impl IterativeRequestRouterFilter { continue; } if let Some(transitions) = step_transitions.get(name) { - for (i, t) in transitions.iter().enumerate() { - if t.filter.is_some() || t.key.is_some() || t.value.is_some() { + for (i, transition) in transitions.iter().enumerate() { + if is_header_safe_failover(transition) && !streaming_transition_order_is_valid(transitions) { + return Err(format!( + "iterative_request_router: step '{name}' transition {i}: \ + header-safe streaming failover rules must precede completion rules" + ) + .into()); + } + if transition.next.is_some() + && transition.filter.is_none() + && (transition.key.is_some() || transition.value.is_some()) + { return Err(format!( "iterative_request_router: step '{name}' transition {i}: \ - filter/key/value predicates are incompatible with \ - streaming-capable pipelines (response body is not \ - available during transition evaluation)" + ambiguous streaming transition predicates" ) .into()); } @@ -254,6 +289,7 @@ impl IterativeRequestRouterFilter { initial_step: Arc::from(cfg.initial_step.as_str()), max_iterations: cfg.max_iterations, max_response_bytes: cfg.max_response_bytes, + max_stream_response_bytes: cfg.max_stream_response_bytes, max_state_bytes: cfg.max_state_bytes, step_pipelines, step_timeout, @@ -279,6 +315,22 @@ impl HttpFilter for IterativeRequestRouterFilter { } } + fn visit_nested_pipelines(&mut self, visitor: &mut dyn FnMut(&mut FilterPipeline)) { + for pipeline in self.step_pipelines.values_mut() { + if let Some(pipeline) = Arc::get_mut(pipeline) { + visitor(pipeline); + } else { + debug_assert!(false, "IRR step pipelines must be uniquely owned during configuration"); + } + } + } + + fn apply_insecure_options(&self, options: &praxis_core::config::InsecureOptions) { + for pipeline in self.step_pipelines.values() { + pipeline.apply_insecure_options(options); + } + } + /// Validate the request, then run the iteration at the router's normal /// request-header position after preceding filters have completed. async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { @@ -305,7 +357,7 @@ impl HttpFilter for IterativeRequestRouterFilter { .into() })?; - Box::pin(self.run_iterations(ctx, request_body)).await + Box::pin(self.run_iterations_with_runner(ctx, request_body)).await } } @@ -314,15 +366,17 @@ impl HttpFilter for IterativeRequestRouterFilter { reason = "lifecycle implementation is kept separate from construction" )] impl IterativeRequestRouterFilter { - /// Run the complete iterative subrequest lifecycle. - #[expect(clippy::too_many_lines, reason = "iteration loop is inherently sequential")] - #[expect(clippy::large_stack_frames, reason = "sub-pipeline execution needs stack space")] - #[expect(clippy::large_futures, reason = "streaming and buffered paths require large future")] + /// Run the logical request through the reusable one-step executor. + #[expect(clippy::too_many_lines, reason = "the loop owns explicit state transitions")] + #[expect( + clippy::large_stack_frames, + reason = "opening a step reconstructs a full filter context" + )] #[expect( clippy::significant_drop_tightening, - reason = "step execution temporaries span iteration loop" + reason = "opened streaming steps are consumed by their selected lifecycle" )] - async fn run_iterations( + async fn run_iterations_with_runner( &self, ctx: &mut HttpFilterContext<'_>, request_body: Bytes, @@ -333,14 +387,12 @@ impl IterativeRequestRouterFilter { .ok_or_else(|| -> FilterError { "iterative_request_router: no sub-request client".to_owned().into() })? .clone(); let max_response_bytes = effective_response_limit(self.max_response_bytes, ctx.response_body_mode); - let original_request = SubRequest { method: ctx.request.method.clone(), uri: ctx.request.uri.clone(), headers: ctx.request.headers.clone(), body: request_body, }; - let mut state = IterationState { original_request: original_request.clone(), previous_response: None, @@ -359,25 +411,43 @@ impl IterativeRequestRouterFilter { return Ok(FilterAction::Reject(Rejection::status(413))); } + let runner = IrrStepRunner::new( + client, + depth, + max_response_bytes, + self.max_state_bytes, + StepRuntime { + client_addr: ctx.client_addr, + downstream_tls: ctx.downstream_tls, + peer_identity: ctx.peer_identity.clone(), + request_start: ctx.request_start, + }, + self.step_pipelines.clone(), + self.step_timeout, + ); let mut current_step = Arc::clone(&self.initial_step); let mut current_request = original_request; + let mut extensions = std::mem::take(&mut ctx.extensions); + let mut pending_chunks = VecDeque::new(); + let mut pending_bytes = 0_usize; loop { if state.iteration >= self.max_iterations { + ctx.extensions = extensions; warn!( iterations = state.iteration, max = self.max_iterations, - "iterative_request_router: max iterations \ - exhausted" + "iterative_request_router: max iterations exhausted" ); return Ok(FilterAction::Reject(Rejection::status(508))); } - - let remaining = state + if state .deadline .checked_duration_since(Instant::now()) - .unwrap_or(Duration::ZERO); - if remaining.is_zero() { + .unwrap_or(Duration::ZERO) + .is_zero() + { + ctx.extensions = extensions; warn!( iterations = state.iteration, "iterative_request_router: deadline exceeded" @@ -385,666 +455,273 @@ impl IterativeRequestRouterFilter { return Ok(FilterAction::Reject(Rejection::status(504))); } - let pipeline = self.step_pipelines.get(¤t_step).ok_or_else(|| -> FilterError { - format!( - "iterative_request_router: step \ - '{current_step}' not found" - ) - .into() - })?; - - let step_span = tracing::info_span!( - "iterative_subrequest", - step = current_step.as_ref(), - iteration = state.iteration, - ); - let _enter = step_span.enter(); - - debug!( - step = current_step.as_ref(), - iteration = state.iteration, - "executing step" - ); - - let mut sub_headers = current_request.headers.clone(); - strip_reserved_headers(&mut sub_headers); - - let sub_req = crate::Request { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: sub_headers.clone(), - }; - let mut routed_req = sub_req.clone(); - - // Keep the response metadata alive for the full step - // context so response-header and response-body hooks share - // the same lifecycle state. - let mut response_header = crate::Response { - headers: HeaderMap::new(), - status: http::StatusCode::OK, - }; - let runtime_resources = SubPipelineRuntimeResources { - client_addr: ctx.client_addr, - downstream_tls: ctx.downstream_tls, - health_registry: ctx.health_registry, - id_generator: ctx.id_generator, - kv_stores: ctx.kv_stores, - peer_identity: ctx.peer_identity.as_ref(), - request_start: ctx.request_start, - subrequest_client: Some(&client), - time_source: ctx.time_source, + let opened = match Box::pin(runner.open_step(¤t_step, ¤t_request, &state, extensions)).await { + Ok(opened) => opened, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, }; - let mut filter_ctx = build_sub_filter_context(pipeline, &sub_req, runtime_resources); - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - filter_ctx.extensions.insert(state.clone()); - filter_ctx.extensions.insert(RetainedFilterResults::default()); - let step_timeout = remaining.min(self.step_timeout); - let step_start = Instant::now(); - let in_transport = Arc::new(AtomicBool::new(false)); - let in_transport_inner = Arc::clone(&in_transport); - let step_result: Result = match tokio::time::timeout(step_timeout, async { - let mut request_body = Some(current_request.body.clone()); - if body_exceeds_limit( - pipeline.body_capabilities().request_body_mode, - request_body.as_ref().map_or(0, Bytes::len), - ) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - let pre_read_body = matches!( - pipeline.body_capabilities().request_body_mode, - crate::body::BodyMode::StreamBuffer { .. } - ); - if pre_read_body { - let action = pipeline - .execute_http_request_body(&mut filter_ctx, &mut request_body, true) - .await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - apply_pre_read_header_mutations(&mut routed_req.headers, &filter_ctx); - filter_ctx.extra_request_headers.clear(); - filter_ctx.request_headers_to_remove.clear(); - filter_ctx.request_headers_to_set.clear(); - filter_ctx.pre_read_mutations.clear(); - sub_headers.clone_from(&routed_req.headers); - filter_ctx.request = &routed_req; - } - - let action = pipeline.execute_http_request(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - if !pre_read_body { - let action = pipeline - .execute_http_request_body(&mut filter_ctx, &mut request_body, true) - .await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); + let runner::OpenedStep { continuation, kind } = opened; + + match kind { + OpenedStepKind::Streaming { body, outcome } => { + let transitions = self.step_transitions.get(¤t_step).map_or(&[][..], Vec::as_slice); + if !streaming_transition_order_is_valid(transitions) { + (*body).cancel().await; + ctx.extensions = continuation.into_parent_extensions(); + return Err(format!( + "iterative_request_router: step '{current_step}' selected streaming with interleaved transition phases" + ) + .into()); } - } - - let upstream = filter_ctx.upstream.as_ref().ok_or_else(|| -> FilterError { - format!( - "iterative_request_router: step \ - '{current_step}' did not resolve an upstream" - ) - .into() - })?; - in_transport_inner.store(true, Ordering::Release); - let peer = build_peer(upstream).await; - - apply_request_header_mutations(&mut sub_headers, &filter_ctx); - ensure_destination_host(&mut sub_headers, &upstream.address)?; - sanitize_subrequest_headers(&mut sub_headers); - let sub_request_for_exec = SubRequest { - method: current_request.method.clone(), - uri: filter_ctx.rewritten_path.as_ref().map_or_else( - || current_request.uri.clone(), - |p| http::Uri::try_from(p.as_str()).unwrap_or_else(|_| current_request.uri.clone()), - ), - headers: sub_headers, - body: request_body.unwrap_or_default(), - }; - - let mut fw_headers = FrameworkHeaders::new(); - fw_headers.set_depth(depth + 1); - - let per_request_timeout = step_timeout.checked_sub(step_start.elapsed()).unwrap_or(Duration::ZERO); - if per_request_timeout.is_zero() { - return Ok(StepExecution::Rejected(Rejection::status(504))); - } - - match filter_ctx.subrequest_response_mode { - SubRequestResponseMode::Streaming => { - // Runtime guard: reject if transitions depend on response body - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - if has_body_dependent_transitions(transitions) { - return Err(format!( - "iterative_request_router: step '{current_step}' selected \ - streaming but has body-dependent transitions" - ) - .into()); - } - if matches!( - pipeline.body_capabilities().response_body_mode, - crate::body::BodyMode::StreamBuffer { .. } - ) { - return Err(format!( - "iterative_request_router: step '{current_step}' selected \ - streaming but has StreamBuffer response body mode" + match evaluate_header_transitions(transitions, &outcome) { + TransitionResult::Next(next_step) => { + debug!( + from = current_step.as_ref(), + to = next_step.as_ref(), + "streaming header failover before response commitment" + ); + let mut skipped = IrrStreamingBody::new(body, continuation); + if let Err(error) = skipped.suppress().await { + ctx.extensions = skipped.into_continuation().into_parent_extensions(); + return Err(error); + } + let mut completion = match skipped.into_continuation().into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, + }; + completion.state.previous_response = None; + completion.state.iteration += 1; + if completion.state.retained_bytes() > self.max_state_bytes { + ctx.extensions = completion.extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } + let next_body = completion + .next_iteration_body + .unwrap_or_else(|| current_request.body.clone()); + state = completion.state; + extensions = completion.extensions; + current_request = SubRequest { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: HeaderMap::new(), + body: next_body, + }; + current_step = next_step; + }, + TransitionResult::Done | TransitionResult::NoMatch => { + let Some(active_state) = continuation.extensions.get::() else { + (*body).cancel().await; + ctx.extensions = continuation.into_parent_extensions(); + return Err( + "iterative_request_router: iteration state missing before stream handoff" + .to_owned() + .into(), + ); + }; + if ensure_combined_retained_limit( + active_state.retained_bytes(), + pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, ) - .into()); - } - - let mut step_origin = config::ResponseOrigin::Upstream; - let mut step_transport_error = None; - - let streaming_result = match peer { - Ok(peer) => { - // Cap the streamed response by the remaining overall - // deadline. Without an absolute lifetime, a slow-drip - // upstream (one byte per idle window) would hold the - // upstream session and its admission permit far past - // timeout_ms, starving further sub-requests. - let limits = StreamLimits { - idle_timeout: STREAMING_IDLE_TIMEOUT, - max_stream_duration: Some( - state - .deadline - .checked_duration_since(Instant::now()) - .unwrap_or(Duration::ZERO), - ), - max_total_bytes: None, - }; - Box::pin(client.send_streaming( - &peer, - &sub_request_for_exec, - per_request_timeout, - limits, - Some(&fw_headers), - )) - .await - }, - Err(error) => Err(praxis_core::subrequest::SubRequestError::Connect(error.to_string())), - }; - in_transport_inner.store(false, Ordering::Release); - - match streaming_result { - Ok(streaming_response) => { - let status = streaming_response.status; - let mut response_headers = streaming_response.headers; - sanitize_subresponse_headers(&mut response_headers); - - response_header.status = - http::StatusCode::from_u16(status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid upstream status: {e}").into() - })?; - response_header.headers.clone_from(&response_headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - streaming_response.body.cancel().await; - return Ok(StepExecution::Rejected(rejection)); - } - - // Extract final response metadata from filter context - let (final_status, mut final_headers) = - if let Some(meta) = filter_ctx.response_header.as_deref() { - (meta.status, meta.headers.clone()) - } else { - ( - http::StatusCode::from_u16(status).unwrap_or(http::StatusCode::BAD_GATEWAY), - response_headers, - ) - }; - sanitize_subresponse_headers(&mut final_headers); - - let outcome = StepOutcome { - response: SubResponse { - status: final_status.as_u16(), - headers: final_headers.clone(), - body: Bytes::new(), + .is_err() + { + (*body).cancel().await; + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); }, - origin: step_origin, - transport_error: step_transport_error, }; - - Ok(StepExecution::Streaming { + ctx.extensions = completion.extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } + let status = normalize_response_status(outcome.response.status); + let headers = outcome.response.headers.clone(); + let terminal = StreamingTerminalResponse::new( + status, + Box::new(IrrStreamingSession::new( + runner, + Arc::clone(¤t_step), + current_request, outcome, - body: Box::new(streaming_response.body), - response_snapshot: crate::Response { - status: final_status, - headers: final_headers, - }, - }) - }, - Err(error) => { - let (status, kind) = classify_transport_failure(&error); - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(kind); - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: streaming sub-request transport failure" + body, + continuation, + pending_chunks, + self.step_transitions.clone(), + self.max_state_bytes, + self.max_stream_response_bytes, + )), + ) + .with_headers(headers); + return Ok(FilterAction::StreamingTerminalResponse(Box::new(terminal))); + }, + } + }, + OpenedStepKind::Complete(mut outcome) => { + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + ctx.extensions = restored_extensions; + return Err(error); + }, + }; + let abnormal_stream_completion = completion.termination.is_some(); + let handled_abnormal_stream_completion = completion + .termination + .as_ref() + .is_some_and(StreamTermination::is_handled); + let completed_pending_chunks = completion.pending_chunks; + state = completion.state; + state.previous_response = Some(outcome.response.clone()); + state.iteration += 1; + let next_iteration_body = completion.next_iteration_body; + let filter_results = completion.filter_results; + extensions = completion.extensions; + if state.retained_bytes() > self.max_state_bytes { + ctx.extensions = extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); + } + + info!( + step = current_step.as_ref(), + iteration = state.iteration - 1, + status = outcome.response.status, + body_bytes = outcome.response.body.len(), + "sub-request complete" + ); + let transitions = self.step_transitions.get(¤t_step).map_or(&[][..], Vec::as_slice); + match evaluate_transitions(transitions, &outcome, &filter_results) { + TransitionResult::Next(next_step) => { + if !abnormal_stream_completion { + let appended = append_pending_chunks( + &mut pending_chunks, + completed_pending_chunks, + pending_bytes, + self.max_state_bytes, + state.retained_bytes(), ); - let response = SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), + let Ok(updated_pending_bytes) = appended else { + ctx.extensions = extensions; + return Ok(FilterAction::Reject(Rejection::status(413))); }; - // Fall through to buffered transition path with synthetic response - // (transport failures don't have a body to stream) - - response_header.status = - http::StatusCode::from_u16(status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid status: {e}").into() + pending_bytes = updated_pending_bytes; + } + let next_body = next_iteration_body.unwrap_or_else(|| current_request.body.clone()); + current_request = SubRequest { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: HeaderMap::new(), + body: next_body, + }; + current_step = next_step; + }, + TransitionResult::Done | TransitionResult::NoMatch => { + if handled_abnormal_stream_completion { + let combined_bytes = pending_chunks + .iter() + .chain(completed_pending_chunks.iter()) + .chain(std::iter::once(&outcome.response.body)) + .try_fold(0_usize, |total, chunk| total.checked_add(chunk.len())) + .ok_or_else(|| -> FilterError { + "iterative_request_router: completion body byte count overflow".into() })?; - response_header.headers.clone_from(&response.headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - - Ok(StepExecution::Complete { - response, - origin: step_origin, - transport_error: step_transport_error, - }) - }, - } - }, - SubRequestResponseMode::Buffered => { - let mut step_origin = config::ResponseOrigin::Upstream; - let mut step_transport_error = None; - let mut response = match peer { - Ok(peer) => match Box::pin(client.execute( - &peer, - &sub_request_for_exec, - max_response_bytes, - per_request_timeout, - Some(&fw_headers), - )) - .await - { - Ok(response) => response, - Err(error) => { - let (status, kind) = classify_transport_failure(&error); - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(kind); - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: sub-request transport failure" + if combined_bytes > max_response_bytes { + ctx.extensions = extensions; + return Err( + "iterative_request_router: abnormal completion exceeds response body limit" + .to_owned() + .into(), ); - SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), - } - }, - }, - Err(error) => { - step_origin = config::ResponseOrigin::Transport; - step_transport_error = Some(config::TransportErrorKind::Connect); - let status = 502; - warn!( - step = current_step.as_ref(), - %error, - status, - "iterative_request_router: sub-request transport failure" - ); - SubResponse { - status, - headers: HeaderMap::new(), - body: Bytes::new(), } - }, - }; - in_transport_inner.store(false, Ordering::Release); - sanitize_subresponse_headers(&mut response.headers); - - response_header.status = - http::StatusCode::from_u16(response.status).map_err(|e| -> FilterError { - format!("iterative_request_router: invalid upstream status: {e}").into() - })?; - response_header.headers.clone_from(&response.headers); - filter_ctx.response_header = Some(&mut response_header); - - let action = pipeline.execute_http_response(&mut filter_ctx).await?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - - let mut response_body = Some(std::mem::take(&mut response.body)); - if response_body_exceeds_limits( - pipeline.body_capabilities().response_body_mode, - max_response_bytes, - response_body.as_ref().map_or(0, Bytes::len), - ) { - return Err("iterative_request_router: step response exceeds configured body limit" - .to_owned() - .into()); - } - let action = pipeline.execute_http_response_body(&mut filter_ctx, &mut response_body, true)?; - if let FilterAction::Reject(rejection) = action { - return Ok(StepExecution::Rejected(rejection)); - } - if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { - return Ok(StepExecution::Rejected(Rejection::status(413))); - } - if response_body_exceeds_limits( - pipeline.body_capabilities().response_body_mode, - max_response_bytes, - response_body.as_ref().map_or(0, Bytes::len), - ) { - return Err( - "iterative_request_router: transformed step response exceeds configured body limit" - .to_owned() - .into(), - ); - } - - if let Some(meta) = filter_ctx.response_header.as_deref() { - response.status = meta.status.as_u16(); - response.headers.clone_from(&meta.headers); - } - response.body = response_body.unwrap_or_default(); - sanitize_subresponse_headers(&mut response.headers); - - Ok(StepExecution::Complete { - response, - origin: step_origin, - transport_error: step_transport_error, - }) - }, - } - }) - .await - { - Ok(result) => result, - Err(_elapsed) => { - if in_transport.load(Ordering::Acquire) { - Ok(StepExecution::Complete { - response: SubResponse { - status: 504, - headers: HeaderMap::new(), - body: Bytes::new(), - }, - origin: config::ResponseOrigin::Transport, - transport_error: Some(config::TransportErrorKind::DeadlineExceeded), - }) - } else { - Ok(StepExecution::Rejected(Rejection::status(504))) + let mut combined = Vec::with_capacity(combined_bytes); + for chunk in pending_chunks.drain(..) { + combined.extend_from_slice(&chunk); + } + for chunk in completed_pending_chunks { + combined.extend_from_slice(&chunk); + } + combined.extend_from_slice(&outcome.response.body); + outcome.response.body = Bytes::from(combined); + } else if abnormal_stream_completion { + pending_chunks.clear(); + outcome.response.body = Bytes::new(); + } else if !pending_chunks.is_empty() || !completed_pending_chunks.is_empty() { + ctx.extensions = extensions; + return Err( + "iterative_request_router: stream chunks were emitted without a streaming response" + .to_owned() + .into(), + ); + } + ctx.extensions = extensions; + return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( + &outcome.response, + current_request.method == http::Method::HEAD, + )))); + }, } }, - }; - - // Handle streaming step results before normal post-step processing. - // The streaming terminal path takes ownership of filter_ctx.extensions - // (which include the parent's extensions swapped in at step start). - if let Ok(StepExecution::Streaming { - outcome, - body: upstream_body, - response_snapshot, - }) = step_result - { - // Extract iteration state and filter results for transition evaluation - if let Some(updated_state) = filter_ctx.extensions.remove::() { - state = updated_state; - } - let next_iteration_body = filter_ctx.extensions.remove::(); - let mut step_filter_results = filter_ctx - .extensions - .remove::() - .unwrap_or_default() - .0; - step_filter_results.extend( - filter_ctx - .filter_results - .iter() - .map(|(name, results)| (*name, results.clone())), - ); - - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - match evaluate_transitions(transitions, &outcome, &step_filter_results) { - TransitionResult::Done | TransitionResult::NoMatch => { - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - let continuation = StepResponseContinuation { - pipeline: Arc::clone(self.step_pipelines.get(¤t_step).ok_or_else( - || -> FilterError { - format!("iterative_request_router: step '{current_step}' not found").into() - }, - )?), - request_snapshot: crate::Request { - method: filter_ctx.request.method.clone(), - uri: filter_ctx.request.uri.clone(), - headers: filter_ctx.request.headers.clone(), - }, - response_snapshot, - extensions: std::mem::take(&mut filter_ctx.extensions), - filter_state: std::mem::take(&mut filter_ctx.filter_state), - filter_results: std::mem::take(&mut filter_ctx.filter_results), - filter_metadata: std::mem::take(&mut filter_ctx.filter_metadata), - structured_metadata: std::mem::take(&mut filter_ctx.structured_metadata), - executed_filter_indices: std::mem::take(&mut filter_ctx.executed_filter_indices), - body_done_indices: std::mem::take(&mut filter_ctx.body_done_indices), - response_body_bytes: filter_ctx.response_body_bytes, - response_body_mode: filter_ctx.response_body_mode, - completed: false, - client_addr: filter_ctx.client_addr, - downstream_tls: filter_ctx.downstream_tls, - request_start: filter_ctx.request_start, - peer_identity: filter_ctx.peer_identity.clone(), - }; - let status = normalize_response_status(outcome.response.status); - let terminal = StreamingTerminalResponse::new( - status, - Box::new(IrrStreamingBody::new(*upstream_body, continuation)), - ) - .with_headers(outcome.response.headers); - return Ok(FilterAction::StreamingTerminalResponse(Box::new(terminal))); - }, - TransitionResult::Next(next_step) => { - debug!( - from = current_step.as_ref(), - to = next_step.as_ref(), - "streaming failover: cancelling unread body" - ); - (*upstream_body).cancel().await; - // Swap extensions back to parent and continue the loop - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - state.previous_response = None; - state.iteration += 1; - let next_body = next_iteration_body.map_or_else(|| current_request.body.clone(), |b| b.0); - current_request = SubRequest { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: HeaderMap::new(), - body: next_body, - }; - current_step = next_step; - continue; - }, - } - } - - // Existing post-step processing continues unchanged for - // Complete/Rejected variants... - if let Some(updated_state) = filter_ctx.extensions.remove::() { - state = updated_state; - } - if state.retained_bytes() > self.max_state_bytes { - return Ok(FilterAction::Reject(Rejection::status(413))); - } - let next_iteration_body = filter_ctx.extensions.remove::(); - let mut step_filter_results = filter_ctx - .extensions - .remove::() - .unwrap_or_default() - .0; - step_filter_results.extend( - filter_ctx - .filter_results - .iter() - .map(|(name, results)| (*name, results.clone())), - ); - std::mem::swap(&mut filter_ctx.extensions, &mut ctx.extensions); - - let step_result_value = step_result?; - let (mut response, origin, transport_error) = match step_result_value { - StepExecution::Complete { - response, - origin, - transport_error, - } => (response, origin, transport_error), - StepExecution::Rejected(rejection) => { - debug!( - step = current_step.as_ref(), - status = rejection.status, - "step pipeline produced a local response" - ); - ( - subresponse_from_rejection(rejection), - config::ResponseOrigin::Local, - None, - ) - }, - StepExecution::Streaming { .. } => { - unreachable!("streaming step results are handled before this point") - }, - }; - sanitize_subresponse_headers(&mut response.headers); - - info!( - step = current_step.as_ref(), - iteration = state.iteration, - status = response.status, - body_bytes = response.body.len(), - "sub-request complete" - ); - - state.previous_response = Some(response.clone()); - state.iteration += 1; - if state.retained_bytes() > self.max_state_bytes { - return Ok(FilterAction::Reject(Rejection::status(413))); - } - - let outcome = StepOutcome { - response, - origin, - transport_error, - }; - - let transitions = self - .step_transitions - .get(¤t_step) - .map_or(&[][..], |v| v.as_slice()); - - match evaluate_transitions(transitions, &outcome, &step_filter_results) { - TransitionResult::Done => { - debug!(step = current_step.as_ref(), "iteration complete, returning response"); - return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( - &outcome.response, - current_request.method == http::Method::HEAD, - )))); - }, - TransitionResult::Next(next_step) => { - debug!( - from = current_step.as_ref(), - to = next_step.as_ref(), - "transitioning to next step" - ); - let next_body = next_iteration_body.map_or_else(|| current_request.body.clone(), |b| b.0); - current_request = SubRequest { - method: current_request.method.clone(), - uri: current_request.uri.clone(), - headers: HeaderMap::new(), - body: next_body, - }; - current_step = next_step; - }, - TransitionResult::NoMatch => { - debug!( - step = current_step.as_ref(), - "no transition matched, returning response" - ); - return Ok(FilterAction::TerminalResponse(Box::new(build_terminal_response( - &outcome.response, - current_request.method == http::Method::HEAD, - )))); - }, } } } } +/// Append locally emitted chunks while preserving the shared retained-state bound. +fn append_pending_chunks( + target: &mut VecDeque, + chunks: VecDeque, + current_bytes: usize, + max_state_bytes: usize, + retained_bytes: usize, +) -> Result { + let added_bytes = chunks.iter().try_fold(0_usize, |total, chunk| { + total + .checked_add(chunk.len()) + .ok_or_else(|| -> FilterError { "iterative_request_router: pending stream byte count overflow".into() }) + })?; + let pending_bytes = current_bytes + .checked_add(added_bytes) + .ok_or_else(|| -> FilterError { "iterative_request_router: pending stream byte count overflow".into() })?; + if retained_bytes + .checked_add(pending_bytes) + .is_none_or(|total| total > max_state_bytes) + { + return Err( + "iterative_request_router: retained state and pending stream output exceed configured limit" + .to_owned() + .into(), + ); + } + target.extend(chunks); + Ok(pending_bytes) +} + // --------------------------------------------------------------------------- // Transition Evaluation // --------------------------------------------------------------------------- -/// Result of executing one step's complete filter and HTTP lifecycle. -enum StepExecution { - /// A step filter rejected the request or response. - Rejected(Rejection), - - /// The upstream exchange completed after all response hooks. - Complete { - /// The sub-request response. - response: SubResponse, - /// Where the response originated. - origin: config::ResponseOrigin, - /// Transport error classification, if any. - transport_error: Option, - }, - - /// The step uses streaming mode; response-header filters ran. - Streaming { - /// Step outcome with header-only response snapshot. - outcome: StepOutcome, - /// Upstream streaming body handle. - body: Box, - /// Snapshot of response headers for continuation. - response_snapshot: crate::Response, - }, -} - /// A step's response together with metadata about where it came from. -struct StepOutcome { +pub(super) struct StepOutcome { /// The sub-request response. - response: SubResponse, + pub(super) response: SubResponse, /// Where the response originated. - origin: config::ResponseOrigin, + pub(super) origin: config::ResponseOrigin, /// Transport error classification, if any. - transport_error: Option, + pub(super) transport_error: Option, } /// Result of evaluating step transition rules. -enum TransitionResult { +pub(super) enum TransitionResult { /// Return the current response to the client. Done, @@ -1056,7 +733,9 @@ enum TransitionResult { } /// Convert transport failures into a gateway status and error classification. -fn classify_transport_failure(error: &praxis_core::subrequest::SubRequestError) -> (u16, config::TransportErrorKind) { +pub(super) fn classify_transport_failure( + error: &praxis_core::subrequest::SubRequestError, +) -> (u16, config::TransportErrorKind) { use praxis_core::subrequest::SubRequestError; match error { SubRequestError::AdmissionTimeout { .. } => (503, config::TransportErrorKind::AdmissionTimeout), @@ -1102,7 +781,7 @@ fn normalize_response_status(status: u16) -> u16 { } /// Evaluate transition rules against a step outcome. -fn evaluate_transitions( +pub(super) fn evaluate_transitions( transitions: &[config::StepTransition], outcome: &StepOutcome, filter_results: &HashMap<&str, crate::results::FilterResultSet>, @@ -1122,6 +801,24 @@ fn evaluate_transitions( TransitionResult::NoMatch } +/// Evaluate only the ordered header-safe failover prefix. +pub(super) fn evaluate_header_transitions( + transitions: &[config::StepTransition], + outcome: &StepOutcome, +) -> TransitionResult { + for transition in transitions + .iter() + .take_while(|transition| is_header_safe_failover(transition)) + { + if matches_transition(transition, outcome, &HashMap::new()) + && let Some(next) = &transition.next + { + return TransitionResult::Next(Arc::from(next.as_str())); + } + } + TransitionResult::NoMatch +} + /// Check if a transition matches the outcome and/or filter results. fn matches_transition( transition: &config::StepTransition, @@ -1169,7 +866,8 @@ fn parse_depth(request: &crate::Request) -> u8 { } /// Strip all reserved internal headers from sub-request headers -/// so the core executor re-injects depth via [`FrameworkHeaders`]. +/// so the core executor re-injects depth via +/// [`FrameworkHeaders`](praxis_core::subrequest::FrameworkHeaders). /// /// The depth header uses a reserved `x-praxis-*` prefix, so it /// is covered by the [`is_reserved`] check. @@ -1328,6 +1026,18 @@ fn effective_response_limit(configured: usize, parent_mode: crate::body::BodyMod } } +/// Extract only the listener-level streaming ceiling from a nested pipeline. +/// +/// The IRR `max_response_bytes` setting is intentionally buffered-only. A +/// nested `SizeLimit` is produced by listener body-limit propagation and must +/// still constrain the live transport. +fn streaming_transport_limit(mode: crate::body::BodyMode) -> Option { + match mode { + crate::body::BodyMode::SizeLimit { max_bytes } => Some(max_bytes), + crate::body::BodyMode::Stream | crate::body::BodyMode::StreamBuffer { .. } => None, + } +} + /// Whether a step filter grew the shared iteration state past its ceiling. fn iteration_state_exceeds_limit(ctx: &HttpFilterContext<'_>, max_state_bytes: usize) -> bool { ctx.extensions diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs new file mode 100644 index 000000000..0ec7c1d77 --- /dev/null +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/runner.rs @@ -0,0 +1,603 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Praxis Contributors + +//! One-step execution for the iterative request router. + +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use bytes::Bytes; +use http::HeaderMap; +use praxis_core::subrequest::{FrameworkHeaders, StreamLimits}; +use tracing::{Instrument as _, warn}; + +use super::{ + StepOutcome, SubPipelineRuntimeResources, apply_pre_read_header_mutations, apply_request_header_mutations, + body_exceeds_limit, build_peer, build_sub_filter_context, classify_transport_failure, config, + ensure_destination_host, iteration_state_exceeds_limit, response_body_exceeds_limits, sanitize_subrequest_headers, + sanitize_subresponse_headers, streaming::StepResponseContinuation, streaming_transport_limit, + strip_reserved_headers, subresponse_from_rejection, +}; +use crate::{ + FilterAction, FilterError, FilterPipeline, IterationState, NextIterationBody, RequestExtensions, StreamTermination, + StreamTerminationCause, SubRequest, SubRequestResponseMode, SubResponse, actions::Rejection, + context::PendingStreamChunks, results::RetainedFilterResults, +}; + +/// Owned request attributes needed after the outer request hook returns. +#[derive(Clone)] +pub(super) struct StepRuntime { + /// Original downstream client address. + pub(super) client_addr: Option, + /// Whether the original downstream uses TLS. + pub(super) downstream_tls: bool, + /// Verified downstream peer identity. + pub(super) peer_identity: Option>, + /// Start time of the logical client request. + pub(super) request_start: Instant, +} + +/// Executes exactly one IRR step and returns owned continuation state. +pub(super) struct IrrStepRunner { + /// Shared transport client. + client: praxis_core::subrequest::SubRequestClient, + /// Nested IRR depth forwarded to subrequests. + depth: u8, + /// Per-step buffered response ceiling. + max_response_bytes: usize, + /// Retained iteration-state and pending-output ceiling. + max_state_bytes: usize, + /// Owned downstream request attributes. + runtime: StepRuntime, + /// Named, pre-built step pipelines. + step_pipelines: HashMap, Arc>, + /// Per-step duration ceiling. + step_timeout: Duration, +} + +/// One opened step, including state needed for body/completion processing. +pub(super) struct OpenedStep { + /// Owned filter lifecycle state. + pub(super) continuation: StepResponseContinuation, + /// Buffered or pull-based response source. + pub(super) kind: OpenedStepKind, +} + +/// A step error together with the parent request extensions it borrowed. +pub(super) struct OpenStepError { + /// Underlying filter or lifecycle error. + error: FilterError, + /// Parent-owned extensions recovered from the nested filter context. + extensions: RequestExtensions, +} + +impl OpenStepError { + /// Build an error before a nested filter context exists. + fn new(error: FilterError, extensions: RequestExtensions) -> Self { + Self { error, extensions } + } + + /// Recover only the parent-owned extensions from a failed nested context. + fn capture(error: FilterError, ctx: &mut crate::HttpFilterContext<'_>) -> Self { + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + ctx.extensions.remove::(); + Self::new(error, std::mem::take(&mut ctx.extensions)) + } + + /// Split the error from the extensions its caller must restore. + pub(super) fn into_parts(self) -> (FilterError, RequestExtensions) { + (self.error, self.extensions) + } +} + +/// Transport/body shape selected by the step filters. +pub(super) enum OpenedStepKind { + /// The complete response was collected and filtered. + Complete(StepOutcome), + /// Response headers are filtered; body remains pull-based. + Streaming { + /// Live upstream response body. + body: Box, + /// Header-time transition metadata. + outcome: StepOutcome, + }, +} + +/// Internal result before owned continuation state is captured. +enum RawStepKind { + /// Complete buffered or synthetic response. + Complete(StepOutcome), + /// Local filter rejection. + Rejected(Rejection), + /// Open pull-based upstream response. + Streaming { + /// Live upstream response body. + body: Box, + /// Header-time transition metadata. + outcome: StepOutcome, + }, +} + +impl IrrStepRunner { + /// Build an owned runner for one logical IRR request. + #[expect(clippy::too_many_arguments, reason = "runner owns explicit IRR limits and resources")] + pub(super) fn new( + client: praxis_core::subrequest::SubRequestClient, + depth: u8, + max_response_bytes: usize, + max_state_bytes: usize, + runtime: StepRuntime, + step_pipelines: HashMap, Arc>, + step_timeout: Duration, + ) -> Self { + Self { + client, + depth, + max_response_bytes, + max_state_bytes, + runtime, + step_pipelines, + step_timeout, + } + } + + /// Open one named step under the remaining overall deadline. + #[expect( + clippy::too_many_lines, + reason = "one step owns the complete filter and transport lifecycle" + )] + #[expect(clippy::large_futures, reason = "step execution spans filter and transport futures")] + #[expect( + clippy::large_stack_frames, + reason = "step execution reconstructs a full filter context" + )] + pub(super) async fn open_step( + &self, + current_step: &Arc, + current_request: &SubRequest, + state: &IterationState, + mut extensions: RequestExtensions, + ) -> Result { + let remaining = state + .deadline() + .checked_duration_since(Instant::now()) + .unwrap_or(Duration::ZERO); + if remaining.is_zero() { + return Err(OpenStepError::new( + "iterative_request_router: overall deadline exceeded".to_owned().into(), + extensions, + )); + } + let Some(pipeline) = self.step_pipelines.get(current_step) else { + return Err(OpenStepError::new( + format!("iterative_request_router: step '{current_step}' not found").into(), + extensions, + )); + }; + + let mut sub_headers = current_request.headers.clone(); + strip_reserved_headers(&mut sub_headers); + let sub_req = crate::Request { + method: current_request.method.clone(), + uri: current_request.uri.clone(), + headers: sub_headers.clone(), + }; + let mut routed_req = sub_req.clone(); + let mut response_header = crate::Response { + headers: HeaderMap::new(), + status: http::StatusCode::OK, + }; + let resources = SubPipelineRuntimeResources { + client_addr: self.runtime.client_addr, + downstream_tls: self.runtime.downstream_tls, + health_registry: pipeline.health_registry(), + id_generator: pipeline.id_generator(), + kv_stores: pipeline.kv_stores(), + peer_identity: self.runtime.peer_identity.as_ref(), + request_start: self.runtime.request_start, + subrequest_client: Some(&self.client), + time_source: pipeline.time_source(), + }; + let mut filter_ctx = build_sub_filter_context(pipeline, &sub_req, resources); + filter_ctx.extensions = std::mem::take(&mut extensions); + filter_ctx.extensions.insert(state.clone()); + filter_ctx.extensions.insert(RetainedFilterResults::default()); + filter_ctx.enable_stream_chunk_emission(self.max_state_bytes); + + let step_budget = remaining.min(self.step_timeout); + let step_started = Instant::now(); + let step_deadline = step_started + .checked_add(step_budget) + .unwrap_or_else(|| state.deadline()); + let in_transport = Arc::new(AtomicBool::new(false)); + let in_transport_inner = Arc::clone(&in_transport); + + let step_span = tracing::info_span!( + "iterative_subrequest", + step = current_step.as_ref(), + iteration = state.iteration, + ); + + let timed: Result, tokio::time::error::Elapsed> = + tokio::time::timeout(step_budget, async { + let mut request_body = Some(current_request.body.clone()); + if body_exceeds_limit( + pipeline.body_capabilities().request_body_mode, + request_body.as_ref().map_or(0, Bytes::len), + ) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + + let pre_read_body = matches!( + pipeline.body_capabilities().request_body_mode, + crate::BodyMode::StreamBuffer { .. } + ); + if pre_read_body { + let action = pipeline + .execute_http_request_body(&mut filter_ctx, &mut request_body, true) + .await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + apply_pre_read_header_mutations(&mut routed_req.headers, &filter_ctx); + filter_ctx.extra_request_headers.clear(); + filter_ctx.request_headers_to_remove.clear(); + filter_ctx.request_headers_to_set.clear(); + filter_ctx.pre_read_mutations.clear(); + sub_headers.clone_from(&routed_req.headers); + filter_ctx.request = &routed_req; + } + + let action = pipeline.execute_http_request(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + if !pre_read_body { + let action = pipeline + .execute_http_request_body(&mut filter_ctx, &mut request_body, true) + .await?; + if let FilterAction::Reject(rejection) = action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + } + + let upstream = filter_ctx.upstream.as_ref().ok_or_else(|| -> FilterError { + format!("iterative_request_router: step '{current_step}' did not resolve an upstream").into() + })?; + in_transport_inner.store(true, Ordering::Release); + let peer = build_peer(upstream).await; + apply_request_header_mutations(&mut sub_headers, &filter_ctx); + ensure_destination_host(&mut sub_headers, &upstream.address)?; + sanitize_subrequest_headers(&mut sub_headers); + let request = SubRequest { + method: current_request.method.clone(), + uri: filter_ctx.rewritten_path.as_ref().map_or_else( + || current_request.uri.clone(), + |path| http::Uri::try_from(path.as_str()).unwrap_or_else(|_| current_request.uri.clone()), + ), + headers: sub_headers, + body: request_body.unwrap_or_default(), + }; + let mut framework_headers = FrameworkHeaders::new(); + framework_headers.set_depth(self.depth + 1); + let transport_budget = step_budget + .checked_sub(step_started.elapsed()) + .unwrap_or(Duration::ZERO); + if transport_budget.is_zero() { + return Ok(RawStepKind::Rejected(Rejection::status(504))); + } + + match filter_ctx.subrequest_response_mode { + SubRequestResponseMode::Streaming => { + if matches!( + pipeline.body_capabilities().response_body_mode, + crate::BodyMode::StreamBuffer { .. } + ) { + return Err(format!( + "iterative_request_router: step '{current_step}' selected streaming with StreamBuffer response mode" + ) + .into()); + } + let limits = StreamLimits { + idle_timeout: super::STREAMING_IDLE_TIMEOUT, + // IrrStreamingBody enforces the original absolute step + // deadline so header time cannot be granted twice. + max_stream_duration: None, + max_total_bytes: streaming_transport_limit( + pipeline.body_capabilities().response_body_mode, + ), + }; + let response = match peer { + Ok(peer) => self + .client + .send_streaming(&peer, &request, transport_budget, limits, Some(&framework_headers)) + .await, + Err(error) => Err(praxis_core::subrequest::SubRequestError::Connect(error.to_string())), + }; + in_transport_inner.store(false, Ordering::Release); + match response { + Ok(response) => { + let status = response.status; + let mut headers = response.headers; + sanitize_subresponse_headers(&mut headers); + response_header.status = http::StatusCode::from_u16(status) + .map_err(|error| -> FilterError { format!("invalid upstream status: {error}").into() })?; + response_header.headers.clone_from(&headers); + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + response.body.cancel().await; + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + response.body.cancel().await; + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let metadata = filter_ctx.response_header.as_deref().ok_or_else(|| -> FilterError { + "iterative_request_router: response metadata missing after header filters" + .to_owned() + .into() + })?; + let status = metadata.status; + let mut headers = metadata.headers.clone(); + sanitize_subresponse_headers(&mut headers); + Ok(RawStepKind::Streaming { + body: Box::new(response.body), + outcome: StepOutcome { + response: SubResponse { status: status.as_u16(), headers, body: Bytes::new() }, + origin: config::ResponseOrigin::Upstream, + transport_error: None, + }, + }) + }, + Err(error) => { + let (status, kind) = classify_transport_failure(&error); + warn!(step = current_step.as_ref(), %error, status, "IRR streaming transport failure"); + let response = SubResponse { status, headers: HeaderMap::new(), body: Bytes::new() }; + response_header.status = http::StatusCode::from_u16(status) + .map_err(|source| -> FilterError { source.into() })?; + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let metadata = filter_ctx.response_header.as_deref().ok_or_else(|| -> FilterError { + "iterative_request_router: response metadata missing after header filters" + .to_owned() + .into() + })?; + let mut headers = metadata.headers.clone(); + sanitize_subresponse_headers(&mut headers); + Ok(RawStepKind::Complete(StepOutcome { + response: SubResponse { + status: metadata.status.as_u16(), + headers, + body: response.body, + }, + origin: config::ResponseOrigin::Transport, + transport_error: Some(kind), + })) + }, + } + }, + SubRequestResponseMode::Buffered => { + let (mut response, origin, transport_error) = match peer { + Ok(peer) => match self + .client + .execute(&peer, &request, self.max_response_bytes, transport_budget, Some(&framework_headers)) + .await + { + Ok(response) => (response, config::ResponseOrigin::Upstream, None), + Err(error) => { + let (status, kind) = classify_transport_failure(&error); + warn!(step = current_step.as_ref(), %error, status, "IRR buffered transport failure"); + ( + SubResponse { status, headers: HeaderMap::new(), body: Bytes::new() }, + config::ResponseOrigin::Transport, + Some(kind), + ) + }, + }, + Err(error) => { + warn!(step = current_step.as_ref(), %error, status = 502_u16, "IRR buffered transport failure"); + ( + SubResponse { status: 502, headers: HeaderMap::new(), body: Bytes::new() }, + config::ResponseOrigin::Transport, + Some(config::TransportErrorKind::Connect), + ) + }, + }; + in_transport_inner.store(false, Ordering::Release); + sanitize_subresponse_headers(&mut response.headers); + response_header.status = http::StatusCode::from_u16(response.status) + .map_err(|error| -> FilterError { error.into() })?; + response_header.headers.clone_from(&response.headers); + filter_ctx.response_header = Some(&mut response_header); + let response_action = pipeline.execute_http_response(&mut filter_ctx).await?; + if let FilterAction::Reject(rejection) = response_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + let mut body = Some(std::mem::take(&mut response.body)); + if response_body_exceeds_limits( + pipeline.body_capabilities().response_body_mode, + self.max_response_bytes, + body.as_ref().map_or(0, Bytes::len), + ) { + return Err("iterative_request_router: step response exceeds configured body limit".into()); + } + let body_action = pipeline.execute_http_response_body(&mut filter_ctx, &mut body, true)?; + if let FilterAction::Reject(rejection) = body_action { + return Ok(RawStepKind::Rejected(rejection)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + return Ok(RawStepKind::Rejected(Rejection::status(413))); + } + if response_body_exceeds_limits( + pipeline.body_capabilities().response_body_mode, + self.max_response_bytes, + body.as_ref().map_or(0, Bytes::len), + ) { + return Err( + "iterative_request_router: transformed step response exceeds configured body limit" + .into(), + ); + } + response.body = body.unwrap_or_default(); + if let Some(metadata) = filter_ctx.response_header.as_deref() { + response.status = metadata.status.as_u16(); + response.headers.clone_from(&metadata.headers); + } + sanitize_subresponse_headers(&mut response.headers); + Ok(RawStepKind::Complete(StepOutcome { response, origin, transport_error })) + }, + } + } + .instrument(step_span)) + .await; + + let mut raw = match timed { + Ok(Ok(raw)) => raw, + Ok(Err(error)) => return Err(OpenStepError::capture(error, &mut filter_ctx)), + Err(_) if in_transport.load(Ordering::Acquire) => RawStepKind::Complete(StepOutcome { + response: SubResponse { + status: 504, + headers: HeaderMap::new(), + body: Bytes::new(), + }, + origin: config::ResponseOrigin::Transport, + transport_error: Some(config::TransportErrorKind::DeadlineExceeded), + }), + Err(_) => RawStepKind::Rejected(Rejection::status(504)), + }; + + filter_ctx.response_header = None; + if filter_ctx.subrequest_response_mode == SubRequestResponseMode::Streaming + && let RawStepKind::Complete(outcome) = &mut raw + && outcome.origin == config::ResponseOrigin::Transport + { + let cause = outcome + .transport_error + .map_or(StreamTerminationCause::Io, stream_termination_cause); + filter_ctx.extensions.insert(StreamTermination::new(cause)); + let response_snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status).unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + let mut completion_body = None; + let completion_action = pipeline + .execute_http_response_body_with_response_header( + &mut filter_ctx, + &mut completion_body, + true, + Some(&response_snapshot), + ) + .map_err(|error| OpenStepError::capture(error, &mut filter_ctx))?; + if let FilterAction::Reject(_) = completion_action { + let error = "iterative_request_router: step completion filter rejected an abnormal stream" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + if iteration_state_exceeds_limit(&filter_ctx, self.max_state_bytes) { + let error = "iterative_request_router: retained state limit exceeded during stream completion" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + if completion_body + .as_ref() + .is_some_and(|body| body.len() > self.max_response_bytes) + { + let error = "iterative_request_router: abnormal completion exceeds response body limit" + .to_owned() + .into(); + return Err(OpenStepError::capture(error, &mut filter_ctx)); + } + outcome.response.body = completion_body.unwrap_or_default(); + } + let (kind, response_snapshot, completed) = match raw { + RawStepKind::Complete(outcome) => { + let snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status) + .unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + (OpenedStepKind::Complete(outcome), snapshot, true) + }, + RawStepKind::Rejected(rejection) => { + let response = subresponse_from_rejection(rejection); + let snapshot = crate::Response { + status: http::StatusCode::from_u16(response.status).unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: response.headers.clone(), + }; + ( + OpenedStepKind::Complete(StepOutcome { + response, + origin: config::ResponseOrigin::Local, + transport_error: None, + }), + snapshot, + true, + ) + }, + RawStepKind::Streaming { body, outcome } => { + let snapshot = crate::Response { + status: http::StatusCode::from_u16(outcome.response.status) + .unwrap_or(http::StatusCode::BAD_GATEWAY), + headers: outcome.response.headers.clone(), + }; + (OpenedStepKind::Streaming { body, outcome }, snapshot, false) + }, + }; + let request_snapshot = crate::Request { + method: filter_ctx.request.method.clone(), + uri: filter_ctx.request.uri.clone(), + headers: filter_ctx.request.headers.clone(), + }; + let continuation = StepResponseContinuation::capture( + Arc::clone(pipeline), + request_snapshot, + response_snapshot, + &mut filter_ctx, + completed, + step_deadline, + ); + Ok(OpenedStep { continuation, kind }) + } +} + +/// Convert transition-level transport metadata into completion-hook metadata. +fn stream_termination_cause(kind: config::TransportErrorKind) -> StreamTerminationCause { + match kind { + config::TransportErrorKind::AdmissionTimeout => StreamTerminationCause::AdmissionTimeout, + config::TransportErrorKind::CircuitOpen => StreamTerminationCause::CircuitOpen, + config::TransportErrorKind::Connect => StreamTerminationCause::Connect, + config::TransportErrorKind::Io => StreamTerminationCause::Io, + config::TransportErrorKind::DeadlineExceeded => StreamTerminationCause::DeadlineExceeded, + config::TransportErrorKind::ResponseTooLarge => StreamTerminationCause::ResponseTooLarge, + } +} diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs index 78bfbbdf0..4774319e7 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/streaming.rs @@ -6,11 +6,13 @@ //! The iterative request router runs step pipelines across multiple //! sub-requests. When a step's pipeline selects streaming mode, //! `on_request()` returns a terminal streaming response whose body -//! is owned by [`IrrStreamingBody`]. +//! is owned by [`IrrStreamingSession`]. //! -//! [`IrrStreamingBody`] pulls upstream chunks through the step's +//! [`IrrStreamingSession`] pulls upstream chunks through each step's //! response-body filters until the stream ends, then runs the step's -//! completion lifecycle exactly once (owned `end_of_stream` hook). +//! completion lifecycle exactly once and evaluates the next transition. +//! A matching `next` opens another step without recommitting downstream +//! response headers. //! //! [`StepResponseContinuation`] holds all state needed to run body //! filters after `on_request()` returns: the step pipeline, request @@ -18,17 +20,55 @@ //! and a completion guard. The continuation owns an `Arc` //! so the pipeline outlives the router filter. -use std::{any::Any, collections::HashMap, sync::Arc}; +use std::{ + any::Any, + collections::{HashMap, VecDeque}, + sync::Arc, +}; use async_trait::async_trait; use bytes::Bytes; use praxis_core::subrequest::SubResponseBody; use crate::{ - FilterError, FilterPipeline, actions::StreamingResponseBody, extensions::RequestExtensions, - results::FilterResultSet, + FilterError, FilterPipeline, IterationState, NextIterationBody, StreamTermination, StreamTerminationCause, + actions::StreamingResponseBody, + context::PendingStreamChunks, + extensions::RequestExtensions, + results::{FilterResultSet, RetainedFilterResults}, }; +/// State made available after a step's completion hook has run. +pub(super) struct StepCompletion { + /// Shared request extensions after step completion. + pub(super) extensions: RequestExtensions, + /// Results retained across all step phases. + pub(super) filter_results: HashMap<&'static str, FilterResultSet>, + /// Optional request body for the next step. + pub(super) next_iteration_body: Option, + /// Bounded locally emitted chunks. + pub(super) pending_chunks: VecDeque, + /// Updated iteration state. + pub(super) state: IterationState, + /// Typed abnormal source termination, when present. + pub(super) termination: Option, +} + +/// A completion conversion failure together with recoverable parent state. +pub(super) struct StepCompletionError { + /// Underlying lifecycle error. + error: FilterError, + /// Parent-owned extensions recovered from the failed continuation. + extensions: RequestExtensions, +} + +impl StepCompletionError { + /// Split the error from the extensions its caller must restore. + pub(super) fn into_parts(self) -> (FilterError, RequestExtensions) { + (self.error, self.extensions) + } +} + /// State continuation for streaming a step's response body. /// /// Owns the step pipeline, request/response snapshots, and all @@ -69,10 +109,87 @@ pub(super) struct StepResponseContinuation { pub(super) downstream_tls: bool, /// Start time of the containing client request. pub(super) request_start: std::time::Instant, + /// Absolute deadline shared by header and body processing for this step. + pub(super) step_deadline: std::time::Instant, /// Verified downstream mTLS identity. pub(super) peer_identity: Option>, } +impl StepResponseContinuation { + /// Capture all owned step context after response headers have run. + #[expect( + clippy::too_many_arguments, + reason = "capture owns the complete response continuation boundary" + )] + pub(super) fn capture( + pipeline: Arc, + request_snapshot: crate::Request, + response_snapshot: crate::Response, + ctx: &mut crate::filter::HttpFilterContext<'_>, + completed: bool, + step_deadline: std::time::Instant, + ) -> Self { + Self { + pipeline, + request_snapshot, + response_snapshot, + extensions: std::mem::take(&mut ctx.extensions), + filter_state: std::mem::take(&mut ctx.filter_state), + filter_results: std::mem::take(&mut ctx.filter_results), + filter_metadata: std::mem::take(&mut ctx.filter_metadata), + structured_metadata: std::mem::take(&mut ctx.structured_metadata), + executed_filter_indices: std::mem::take(&mut ctx.executed_filter_indices), + body_done_indices: std::mem::take(&mut ctx.body_done_indices), + response_body_bytes: ctx.response_body_bytes, + response_body_mode: ctx.response_body_mode, + completed, + client_addr: ctx.client_addr, + downstream_tls: ctx.downstream_tls, + request_start: ctx.request_start, + step_deadline, + peer_identity: ctx.peer_identity.clone(), + } + } + + /// Recover only caller-owned extensions when an internal invariant fails. + pub(super) fn into_parent_extensions(mut self) -> RequestExtensions { + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions.remove::(); + self.extensions + } + + /// Consume the completed continuation into transition inputs. + pub(super) fn into_completion(mut self) -> Result { + let Some(state) = self.extensions.remove::() else { + return Err(StepCompletionError { + error: "iterative_request_router: iteration state missing after step completion" + .to_owned() + .into(), + extensions: self.into_parent_extensions(), + }); + }; + let next_iteration_body = self.extensions.remove::().map(|body| body.0); + let pending_chunks = self + .extensions + .remove::() + .map_or_else(VecDeque::new, PendingStreamChunks::into_chunks); + let termination = self.extensions.remove::(); + let mut filter_results = self.extensions.remove::().unwrap_or_default().0; + filter_results.extend(self.filter_results); + Ok(StepCompletion { + extensions: self.extensions, + filter_results, + next_iteration_body, + pending_chunks, + state, + termination, + }) + } +} + /// Streaming body implementation for iterative request router steps. /// /// Pulls upstream chunks through the step's response-body filters, @@ -85,23 +202,44 @@ pub(super) struct StepResponseContinuation { /// pattern: `.take()` to move it out for cancellation. pub(super) struct IrrStreamingBody { /// Upstream streaming body handle. `None` after cancellation. - upstream: Option, + upstream: Option>, /// Owned state for running step response-body filters. continuation: StepResponseContinuation, /// Whether the stream has finished (EOF or error). finished: bool, + /// Completion-hook body output held until IRR selects a transition. + deferred_completion_output: Option, + /// Per-callback local output waiting to be pulled downstream. + pending_chunks: VecDeque, } impl IrrStreamingBody { /// Create a new streaming body for a step's response. - pub(super) fn new(upstream: SubResponseBody, continuation: StepResponseContinuation) -> Self { + pub(super) fn new(upstream: Box, continuation: StepResponseContinuation) -> Self { Self { upstream: Some(upstream), continuation, finished: false, + deferred_completion_output: None, + pending_chunks: VecDeque::new(), } } + /// Consume the body wrapper after EOF and recover its owned step state. + pub(super) fn into_continuation(self) -> StepResponseContinuation { + self.continuation + } + + /// Consume a finished body into its continuation and deferred output. + fn into_finished_parts(self) -> (StepResponseContinuation, Option) { + (self.continuation, self.deferred_completion_output) + } + + /// Exchange extensions with the outer protocol lifecycle. + fn exchange_extensions(&mut self, extensions: &mut RequestExtensions) { + std::mem::swap(&mut self.continuation.extensions, extensions); + } + /// Run the step's response-body filters on a single chunk. /// /// Reconstructs a temporary `HttpFilterContext` from the @@ -125,9 +263,9 @@ impl IrrStreamingBody { filter_metadata: std::mem::take(&mut cont.filter_metadata), filter_results: std::mem::take(&mut cont.filter_results), filter_state: std::mem::take(&mut cont.filter_state), - health_registry: None, + health_registry: cont.pipeline.health_registry(), id_generator: cont.pipeline.id_generator(), - kv_stores: None, + kv_stores: cont.pipeline.kv_stores(), metrics_route: None, peer_identity: cont.peer_identity.clone(), pre_read_mutations: Vec::new(), @@ -152,8 +290,8 @@ impl IrrStreamingBody { pinned_endpoint_address: None, session_stores: None, structured_metadata: std::mem::take(&mut cont.structured_metadata), - subrequest_client: None, - subrequest_response_mode: crate::context::SubRequestResponseMode::Buffered, + subrequest_client: cont.pipeline.subrequest_client(), + subrequest_response_mode: crate::context::SubRequestResponseMode::Streaming, time_source: cont.pipeline.time_source(), upstream: None, }; @@ -204,18 +342,41 @@ impl IrrStreamingBody { fn handle_upstream_chunk(&mut self, chunk: Bytes) -> Result, FilterError> { let mut body = Some(chunk); self.run_step_body_filters(&mut body, false)?; - Ok(body) + let emitted = self + .continuation + .extensions + .get_mut::() + .map_or_else(VecDeque::new, PendingStreamChunks::drain_chunks); + self.pending_chunks.extend(emitted); + self.pending_chunks.extend(body.filter(|bytes| !bytes.is_empty())); + Ok(self.pending_chunks.pop_front()) + } + + /// Complete the step after a response-body filter failure. + async fn handle_filter_error(&mut self, error: FilterError) -> Result, FilterError> { + if let Some(upstream_body) = self.upstream.take() { + (*upstream_body).cancel().await; + } + self.continuation + .extensions + .insert(StreamTermination::new(StreamTerminationCause::Filter)); + let completion = self.complete_step().map_err(|completion_error| -> FilterError { + format!( + "iterative_request_router: response filter failed ({error}); completion also failed ({completion_error})" + ) + .into() + })?; + self.finished = true; + self.deferred_completion_output = self.handled_completion_output(completion); + Ok(None) } /// Handle upstream EOF. fn handle_upstream_eof(&mut self) -> Result, FilterError> { let completion = self.complete_step()?; self.finished = true; - if completion.as_ref().is_some_and(|b| !b.is_empty()) { - Ok(completion) - } else { - Ok(None) - } + self.deferred_completion_output = completion.filter(|bytes| !bytes.is_empty()); + Ok(None) } /// Handle an upstream error. @@ -223,17 +384,52 @@ impl IrrStreamingBody { &mut self, e: praxis_core::subrequest::SubRequestError, ) -> Result, FilterError> { - self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; } - Err(format!("iterative_request_router: upstream stream error: {e}").into()) + self.continuation + .extensions + .insert(StreamTermination::new(termination_cause(&e))); + let completion = self.complete_step()?; + self.finished = true; + self.deferred_completion_output = self.handled_completion_output(completion); + Ok(None) + } + + /// Expose an abnormal completion body only when a filter explicitly + /// converted the failure into a valid terminal sequence. + fn handled_completion_output(&self, completion: Option) -> Option { + self.continuation + .extensions + .get::() + .is_some_and(StreamTermination::is_handled) + .then_some(completion) + .flatten() + .filter(|bytes| !bytes.is_empty()) + } +} + +/// Map transport detail to the provider-neutral completion classification. +fn termination_cause(error: &praxis_core::subrequest::SubRequestError) -> StreamTerminationCause { + use praxis_core::subrequest::SubRequestError; + match error { + SubRequestError::AdmissionTimeout { .. } => StreamTerminationCause::AdmissionTimeout, + SubRequestError::CircuitOpen { .. } => StreamTerminationCause::CircuitOpen, + SubRequestError::Connect(_) => StreamTerminationCause::Connect, + SubRequestError::DeadlineExceeded => StreamTerminationCause::DeadlineExceeded, + SubRequestError::StreamIdleTimeout { .. } => StreamTerminationCause::IdleTimeout, + SubRequestError::ResponseTooLarge { .. } => StreamTerminationCause::ResponseTooLarge, + _ => StreamTerminationCause::Io, } } #[async_trait] impl StreamingResponseBody for IrrStreamingBody { + #[expect(clippy::too_many_lines, reason = "pull loop applies deadlines and completion state")] async fn next_chunk(&mut self) -> Result, FilterError> { + if let Some(chunk) = self.pending_chunks.pop_front() { + return Ok(Some(chunk)); + } if self.finished { return Ok(None); } @@ -243,11 +439,24 @@ impl StreamingResponseBody for IrrStreamingBody { "iterative_request_router: upstream already consumed".to_owned().into() })?; - match upstream.next_chunk().await { - Ok(Some(chunk)) => { - if let Some(bytes) = self.handle_upstream_chunk(chunk)? { - return Ok(Some(bytes)); - } + let remaining = self + .continuation + .step_deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_default(); + let next = if remaining.is_zero() { + Err(praxis_core::subrequest::SubRequestError::DeadlineExceeded) + } else { + tokio::time::timeout(remaining, upstream.next_chunk()) + .await + .unwrap_or(Err(praxis_core::subrequest::SubRequestError::DeadlineExceeded)) + }; + + match next { + Ok(Some(chunk)) => match self.handle_upstream_chunk(chunk) { + Ok(Some(bytes)) => return Ok(Some(bytes)), + Ok(None) => {}, + Err(error) => return Box::pin(self.handle_filter_error(error)).await, }, Ok(None) => return self.handle_upstream_eof(), Err(e) => return Box::pin(self.handle_upstream_error(e)).await, @@ -259,7 +468,7 @@ impl StreamingResponseBody for IrrStreamingBody { if !self.finished { self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; } self.complete_step()?; } @@ -270,8 +479,472 @@ impl StreamingResponseBody for IrrStreamingBody { if !self.finished { self.finished = true; if let Some(upstream_body) = self.upstream.take() { - upstream_body.cancel().await; + (*upstream_body).cancel().await; + } + } + } + + fn swap_extensions(&mut self, extensions: &mut RequestExtensions) { + self.exchange_extensions(extensions); + } +} + +/// A committed logical response that can span multiple IRR steps. +pub(super) struct IrrStreamingSession { + /// Active step body, when a streamed step is being consumed. + current: Option, + /// Outcome corresponding to `current`. + current_outcome: Option, + /// Request inherited or replaced for the current step. + current_request: crate::SubRequest, + /// Current configured step name. + current_step: Arc, + /// Whether the logical response reached a terminal state. + done: bool, + /// Unhandled failure returned after final pending chunks drain. + deferred_error: Option, + /// Cumulative downstream payload bytes produced by this session. + emitted_bytes: usize, + /// Extensions held between active steps. + extensions: Option, + /// Whether EOF follows the pending local output queue. + finish_after_pending: bool, + /// Retained-state and pending-output ceiling. + max_state_bytes: usize, + /// Optional logical streamed-response byte ceiling. + max_stream_response_bytes: Option, + /// Step selected after the current completion. + next_step: Option>, + /// Locally emitted or terminal buffered chunks awaiting delivery. + pending_chunks: VecDeque, + /// Reusable one-step executor. + runner: super::runner::IrrStepRunner, + /// Iteration state held between active steps. + state: Option, + /// Ordered transition rules for every named step. + step_transitions: HashMap, Vec>, +} + +impl IrrStreamingSession { + /// Create a logical session from the already opened first streamed step. + #[expect( + clippy::too_many_arguments, + reason = "session owns the complete logical response state" + )] + pub(super) fn new( + runner: super::runner::IrrStepRunner, + current_step: Arc, + current_request: crate::SubRequest, + outcome: super::StepOutcome, + body: Box, + continuation: StepResponseContinuation, + pending_chunks: VecDeque, + step_transitions: HashMap, Vec>, + max_state_bytes: usize, + max_stream_response_bytes: Option, + ) -> Self { + Self { + current: Some(IrrStreamingBody::new(body, continuation)), + current_outcome: Some(outcome), + current_request, + current_step, + done: false, + deferred_error: None, + emitted_bytes: 0, + extensions: None, + finish_after_pending: false, + max_state_bytes, + max_stream_response_bytes, + next_step: None, + pending_chunks, + runner, + state: None, + step_transitions, + } + } + + /// Account for one outgoing chunk against the logical byte ceiling. + fn checked_chunk(&mut self, chunk: Bytes) -> Result, FilterError> { + let total = self + .emitted_bytes + .checked_add(chunk.len()) + .ok_or_else(|| -> FilterError { "iterative_request_router: stream byte count overflow".into() })?; + if self.max_stream_response_bytes.is_some_and(|limit| total > limit) { + return Err("iterative_request_router: logical stream byte limit exceeded" + .to_owned() + .into()); + } + self.emitted_bytes = total; + Ok(Some(chunk)) + } + + /// Transition rules for the current step. + fn transitions(&self) -> &[super::config::StepTransition] { + self.step_transitions.get(&self.current_step).map_or(&[], Vec::as_slice) + } + + /// Persist a completed step and select the next session phase. + #[expect( + clippy::too_many_lines, + reason = "completion handles limits, failures, and transitions" + )] + fn apply_completion( + &mut self, + mut completion: StepCompletion, + outcome: &super::StepOutcome, + completion_output: Option, + terminal_body: Option, + ) -> Result<(), FilterError> { + completion.state.previous_response = Some(outcome.response.clone()); + completion.state.iteration += 1; + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + let transition = super::evaluate_transitions(self.transitions(), outcome, &completion.filter_results); + let abnormal_completion = completion.termination.is_some(); + let unhandled_termination = completion + .termination + .as_ref() + .is_some_and(|termination| !termination.is_handled()); + if unhandled_termination + && matches!( + transition, + super::TransitionResult::Done | super::TransitionResult::NoMatch + ) + { + let cause = completion.termination.as_ref().map(StreamTermination::cause); + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + self.deferred_error = + Some(format!("iterative_request_router: unhandled upstream stream termination: {cause:?}").into()); + return Ok(()); + } + let completion_output = (!abnormal_completion || !matches!(transition, super::TransitionResult::Next(_))) + .then_some(completion_output) + .flatten() + .filter(|body| !body.is_empty()); + if abnormal_completion && matches!(transition, super::TransitionResult::Next(_)) { + completion.pending_chunks.clear(); + } + let terminal_body = matches!( + transition, + super::TransitionResult::Done | super::TransitionResult::NoMatch + ) + .then_some(terminal_body) + .flatten() + .filter(|body| !body.is_empty()); + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks + .iter() + .chain(completion.pending_chunks.iter()) + .chain(completion_output.iter()) + .chain(terminal_body.iter()) + .map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + self.pending_chunks.extend(completion.pending_chunks); + self.pending_chunks.extend(completion_output); + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + match transition { + super::TransitionResult::Next(next) => { + let next_body = completion + .next_iteration_body + .unwrap_or_else(|| self.current_request.body.clone()); + self.current_request = crate::SubRequest { + method: self.current_request.method.clone(), + uri: self.current_request.uri.clone(), + headers: http::HeaderMap::new(), + body: next_body, + }; + self.next_step = Some(next); + }, + super::TransitionResult::Done | super::TransitionResult::NoMatch => { + self.pending_chunks.extend(terminal_body); + self.finish_after_pending = true; + }, + } + Ok(()) + } + + /// Consume the current body after clean or typed completion. + #[expect( + clippy::significant_drop_tightening, + reason = "consuming the body also consumes its continuation" + )] + fn finish_current(&mut self) -> Result<(), FilterError> { + let current = self + .current + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: current stream missing at EOF".into() })?; + let (continuation, completion_output) = current.into_finished_parts(); + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + return Err(error); + }, + }; + let outcome = self + .current_outcome + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: current step outcome missing at EOF".into() })?; + self.apply_completion(completion, &outcome, completion_output, None) + } + + /// Open and classify the next selected step. + #[expect(clippy::too_many_lines, reason = "next step may fail over, buffer, or stream")] + #[expect( + clippy::significant_drop_tightening, + reason = "opened step is destructured across match arms" + )] + #[expect( + clippy::large_stack_frames, + reason = "opening a step reconstructs its filter context" + )] + async fn open_next(&mut self) -> Result<(), FilterError> { + let next = self + .next_step + .take() + .ok_or_else(|| -> FilterError { "iterative_request_router: next step missing".into() })?; + self.current_step = next; + let state = self.state.take().ok_or_else(|| -> FilterError { + "iterative_request_router: iteration state missing between steps".into() + })?; + if state.iteration >= state.max_iterations { + return Err("iterative_request_router: max iterations exhausted".to_owned().into()); + } + let extensions = self.extensions.take().unwrap_or_default(); + let opened = + match Box::pin( + self.runner + .open_step(&self.current_step, &self.current_request, &state, extensions), + ) + .await + { + Ok(opened) => opened, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; + let super::runner::OpenedStep { continuation, kind } = opened; + match kind { + super::runner::OpenedStepKind::Streaming { body, outcome } => { + if !super::streaming_transition_order_is_valid(self.transitions()) { + (*body).cancel().await; + self.extensions = Some(continuation.into_parent_extensions()); + self.state = Some(state); + return Err(format!( + "iterative_request_router: step '{}' selected streaming with interleaved transition phases", + self.current_step + ) + .into()); + } + match super::evaluate_header_transitions(self.transitions(), &outcome) { + super::TransitionResult::Next(next) => { + let mut skipped = IrrStreamingBody::new(body, continuation); + if let Err(error) = skipped.suppress().await { + self.extensions = Some(skipped.into_continuation().into_parent_extensions()); + self.state = Some(state); + return Err(error); + } + let mut completion = match skipped.into_continuation().into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; + completion.state.previous_response = None; + completion.state.iteration += 1; + if let Err(error) = ensure_combined_retained_limit( + completion.state.retained_bytes(), + self.pending_chunks.iter().map(Bytes::len), + self.max_state_bytes, + ) { + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + return Err(error); + } + self.extensions = Some(completion.extensions); + self.state = Some(completion.state); + self.current_request = crate::SubRequest { + method: self.current_request.method.clone(), + uri: self.current_request.uri.clone(), + headers: http::HeaderMap::new(), + body: completion + .next_iteration_body + .unwrap_or_else(|| self.current_request.body.clone()), + }; + self.next_step = Some(next); + }, + super::TransitionResult::Done | super::TransitionResult::NoMatch => { + self.current = Some(IrrStreamingBody::new(body, continuation)); + self.current_outcome = Some(outcome); + }, + } + }, + super::runner::OpenedStepKind::Complete(outcome) => { + let completion = match continuation.into_completion() { + Ok(completion) => completion, + Err(error) => { + let (error, restored_extensions) = error.into_parts(); + self.extensions = Some(restored_extensions); + self.state = Some(state); + return Err(error); + }, + }; + let abnormal_completion = completion.termination.is_some(); + let completion_output = abnormal_completion.then(|| outcome.response.body.clone()); + let terminal_body = (!abnormal_completion).then(|| outcome.response.body.clone()); + self.apply_completion(completion, &outcome, completion_output, terminal_body)?; + }, + } + Ok(()) + } +} + +/// Enforce the shared retained-state and pending-output ceiling using the +/// final state produced by the complete response-filter lifecycle. +pub(super) fn ensure_combined_retained_limit( + state_bytes: usize, + mut chunk_lengths: impl Iterator, + limit: usize, +) -> Result<(), FilterError> { + let retained = chunk_lengths.try_fold(state_bytes, |retained, chunk_len| { + retained + .checked_add(chunk_len) + .ok_or_else(|| -> FilterError { "iterative_request_router: retained state size overflow".into() }) + })?; + if retained > limit { + return Err("iterative_request_router: retained state limit exceeded" + .to_owned() + .into()); + } + Ok(()) +} + +#[async_trait] +impl StreamingResponseBody for IrrStreamingSession { + async fn next_chunk(&mut self) -> Result, FilterError> { + loop { + if let Some(chunk) = self.pending_chunks.pop_front() { + return self.checked_chunk(chunk); } + if let Some(error) = self.deferred_error.take() { + self.done = true; + return Err(error); + } + if self.finish_after_pending || self.done { + self.done = true; + return Ok(None); + } + if let Some(current) = self.current.as_mut() { + if let Some(chunk) = current.next_chunk().await? { + return self.checked_chunk(chunk); + } + self.finish_current()?; + continue; + } + if self.next_step.is_some() { + Box::pin(self.open_next()).await?; + continue; + } + return Err("iterative_request_router: streaming session has no runnable phase" + .to_owned() + .into()); + } + } + + async fn suppress(&mut self) -> Result<(), FilterError> { + loop { + self.pending_chunks.clear(); + if let Some(error) = self.deferred_error.take() { + self.done = true; + return Err(error); + } + if self.finish_after_pending || self.done { + self.done = true; + return Ok(()); + } + if let Some(current) = self.current.as_mut() { + current.suppress().await?; + self.finish_current()?; + continue; + } + if self.next_step.is_some() { + Box::pin(self.open_next()).await?; + continue; + } + return Err( + "iterative_request_router: suppressed streaming session has no runnable phase" + .to_owned() + .into(), + ); + } + } + + async fn cancel(&mut self) { + if let Some(current) = self.current.as_mut() { + current.cancel().await; + } + self.current = None; + self.pending_chunks.clear(); + self.next_step = None; + self.done = true; + } + + fn swap_extensions(&mut self, extensions: &mut RequestExtensions) { + if let Some(current) = self.current.as_mut() { + current.swap_extensions(extensions); + } else if let Some(owned) = self.extensions.as_mut() { + std::mem::swap(owned, extensions); } } } + +#[cfg(test)] +mod tests { + use bytes::Bytes; + + use super::ensure_combined_retained_limit; + + #[test] + fn completion_rechecks_final_state_with_pending_output() { + let chunks = [Bytes::from_static(b"123")]; + let result = ensure_combined_retained_limit(8, chunks.iter().map(Bytes::len), 10); + assert!( + result + .as_ref() + .is_err_and(|error| error.to_string().contains("retained state limit")), + "combined limit failure should identify retained state: {result:?}" + ); + } + + #[test] + fn completion_accepts_exact_combined_retained_limit() { + let chunks = [Bytes::from_static(b"12")]; + let result = ensure_combined_retained_limit(8, chunks.iter().map(Bytes::len), 10); + assert!( + result.is_ok(), + "final state and pending chunks should be accepted at the exact limit: {result:?}" + ); + } +} diff --git a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs index c0fee4481..089dddc06 100644 --- a/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs +++ b/filter/src/builtins/http/traffic_management/iterative_request_router/tests.rs @@ -472,6 +472,28 @@ steps: assert!(err.to_string().contains("max_state_bytes")); } +#[test] +fn rejects_zero_max_stream_response_bytes() { + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: step1 +max_stream_response_bytes: 0 +steps: + - name: step1 + filters: + - filter: static_response + status: 200 +", + ) + .unwrap(); + let cfg: IterativeRequestRouterConfig = parse_filter_config("iterative_request_router", &yaml).unwrap(); + let error = config::validate(&cfg).unwrap_err(); + assert!( + error.to_string().contains("max_stream_response_bytes"), + "zero logical stream limit should be rejected: {error}" + ); +} + #[test] fn accepts_multi_step_config() { let yaml: serde_yaml::Value = serde_yaml::from_str( @@ -834,7 +856,7 @@ fn classify_invalid_request_falls_through_to_io() { #[test] #[expect(clippy::too_many_lines, reason = "YAML config literal")] -fn rejects_filter_result_transition_on_streaming_capable_step() { +fn accepts_completion_result_transition_on_streaming_capable_step() { let mut registry = crate::FilterRegistry::with_builtins(); registry .register( @@ -864,16 +886,48 @@ steps: .unwrap(); let result = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry); assert!( - result.is_err(), - "filter/key/value transition should be rejected for streaming step" + result.is_ok(), + "filter/key/value transition should be evaluated after streaming EOF" + ); +} + +#[test] +#[expect(clippy::too_many_lines, reason = "YAML transition-order fixture")] +fn rejects_header_failover_after_stream_completion_rule() { + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StreamingSelectorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: s +steps: + - name: s + filters: + - filter: test_streaming_selector + - filter: static_response + status: 200 + on_result: + - filter: test_streaming_selector + key: action + value: loop + next: s + - status: [503] + next: s + - default: true + done: true +", + ) + .unwrap(); + let result = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry); + let error = result.err().expect("interleaved streaming transitions must fail"); + assert!( + error.to_string().contains("must precede completion rules"), + "validation error should explain phase ordering: {error}" ); - if let Err(err) = result { - let msg = err.to_string(); - assert!( - msg.contains("streaming") && msg.contains("filter"), - "error should mention streaming and filter: {msg}" - ); - } } #[test] @@ -957,6 +1011,57 @@ fn streaming_runtime_guard_accepts_default_transition() { struct StreamingSelectorFilter; +struct UndeclaredStreamingSelectorFilter; + +#[derive(Debug, PartialEq, Eq)] +struct ParentExtension(&'static str); + +struct StepErrorFilter; + +struct RemoveIterationStateFilter; + +#[async_trait::async_trait] +impl crate::HttpFilter for StepErrorFilter { + fn name(&self) -> &'static str { + "test_step_error" + } + + async fn on_request( + &self, + _ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + Err("nested step failure".to_owned().into()) + } +} + +#[async_trait::async_trait] +impl crate::HttpFilter for RemoveIterationStateFilter { + fn name(&self) -> &'static str { + "test_remove_iteration_state" + } + + async fn on_request( + &self, + _ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + Ok(crate::FilterAction::Continue) + } + + fn response_body_access(&self) -> crate::BodyAccess { + crate::BodyAccess::ReadOnly + } + + fn on_response_body( + &self, + ctx: &mut crate::HttpFilterContext<'_>, + _body: &mut Option, + _end_of_stream: bool, + ) -> Result { + ctx.extensions.remove::(); + Ok(crate::FilterAction::Continue) + } +} + #[async_trait::async_trait] impl crate::HttpFilter for StreamingSelectorFilter { fn name(&self) -> &'static str { @@ -976,6 +1081,21 @@ impl crate::HttpFilter for StreamingSelectorFilter { } } +#[async_trait::async_trait] +impl crate::HttpFilter for UndeclaredStreamingSelectorFilter { + fn name(&self) -> &'static str { + "test_undeclared_streaming_selector" + } + + async fn on_request( + &self, + ctx: &mut crate::HttpFilterContext<'_>, + ) -> Result { + ctx.set_subrequest_response_mode(crate::SubRequestResponseMode::Streaming); + Ok(crate::FilterAction::Continue) + } +} + /// Build a default-done transition. fn make_default_done() -> config::StepTransition { config::StepTransition { @@ -1647,6 +1767,283 @@ async fn on_request_no_connector() { assert!(result.is_err(), "no connector should return error"); } +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "nested pipeline setup and ownership assertions")] +async fn step_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_step_error", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StepErrorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str( + " +initial_step: failing +steps: + - name: failing + filters: + - filter: test_step_error + on_result: + - default: true + done: true +", + ) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!(result.is_err(), "the nested step error should propagate"); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a nested step error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "nested pipeline setup and ownership assertions")] +async fn completion_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_remove_iteration_state", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(RemoveIterationStateFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: completion_error +steps: + - name: completion_error + filters: + - filter: test_remove_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!( + result.as_ref().is_err_and(|error| error + .to_string() + .contains("iteration state missing after step completion")), + "missing completion state should propagate an invariant error: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a completion conversion error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "streaming pipeline setup and ownership assertions")] +async fn streaming_completion_error_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(StreamingSelectorFilter)))), + ) + .unwrap(); + registry + .register( + "test_remove_iteration_state", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(RemoveIterationStateFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: completion_error +steps: + - name: completion_error + filters: + - filter: test_streaming_selector + - filter: test_remove_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let action = filter.on_request(&mut ctx).await.unwrap(); + let action_debug = format!("{action:?}"); + let terminal = match action { + crate::FilterAction::StreamingTerminalResponse(terminal) => Some(terminal), + _ => None, + }; + assert!( + terminal.is_some(), + "streaming selector should return a terminal stream: {action_debug}" + ); + let Some(mut terminal) = terminal else { + return; + }; + let first = terminal.body.next_chunk().await.unwrap(); + assert_eq!( + first, + Some(bytes::Bytes::from_static(b"x")), + "upstream chunk should arrive first" + ); + let result = terminal.body.next_chunk().await; + terminal.body.swap_extensions(&mut ctx.extensions); + + assert!( + result.as_ref().is_err_and(|error| error + .to_string() + .contains("iteration state missing after step completion")), + "missing streaming completion state should propagate an invariant error: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the outer request extension must survive a streaming completion error" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "runtime guard setup and ownership assertions")] +async fn streaming_runtime_guard_restores_parent_request_extensions() { + use praxis_core::subrequest::{SubRequestClient, SubRequestConnector}; + + let backend_port = start_unit_stream_backend(); + let mut registry = crate::FilterRegistry::with_builtins(); + registry + .register( + "test_undeclared_streaming_selector", + crate::FilterFactory::Http(std::sync::Arc::new(|_| Ok(Box::new(UndeclaredStreamingSelectorFilter)))), + ) + .unwrap(); + let yaml: serde_yaml::Value = serde_yaml::from_str(&format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_undeclared_streaming_selector + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true + - status: [502] + next: stream +"# + )) + .unwrap(); + let filter = super::IterativeRequestRouterFilter::from_config_with_registry(&yaml, ®istry).unwrap(); + let request = crate::test_utils::make_request(http::Method::POST, "/v1/responses"); + let client = SubRequestClient::new(SubRequestConnector::new(1, None)); + let mut ctx = crate::test_utils::make_filter_context(&request); + ctx.buffered_request_body = Some(bytes::Bytes::from_static(b"request")); + ctx.subrequest_client = Some(&client); + ctx.extensions.insert(ParentExtension("preserved")); + + let result = filter.on_request(&mut ctx).await; + + assert!( + result + .as_ref() + .is_err_and(|error| error.to_string().contains("interleaved transition phases")), + "the runtime guard should reject a misdeclared streaming selector: {result:?}" + ); + assert_eq!( + ctx.extensions.get::(), + Some(&ParentExtension("preserved")), + "the runtime guard must restore outer request extensions" + ); + assert!( + ctx.extensions.get::().is_none(), + "IRR-private iteration state must not escape into the parent context" + ); +} + +fn start_unit_stream_backend() -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + let _sent = stream.write_all( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n1\r\nx\r\n0\r\n\r\n", + ); + }); + port +} + // --------------------------------------------------------------------------- // on_request_body - not end_of_stream // --------------------------------------------------------------------------- @@ -2122,6 +2519,15 @@ fn listener_response_limit_clamps_router_limit() { assert_eq!(super::effective_response_limit(4, crate::BodyMode::Stream), 4); } +#[test] +fn streaming_transport_uses_only_listener_limit() { + assert_eq!( + super::streaming_transport_limit(crate::BodyMode::SizeLimit { max_bytes: 4 }), + Some(4) + ); + assert_eq!(super::streaming_transport_limit(crate::BodyMode::Stream), None); +} + #[test] fn strip_request_framing_headers_removes_stale_lengths() { let mut headers = HeaderMap::new(); @@ -3382,7 +3788,7 @@ fn sneaky_registry() -> crate::FilterRegistry { } #[tokio::test] -async fn runtime_guard_rejects_streaming_with_body_dependent_transitions() { +async fn runtime_guard_rejects_streaming_with_interleaved_transition_phases() { let (addr, backend) = spawn_raw_backend("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n").await; let yaml = format!( " @@ -3393,12 +3799,10 @@ steps: - filter: test_sneaky_streaming {} on_result: - - filter: router - key: matched - value: \"true\" - done: true - default: true done: true + - status: [502] + next: s ", routed_step_yaml(addr) ); @@ -3412,8 +3816,8 @@ steps: backend.abort(); assert!( - err.to_string().contains("body-dependent transitions"), - "runtime streaming with body-dependent transitions must error: {err}" + err.to_string().contains("interleaved transition phases"), + "runtime streaming with interleaved transitions must error: {err}" ); } @@ -3445,7 +3849,7 @@ steps: backend.abort(); assert!( - err.to_string().contains("StreamBuffer response body mode"), + err.to_string().contains("StreamBuffer response mode"), "runtime streaming with StreamBuffer response mode must error: {err}" ); } diff --git a/filter/src/context.rs b/filter/src/context.rs index 14b51048d..533a843ca 100644 --- a/filter/src/context.rs +++ b/filter/src/context.rs @@ -3,7 +3,14 @@ //! Transport-agnostic HTTP request/response metadata and per-request filter context. -use std::{any::Any, borrow::Cow, collections::HashMap, net::IpAddr, sync::Arc, time::Instant}; +use std::{ + any::Any, + borrow::Cow, + collections::{HashMap, VecDeque}, + net::IpAddr, + sync::Arc, + time::Instant, +}; use http::{HeaderMap, Method, StatusCode, Uri, header::HeaderName}; use praxis_core::{ @@ -11,7 +18,42 @@ use praxis_core::{ }; use praxis_tls::TlsPeerIdentity; -use crate::{body::BodyMode, extensions::RequestExtensions, pipeline::body::merge_body_mode, results::FilterResultSet}; +use crate::{ + FilterError, IterationState, body::BodyMode, extensions::RequestExtensions, pipeline::body::merge_body_mode, + results::FilterResultSet, +}; + +/// Bounded opaque chunks emitted by filters while IRR owns a logical stream. +pub(crate) struct PendingStreamChunks { + /// FIFO ordering of locally emitted opaque chunks. + chunks: VecDeque, + /// Combined iteration-state and pending-output ceiling. + max_retained_bytes: usize, + /// Bytes currently retained in `chunks`. + retained_bytes: usize, +} + +impl PendingStreamChunks { + /// Create an empty bounded pending-output queue. + pub(crate) fn new(max_retained_bytes: usize) -> Self { + Self { + chunks: VecDeque::new(), + max_retained_bytes, + retained_bytes: 0, + } + } + + /// Consume the accounting wrapper and return its FIFO queue. + pub(crate) fn into_chunks(self) -> VecDeque { + self.chunks + } + + /// Drain queued chunks and reset their retained-byte accounting. + pub(crate) fn drain_chunks(&mut self) -> VecDeque { + self.retained_bytes = 0; + std::mem::take(&mut self.chunks) + } +} // ----------------------------------------------------------------------------- // Constants @@ -109,6 +151,53 @@ pub enum SubRequestResponseMode { Streaming, } +/// Provider-neutral reason an owned streaming source terminated abnormally. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StreamTerminationCause { + /// Admission capacity was not acquired in time. + AdmissionTimeout, + /// The peer circuit breaker rejected the exchange. + CircuitOpen, + /// Connection establishment failed. + Connect, + /// The overall or per-step deadline expired. + DeadlineExceeded, + /// The upstream produced no bytes within its idle budget. + IdleTimeout, + /// Transport I/O failed after connection establishment. + Io, + /// A response-body filter failed after commitment. + Filter, + /// A configured response byte ceiling was exceeded. + ResponseTooLarge, +} + +/// Typed abnormal termination exposed to streaming completion filters. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StreamTermination { + /// Framework-level failure classification. + cause: StreamTerminationCause, + /// Set by a completion filter that produced a valid terminal sequence. + handled: bool, +} + +impl StreamTermination { + /// Create an unhandled termination value. + pub(crate) fn new(cause: StreamTerminationCause) -> Self { + Self { cause, handled: false } + } + + /// The provider-neutral termination classification. + pub fn cause(&self) -> StreamTerminationCause { + self.cause + } + + /// Whether a completion filter converted the failure into final bytes. + pub fn is_handled(&self) -> bool { + self.handled + } +} + // ----------------------------------------------------------------------------- // HttpFilterContext // ----------------------------------------------------------------------------- @@ -399,6 +488,72 @@ impl HttpFilterContext<'_> { self.subrequest_response_mode = mode; } + /// Emit an opaque response chunk into the IRR-owned logical stream. + /// + /// Emission is available only inside an iterative session. A buffered + /// intermediate step may retain chunks for a later streaming step; if the + /// iteration instead terminates with a buffered response, IRR rejects the + /// pending chunks. Chunks emitted by a streaming response-body callback are + /// delivered in FIFO order before that callback's body output. Pending + /// chunks are bounded together with retained [`IterationState`]; exceeding + /// that bound returns an error and enqueues nothing. Praxis does not inspect + /// or reinterpret the bytes. + /// + /// # Errors + /// + /// Returns an error outside an IRR step or when the retained-state limit + /// would be exceeded. + pub fn emit_stream_chunk(&mut self, bytes: bytes::Bytes) -> Result<(), FilterError> { + let state_bytes = self + .extensions + .get::() + .map_or(0, IterationState::retained_bytes); + let pending = self + .extensions + .get_mut::() + .ok_or_else(|| -> FilterError { + "stream chunk emission is only available inside iterative_request_router" + .to_owned() + .into() + })?; + let retained = state_bytes + .checked_add(pending.retained_bytes) + .and_then(|value| value.checked_add(bytes.len())) + .ok_or_else(|| -> FilterError { "stream chunk retained-state size overflow".to_owned().into() })?; + if retained > pending.max_retained_bytes { + return Err(format!( + "stream chunk emission exceeds retained-state limit ({} > {})", + retained, pending.max_retained_bytes + ) + .into()); + } + pending.retained_bytes += bytes.len(); + pending.chunks.push_back(bytes); + Ok(()) + } + + /// Abnormal source termination visible during a streaming completion hook. + pub fn stream_termination(&self) -> Option<&StreamTermination> { + self.extensions.get::() + } + + /// Mark the current abnormal stream termination as converted to a valid + /// provider-specific terminal sequence by this filter. + /// + /// Returns `false` when the step is completing normally. + pub fn mark_stream_termination_handled(&mut self) -> bool { + let Some(termination) = self.extensions.get_mut::() else { + return false; + }; + termination.handled = true; + true + } + + /// Enable bounded local stream emission for an IRR step. + pub(crate) fn enable_stream_chunk_emission(&mut self, max_retained_bytes: usize) { + self.extensions.insert(PendingStreamChunks::new(max_retained_bytes)); + } + /// Read a durable metadata value by key. pub fn get_metadata(&self, key: &str) -> Option<&str> { self.filter_metadata.get(key).map(String::as_str) @@ -1756,4 +1911,51 @@ mod tests { "merge should drop new key past limit" ); } + + #[test] + fn stream_chunk_emission_is_bounded() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + ctx.enable_stream_chunk_emission(5); + ctx.emit_stream_chunk(bytes::Bytes::from_static(b"12345")).unwrap(); + let error = ctx.emit_stream_chunk(bytes::Bytes::from_static(b"6")).unwrap_err(); + assert!( + error.to_string().contains("retained-state limit"), + "overflow should report the retained-state limit: {error}" + ); + } + + #[test] + fn stream_chunk_emission_requires_irr_session() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + let error = ctx.emit_stream_chunk(bytes::Bytes::from_static(b"event")).unwrap_err(); + assert!( + error + .to_string() + .contains("only available inside iterative_request_router"), + "out-of-session emission should be rejected: {error}" + ); + } + + #[test] + fn stream_termination_requires_explicit_handling() { + let req = crate::test_utils::make_request(Method::GET, "/"); + let mut ctx = crate::test_utils::make_filter_context(&req); + ctx.extensions + .insert(StreamTermination::new(StreamTerminationCause::IdleTimeout)); + assert_eq!( + ctx.stream_termination().map(StreamTermination::cause), + Some(StreamTerminationCause::IdleTimeout), + "completion filters should see the typed cause" + ); + assert!( + ctx.mark_stream_termination_handled(), + "an abnormal completion should be markable as handled" + ); + assert!( + ctx.stream_termination().is_some_and(StreamTermination::is_handled), + "handled state should persist for the session" + ); + } } diff --git a/filter/src/filter.rs b/filter/src/filter.rs index 44b9c1850..daaa8bcaa 100644 --- a/filter/src/filter.rs +++ b/filter/src/filter.rs @@ -14,6 +14,7 @@ use crate::{ actions::FilterAction, body::{BodyAccess, BodyMode}, builtins::http::payload_processing::compression_config::CompressionConfig, + pipeline::FilterPipeline, }; // ----------------------------------------------------------------------------- @@ -118,6 +119,14 @@ pub trait HttpFilter: Send + Sync { false } + /// Visit pipelines owned by this filter. + /// + /// Framework filters that embed nested pipelines override this hook so + /// server-provided runtime resources are propagated into those pipelines. + /// Ordinary filters should use the default no-op implementation. + #[doc(hidden)] + fn visit_nested_pipelines(&mut self, _visitor: &mut dyn FnMut(&mut FilterPipeline)) {} + /// Called for each response, in reverse pipeline order. /// /// Default: [`FilterAction::Continue`] diff --git a/filter/src/lib.rs b/filter/src/lib.rs index 38652acf5..e1734d7ba 100644 --- a/filter/src/lib.rs +++ b/filter/src/lib.rs @@ -76,7 +76,8 @@ pub use builtins::{ pub use builtins::{PolicyFilter, PolicyPluginFactoryFn, register_policy_plugin_factory}; pub use condition::{should_execute, should_execute_response, should_execute_response_ref}; pub use context::{ - HttpFilterContext, PendingHeaderResult, Request, Response, SubRequestResponseMode, TrustedHeaderMutation, + HttpFilterContext, PendingHeaderResult, Request, Response, StreamTermination, StreamTerminationCause, + SubRequestResponseMode, TrustedHeaderMutation, }; pub use error_response::{ ErrorResponseContext, ErrorResponseFormatter, ErrorResponseFormatterHandle, FormattedErrorResponse, diff --git a/filter/src/pipeline/build.rs b/filter/src/pipeline/build.rs index 5cd15c234..3207ed9e3 100644 --- a/filter/src/pipeline/build.rs +++ b/filter/src/pipeline/build.rs @@ -100,24 +100,29 @@ impl FilterPipeline { let compression = extract_compression_config(&filters); let may_select_streaming_subrequest_response = filters_may_select_streaming_subrequest_response(&filters); let (request_body_filter_indices, response_body_filter_indices) = body_filter_indices(&filters); - Self { + let id_generator = Arc::new(IdGenerator::new()); + let time_source: Arc = Arc::new(SystemTimeSource); + let mut pipeline = Self { body_capabilities, compression, filters, request_body_filter_indices, response_body_filter_indices, health_registry: None, - id_generator: Arc::new(IdGenerator::new()), + id_generator: Arc::clone(&id_generator), kv_stores: None, session_stores: None, pipeline_extensions: Vec::new(), record_filter_duration_metrics: false, subrequest_client: None, may_select_streaming_subrequest_response, - time_source: Arc::new(SystemTimeSource), + time_source: Arc::clone(&time_source), request_body_ceiling: None, response_body_ceiling: None, - } + }; + pipeline.set_id_generator(id_generator); + pipeline.set_time_source(time_source); + pipeline } /// Validate the pipeline for structural misconfigurations that diff --git a/filter/src/pipeline/mod.rs b/filter/src/pipeline/mod.rs index c75196d69..a2d304016 100644 --- a/filter/src/pipeline/mod.rs +++ b/filter/src/pipeline/mod.rs @@ -176,6 +176,8 @@ impl FilterPipeline { max_response: Option, allow_unbounded: bool, ) -> Result<(), FilterError> { + self.apply_nested_body_limits(max_request, max_response, allow_unbounded)?; + if let Some(ceiling) = max_request { self.body_capabilities.request_body_mode = clamp_body_mode( self.body_capabilities.request_body_mode, @@ -211,6 +213,27 @@ impl FilterPipeline { Ok(()) } + /// Apply the listener ceilings recursively to nested pipelines. + fn apply_nested_body_limits( + &mut self, + max_request: Option, + max_response: Option, + allow_unbounded: bool, + ) -> Result<(), FilterError> { + let mut nested_error = None; + self.visit_nested_pipelines(&mut |pipeline| { + if nested_error.is_none() + && let Err(error) = pipeline.apply_body_limits(max_request, max_response, allow_unbounded) + { + nested_error = Some(error); + } + }); + if let Some(error) = nested_error { + return Err(error); + } + Ok(()) + } + /// Global request body ceiling; `None` means unbounded was allowed. #[must_use] pub fn request_body_ceiling(&self) -> Option { @@ -290,11 +313,13 @@ impl FilterPipeline { /// Set the shared [`HealthRegistry`] for this pipeline. pub fn set_health_registry(&mut self, registry: HealthRegistry) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_health_registry(Arc::clone(®istry))); self.health_registry = Some(registry); } /// Enable or disable recording of per-filter duration metrics. pub fn set_record_filter_duration_metrics(&mut self, enabled: bool) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_record_filter_duration_metrics(enabled)); self.record_filter_duration_metrics = enabled; } @@ -315,6 +340,7 @@ impl FilterPipeline { /// Override the [`IdGenerator`] for this pipeline. pub fn set_id_generator(&mut self, generator: Arc) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_id_generator(Arc::clone(&generator))); self.id_generator = generator; } @@ -325,6 +351,7 @@ impl FilterPipeline { /// Set the shared [`KvStoreRegistry`] for this pipeline. pub fn set_kv_stores(&mut self, stores: KvStoreRegistry) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_kv_stores(stores.clone())); self.kv_stores = Some(stores); } @@ -353,6 +380,7 @@ impl FilterPipeline { /// /// [`SubRequestClient`]: praxis_core::subrequest::SubRequestClient pub fn set_subrequest_client(&mut self, client: praxis_core::subrequest::SubRequestClient) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_subrequest_client(client.clone())); self.subrequest_client = Some(client); } @@ -386,6 +414,7 @@ impl FilterPipeline { /// Override the [`TimeSource`] for this pipeline. pub fn set_time_source(&mut self, source: Arc) { + self.visit_nested_pipelines(&mut |pipeline| pipeline.set_time_source(Arc::clone(&source))); self.time_source = source; } @@ -427,6 +456,15 @@ impl FilterPipeline { } } } + + /// Apply a mutation to every pipeline directly embedded by a filter. + fn visit_nested_pipelines(&mut self, visitor: &mut dyn FnMut(&mut FilterPipeline)) { + for pf in &mut self.filters { + if let crate::any_filter::AnyFilter::Http(filter) = &mut pf.filter { + filter.visit_nested_pipelines(visitor); + } + } + } } // ----------------------------------------------------------------------------- diff --git a/protocol/src/http/pingora/handler/request_filter/mod.rs b/protocol/src/http/pingora/handler/request_filter/mod.rs index 93b2b9f91..fdd9a5430 100644 --- a/protocol/src/http/pingora/handler/request_filter/mod.rs +++ b/protocol/src/http/pingora/handler/request_filter/mod.rs @@ -595,6 +595,7 @@ async fn run_streaming_terminal_response( headers, body: mut streaming_body, } = terminal; + streaming_body.swap_extensions(&mut ctx.extensions); let mut resp = match prepare_terminal_response(pipeline, ctx, status, headers).await { Ok(resp) => resp, Err(rejection) => { @@ -603,9 +604,11 @@ async fn run_streaming_terminal_response( return; }, }; + streaming_body.swap_extensions(&mut ctx.extensions); if matches!(ctx.response_body_mode, BodyMode::StreamBuffer { .. }) { error!("streaming terminal response is incompatible with StreamBuffer response mode"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; send_rejection(session, Rejection::status(500)).await; return; @@ -624,20 +627,45 @@ async fn run_streaming_terminal_response( let http_version = session.req_header().version; prepare_streaming_headers(&mut resp, false, false, http_version); let Some(header) = build_streaming_terminal_header(&resp) else { + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; send_rejection(session, Rejection::status(500)).await; return; }; if let Err(e) = session.write_response_header(Box::new(header), false).await { debug!(error = %e, "failed to write streaming terminal response header"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; return; } + // A client may validly half-close its HTTP/1 write side after sending the + // request while continuing to read the response. Keep FIN distinct from a + // real disconnect; resets and failed response writes still abort promptly. + session.as_downstream_mut().set_abort_on_close(false); loop { - match streaming_body.next_chunk().await { + // Cancellation contract: dropping an in-flight `next_chunk()` also + // drops any partially opened step resources. A constructed + // `SubResponseBody` schedules protocol-aware cleanup from `Drop`; + // earlier transport state releases its permit, circuit guard, and + // session through ordinary RAII ownership. + let source_result = tokio::select! { + result = streaming_body.next_chunk() => Some(result), + downstream = session.as_downstream_mut().read_body_or_idle(true) => { + debug!(?downstream, "downstream disconnected while terminal stream source was pending"); + None + }, + }; + let Some(source_result) = source_result else { + streaming_body.swap_extensions(&mut ctx.extensions); + streaming_body.cancel().await; + session.as_downstream_mut().shutdown().await; + return; + }; + match source_result { Ok(Some(chunk)) => { + streaming_body.swap_extensions(&mut ctx.extensions); let mut body = Some(chunk); if run_parent_terminal_body_filters(pipeline, ctx, &resp, &mut body, false).is_err() || streaming_size_limit_exceeded(ctx, pipeline) @@ -646,14 +674,19 @@ async fn run_streaming_terminal_response( session.as_downstream_mut().shutdown().await; return; } + streaming_body.swap_extensions(&mut ctx.extensions); if let Err(e) = session.write_response_body(body, false).await { debug!(error = %e, "failed to write streaming terminal response body"); + streaming_body.swap_extensions(&mut ctx.extensions); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; return; } }, Ok(None) => { + // Restore the default before a clean keep-alive session can be reused. + session.as_downstream_mut().set_abort_on_close(true); + streaming_body.swap_extensions(&mut ctx.extensions); let mut completion_body = None; if run_parent_terminal_body_filters(pipeline, ctx, &resp, &mut completion_body, true).is_err() { streaming_body.cancel().await; @@ -667,6 +700,7 @@ async fn run_streaming_terminal_response( return; }, Err(e) => { + streaming_body.swap_extensions(&mut ctx.extensions); warn!(error = %e, "streaming terminal response source failed after commitment"); streaming_body.cancel().await; session.as_downstream_mut().shutdown().await; @@ -694,11 +728,6 @@ async fn suppress_streaming_terminal_response( send_rejection(session, Rejection::status(500)).await; return; } - // This path handles HEAD and 204/304 — bodyless responses with no body - // phase. Running the body-EOS hook would make access_log emit a second - // record after its on_response already logged the bodyless completion - // (mirrors the buffered terminal path). Mark delivery complete so the - // logging-phase fallback does not fire a third. ctx.response_delivery_complete = true; let is_not_modified = resp.status == http::StatusCode::NOT_MODIFIED; diff --git a/tests/integration/tests/suite/iterative_request_router.rs b/tests/integration/tests/suite/iterative_request_router.rs index 6a994e81a..3d69541bc 100644 --- a/tests/integration/tests/suite/iterative_request_router.rs +++ b/tests/integration/tests/suite/iterative_request_router.rs @@ -12,6 +12,7 @@ use bytes::Bytes; use praxis_core::config::Config; use praxis_filter::{ BodyAccess, BodyMode, FilterAction, FilterError, FilterFactory, FilterRegistry, HttpFilter, HttpFilterContext, + StreamTerminationCause, }; use praxis_test_utils::{ Backend, free_port, http_get, http_post, http_send, json_post, parse_body, parse_header, parse_status, @@ -1271,6 +1272,45 @@ steps: assert_eq!(status, 413, "iteration state above max_state_bytes should be rejected"); } +#[test] +fn terminal_response_rechecks_retained_state_limit() { + let backend = Backend::fixed(&"x".repeat(512)).start_with_shutdown(); + let proxy_port = free_port(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: terminal +max_state_bytes: 256 +steps: + - name: terminal + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy(&config); + + let (status, _) = http_get(proxy.addr(), "/", None); + + assert_eq!( + status, 413, + "the retained terminal response must remain inside max_state_bytes" + ); +} + #[test] fn combined_status_and_filter_result_transition() { let model = start_stateful_backend(vec![(200, "final-answer".to_owned())]); @@ -1636,6 +1676,95 @@ steps: ); } +#[test] +fn nested_response_filter_reserved_header_is_stripped() { + let backend = Backend::fixed("ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = nested_header_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: primary +steps: + - name: primary + filters: + - filter: test_nested_response_headers + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send(proxy.addr(), &http_get_raw("/")); + + assert_eq!(parse_status(&raw), 200, "buffered step should succeed"); + assert!( + raw.contains("x-step-visible: true"), + "ordinary nested response-header mutations should reach the client" + ); + assert!( + !raw.contains("x-praxis-step-private"), + "reserved nested response headers must be stripped after filters" + ); +} + +#[test] +fn nested_response_filter_reserved_header_is_stripped_on_transport_error() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = nested_header_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: primary +steps: + - name: primary + filters: + - filter: test_nested_response_headers + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send(proxy.addr(), &http_get_raw("/")); + + assert_eq!(parse_status(&raw), 502, "transport failure should remain synthetic 502"); + assert!( + raw.contains("x-step-visible: true"), + "ordinary synthetic-response mutations should reach the client" + ); + assert!( + !raw.contains("x-praxis-step-private"), + "reserved synthetic-response headers must be stripped after filters" + ); +} + #[test] fn response_hooks_execute_once() { let backend_port = start_backend("once"); @@ -2033,6 +2162,195 @@ steps: ); } +#[test] +fn streaming_body_callback_emission_precedes_its_body_output() { + let backend = Backend::chunked(vec!["upstream".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: test_per_chunk_emission + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "streaming response should succeed"); + assert_eq!( + parse_body(&raw), + "local|upstream", + "a callback's local emissions must not be delayed until stream completion" + ); +} + +#[test] +fn streaming_completion_emission_precedes_completion_body() { + let backend = Backend::chunked(vec!["upstream|".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: test_completion_emission_and_body + - filter: router + routes: + - path_prefix: "/" + cluster: backend + - filter: load_balancer + clusters: + - name: backend + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "streaming response should succeed"); + assert_eq!( + parse_body(&raw), + "upstream|emitted|completion-body", + "completion emissions must precede the same callback's body output" + ); +} + +#[test] +fn handled_initial_transport_completion_preserves_emission_order() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: unavailable +steps: + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_handled_termination_order + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 502, + "the handled connect failure should retain its status" + ); + assert_eq!( + parse_body(&raw), + "emitted|completion-body", + "pre-commit abnormal completion must preserve callback emission order" + ); +} + +#[test] +fn streaming_step_deadline_includes_header_time() { + let backend_port = start_split_delay_chunked_backend( + std::time::Duration::from_millis(450), + std::time::Duration::from_millis(450), + ); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +timeout_ms: 800 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: slow + - filter: load_balancer + clusters: + - name: slow + endpoints: ["127.0.0.1:{backend_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!(parse_status(&raw), 200, "headers should arrive before the deadline"); + assert_eq!( + parse_body(&raw), + "", + "body time must use only the deadline remaining after headers" + ); +} + #[test] fn streaming_header_failover_cancels_unread_body() { let failing = Backend::status(503, "fail").start_with_shutdown(); @@ -2048,6 +2366,7 @@ steps: - name: primary filters: - filter: test_streaming_selector + - filter: test_stream_completion - filter: router routes: - path_prefix: "/" @@ -2132,15 +2451,822 @@ steps: ); } -// --------------------------------------------------------------------------- -// Test-Only Streaming Selector Filter -// --------------------------------------------------------------------------- - -struct IntegrationStreamingSelectorFilter; - -#[async_trait::async_trait] -impl HttpFilter for IntegrationStreamingSelectorFilter { - fn name(&self) -> &'static str { +#[test] +fn streaming_completion_transition_resumes_second_step_in_same_response() { + let first = Backend::chunked(vec!["first-a|".to_owned(), "first-b|".to_owned()]).start_with_shutdown(); + let second = Backend::chunked(vec!["second-a|".to_owned(), "second-b".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_stream_response_bytes: 65536 +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - default: true + done: true + - name: second + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "logical stream should retain initial status"); + assert_eq!( + parse_body(&raw), + "first-a|first-b|between|second-a|second-b", + "both upstream streams and the local completion chunk should share one ordered response" + ); +} + +#[test] +fn streaming_handoff_checks_pending_chunks_with_updated_state() { + let first = Backend::fixed("first").start_with_shutdown(); + let second = Backend::chunked(vec!["second".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + registry + .register( + "test_fill_iteration_state", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(FillIterationStateFilter)))), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_state_bytes: 2048 +steps: + - name: first + filters: + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - name: second + filters: + - filter: test_always_streaming + - filter: test_fill_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 413, + "updated state and carried completion chunks must share one retained-byte ceiling" + ); +} + +#[test] +fn buffered_transition_pending_limit_returns_413() { + let first = Backend::fixed("first").start_with_shutdown(); + let second = Backend::fixed("second").start_with_shutdown(); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + registry + .register( + "test_fill_iteration_state", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(FillIterationStateFilter)))), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +max_state_bytes: 2048 +steps: + - name: first + filters: + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - name: second + filters: + - filter: test_fill_iteration_state + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + next: final + - name: final + filters: + - filter: static_response + status: 200 + body: unreachable + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + + assert_eq!( + parse_status(&raw), + 413, + "pre-commit pending-output overflow must remain a bounded-request rejection" + ); +} + +#[test] +fn streaming_head_suppression_still_runs_completion_transition() { + let first = Backend::chunked(vec!["first".to_owned()]).start_with_shutdown(); + let second = Backend::chunked(vec!["second".to_owned()]).start_with_shutdown(); + let second_step_calls = Arc::new(AtomicUsize::new(0)); + let proxy_port = free_port(); + let mut registry = streaming_registry(); + let counter = Arc::clone(&second_step_calls); + registry + .register( + "test_request_counter", + FilterFactory::Http(Arc::new(move |_| { + Ok(Box::new(IntegrationRequestCounterFilter { + count: Arc::clone(&counter), + })) + })), + ) + .unwrap(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: second + - default: true + done: true + - name: second + filters: + - filter: test_request_counter + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: second + - filter: load_balancer + clusters: + - name: second + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + second.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "HEAD / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "HEAD should retain the committed logical status" + ); + assert_eq!(parse_body(&raw), "", "HEAD must not expose logical stream bytes"); + assert_eq!( + second_step_calls.load(Ordering::SeqCst), + 1, + "suppression must still evaluate completion transitions and execute the next step" + ); +} + +#[test] +fn buffered_response_limit_does_not_cap_streaming_transport() { + let backend = Backend::chunked(vec!["12345".to_owned(), "67890".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +max_response_bytes: 5 +max_stream_response_bytes: 20 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: stream + - filter: load_balancer + clusters: + - name: stream + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "streaming response should commit normally"); + assert_eq!( + parse_body(&raw), + "1234567890", + "the buffered per-step limit must not truncate a live stream" + ); +} + +#[test] +fn streaming_logical_byte_limit_terminates_after_exact_boundary() { + let backend = + Backend::chunked(vec!["12345".to_owned(), "67890".to_owned(), "overflow".to_owned()]).start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: stream +max_stream_response_bytes: 10 +steps: + - name: stream + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: stream + - filter: load_balancer + clusters: + - name: stream + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + backend.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "headers are committed before the late limit failure" + ); + assert_eq!( + parse_body(&raw), + "1234567890", + "the chunk crossing the logical byte ceiling must not be exposed" + ); +} + +#[test] +fn streaming_resumed_transport_failure_can_complete_with_typed_termination() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_termination + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"#, + first.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "the first step should commit the logical response" + ); + assert_eq!( + parse_body(&raw), + "first|between|handled-connect", + "the resumed step completion hook should classify and handle the connect failure" + ); +} + +#[test] +fn streaming_resumed_transport_failure_can_fail_over() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - origin: transport + transport_error: connect + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "the first step should commit the response"); + assert_eq!( + parse_body(&raw), + "first|between|fallback-ok", + "the failed resumed step should transition without leaking its completion output" + ); +} + +#[test] +fn streaming_mid_body_failure_defers_completion_output_until_transition() { + let incomplete_port = start_incomplete_chunked_backend(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: incomplete +steps: + - name: incomplete + filters: + - filter: test_always_streaming + - filter: test_stream_termination + - filter: router + routes: + - path_prefix: "/" + cluster: incomplete + - filter: load_balancer + clusters: + - name: incomplete + endpoints: ["127.0.0.1:{incomplete_port}"] + on_result: + - filter: test_stream_termination + key: action + value: next + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "initial headers should commit the response"); + assert_eq!( + parse_body(&raw), + "first|xfallback-ok", + "failure completion bytes must be suppressed when the transition selects a fallback" + ); +} + +#[test] +fn streaming_resumed_header_failover_clears_previous_response() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let failing = Backend::status(503, "skip-me").start_with_shutdown(); + let fallback = Backend::fixed("fallback-ok").start_with_shutdown(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: failing + - default: true + done: true + - name: failing + filters: + - filter: test_always_streaming + - filter: router + routes: + - path_prefix: "/" + cluster: failing + - filter: load_balancer + clusters: + - name: failing + endpoints: ["127.0.0.1:{}"] + on_result: + - status: [503] + next: fallback + - default: true + done: true + - name: fallback + filters: + - filter: test_previous_response_absent + - filter: router + routes: + - path_prefix: "/" + cluster: fallback + - filter: load_balancer + clusters: + - name: fallback + endpoints: ["127.0.0.1:{}"] + on_result: + - default: true + done: true +"#, + first.port(), + failing.port(), + fallback.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!(parse_status(&raw), 200, "the first step should commit the response"); + assert_eq!( + parse_body(&raw), + "first|between|fallback-ok", + "the header-failover target must not inherit the previous completed response" + ); +} + +#[test] +fn streaming_unhandled_transport_failure_discards_completion_output() { + let first = Backend::chunked(vec!["first|".to_owned()]).start_with_shutdown(); + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: first +steps: + - name: first + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: first + - filter: load_balancer + clusters: + - name: first + endpoints: ["127.0.0.1:{}"] + on_result: + - filter: test_stream_completion + key: action + value: next + next: unavailable + - default: true + done: true + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"#, + first.port() + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 200, + "the first step should commit the logical response" + ); + assert_eq!( + parse_body(&raw), + "first|between|", + "an unhandled failure must not expose completion chunks from the failed step" + ); +} + +#[test] +fn streaming_unhandled_initial_transport_failure_discards_completion_output() { + let unavailable_port = free_port(); + let proxy_port = free_port(); + let registry = streaming_registry(); + let config = Config::from_yaml(&irr_yaml( + proxy_port, + &format!( + r#" +initial_step: unavailable +steps: + - name: unavailable + filters: + - filter: test_always_streaming + - filter: test_stream_completion + - filter: router + routes: + - path_prefix: "/" + cluster: unavailable + - filter: load_balancer + clusters: + - name: unavailable + endpoints: ["127.0.0.1:{unavailable_port}"] + on_result: + - default: true + done: true +"# + ), + )) + .unwrap(); + let proxy = start_full_proxy_with_registry(&config, ®istry); + + let raw = http_send( + proxy.addr(), + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ); + assert_eq!( + parse_status(&raw), + 502, + "an unhandled initial connect failure should remain a 502" + ); + assert_eq!( + parse_body(&raw), + "", + "an unhandled pre-commit failure must not expose completion output" + ); +} + +// --------------------------------------------------------------------------- +// Test-Only Streaming Selector Filter +// --------------------------------------------------------------------------- + +struct IntegrationStreamingSelectorFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamingSelectorFilter { + fn name(&self) -> &'static str { "test_streaming_selector" } @@ -2161,8 +3287,276 @@ impl HttpFilter for IntegrationStreamingSelectorFilter { } } +struct IntegrationAlwaysStreamingFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationAlwaysStreamingFilter { + fn name(&self) -> &'static str { + "test_always_streaming" + } + + fn may_select_streaming_subrequest_response(&self) -> bool { + true + } + + async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + ctx.set_subrequest_response_mode(praxis_filter::SubRequestResponseMode::Streaming); + Ok(FilterAction::Continue) + } +} + +struct IntegrationRequestCounterFilter { + count: Arc, +} + +struct FillIterationStateFilter; + +#[async_trait::async_trait] +impl HttpFilter for FillIterationStateFilter { + fn name(&self) -> &'static str { + "test_fill_iteration_state" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + async fn on_response(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + const LIMIT: usize = 2_048; + const KEY: &str = "fill"; + let state = ctx + .extensions + .get_mut::() + .ok_or_else(|| -> FilterError { "iteration state missing in test filter".into() })?; + let fill_bytes = LIMIT + .checked_sub(state.retained_bytes().saturating_add(KEY.len())) + .ok_or_else(|| -> FilterError { "iteration state already exceeds test limit".into() })?; + state + .accumulator + .insert(KEY.to_owned(), Bytes::from(vec![b'x'; fill_bytes])); + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationRequestCounterFilter { + fn name(&self) -> &'static str { + "test_request_counter" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + self.count.fetch_add(1, Ordering::SeqCst); + Ok(FilterAction::Continue) + } +} + +struct IntegrationStreamCompletionFilter; + +struct IntegrationPerChunkEmissionFilter; + +struct IntegrationCompletionEmissionAndBodyFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationCompletionEmissionAndBodyFilter { + fn name(&self) -> &'static str { + "test_completion_emission_and_body" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadWrite + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream { + ctx.emit_stream_chunk(Bytes::from_static(b"emitted|"))?; + *body = Some(Bytes::from_static(b"completion-body")); + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationHandledTerminationOrderFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationHandledTerminationOrderFilter { + fn name(&self) -> &'static str { + "test_handled_termination_order" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadWrite + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream && ctx.stream_termination().is_some() { + assert!( + ctx.mark_stream_termination_handled(), + "the termination should be available to the completion hook" + ); + ctx.emit_stream_chunk(Bytes::from_static(b"emitted|"))?; + *body = Some(Bytes::from_static(b"completion-body")); + } + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationPerChunkEmissionFilter { + fn name(&self) -> &'static str { + "test_per_chunk_emission" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if !end_of_stream { + ctx.emit_stream_chunk(Bytes::from_static(b"local|"))?; + } + Ok(FilterAction::Continue) + } +} + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamCompletionFilter { + fn name(&self) -> &'static str { + "test_stream_completion" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream { + ctx.filter_results + .entry("test_stream_completion") + .or_default() + .set("action", "next")?; + ctx.emit_stream_chunk(Bytes::from_static(b"between|"))?; + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationStreamTerminationFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationStreamTerminationFilter { + fn name(&self) -> &'static str { + "test_stream_termination" + } + + fn response_body_access(&self) -> BodyAccess { + BodyAccess::ReadOnly + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + fn on_response_body( + &self, + ctx: &mut HttpFilterContext<'_>, + _body: &mut Option, + end_of_stream: bool, + ) -> Result { + if end_of_stream && let Some(cause) = ctx.stream_termination().map(|termination| termination.cause()) { + assert!( + ctx.mark_stream_termination_handled(), + "the abnormal completion should remain available while the hook runs" + ); + ctx.filter_results + .entry("test_stream_termination") + .or_default() + .set("action", "next")?; + let output = match cause { + StreamTerminationCause::Connect => Bytes::from_static(b"handled-connect"), + _ => Bytes::from_static(b"handled-stream-error|"), + }; + ctx.emit_stream_chunk(output)?; + } + Ok(FilterAction::Continue) + } +} + +struct IntegrationPreviousResponseAbsentFilter; + +#[async_trait::async_trait] +impl HttpFilter for IntegrationPreviousResponseAbsentFilter { + fn name(&self) -> &'static str { + "test_previous_response_absent" + } + + async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + let previous_response_present = ctx + .extensions + .get::() + .is_some_and(|state| state.previous_response.is_some()); + if previous_response_present { + return Ok(FilterAction::Reject(praxis_filter::Rejection::status(500))); + } + Ok(FilterAction::Continue) + } +} + fn streaming_registry() -> FilterRegistry { let mut registry = FilterRegistry::with_builtins(); + registry + .register( + "test_completion_emission_and_body", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationCompletionEmissionAndBodyFilter)))), + ) + .unwrap(); + registry + .register( + "test_handled_termination_order", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationHandledTerminationOrderFilter)))), + ) + .unwrap(); + registry + .register( + "test_per_chunk_emission", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationPerChunkEmissionFilter)))), + ) + .unwrap(); registry .register( "test_streaming_selector", @@ -2170,6 +3564,84 @@ fn streaming_registry() -> FilterRegistry { ) .unwrap(); registry + .register( + "test_always_streaming", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationAlwaysStreamingFilter)))), + ) + .unwrap(); + registry + .register( + "test_stream_completion", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationStreamCompletionFilter)))), + ) + .unwrap(); + registry + .register( + "test_stream_termination", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationStreamTerminationFilter)))), + ) + .unwrap(); + registry + .register( + "test_previous_response_absent", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(IntegrationPreviousResponseAbsentFilter)))), + ) + .unwrap(); + registry +} + +fn nested_header_registry() -> FilterRegistry { + let mut registry = FilterRegistry::with_builtins(); + registry + .register( + "test_nested_response_headers", + FilterFactory::Http(Arc::new(|_| Ok(Box::new(NestedResponseHeadersFilter)))), + ) + .unwrap(); + registry +} + +#[expect( + clippy::disallowed_methods, + reason = "blocking backend runs on a dedicated test thread" +)] +fn start_split_delay_chunked_backend(header_delay: std::time::Duration, body_delay: std::time::Duration) -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + std::thread::sleep(header_delay); + let _sent = stream.write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"); + let _flushed = stream.flush(); + std::thread::sleep(body_delay); + let _sent = stream.write_all(b"4\r\nlate\r\n0\r\n\r\n"); + }); + port +} + +fn start_incomplete_chunked_backend() -> u16 { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = [0_u8; 4_096]; + let _read = stream.read(&mut request); + let _sent = stream.write_all( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n6\r\nfirst|\r\n5\r\nx", + ); + let _flushed = stream.flush(); + }); + port } // --------------------------------------------------------------------------- @@ -2226,6 +3698,33 @@ impl HttpFilter for ResponseProbeFilter { struct ResponseTaggerFilter; +struct NestedResponseHeadersFilter; + +#[async_trait::async_trait] +impl HttpFilter for NestedResponseHeadersFilter { + fn name(&self) -> &'static str { + "test_nested_response_headers" + } + + async fn on_request(&self, _ctx: &mut HttpFilterContext<'_>) -> Result { + Ok(FilterAction::Continue) + } + + async fn on_response(&self, ctx: &mut HttpFilterContext<'_>) -> Result { + let response = ctx + .response_header + .as_mut() + .ok_or_else(|| -> FilterError { "response header missing in test filter".into() })?; + response + .headers + .insert("x-step-visible", http::HeaderValue::from_static("true")); + response + .headers + .insert("x-praxis-step-private", http::HeaderValue::from_static("secret")); + Ok(FilterAction::Continue) + } +} + #[async_trait::async_trait] impl HttpFilter for ResponseTaggerFilter { fn name(&self) -> &'static str { diff --git a/tests/integration/tests/suite/streaming_terminal_response.rs b/tests/integration/tests/suite/streaming_terminal_response.rs index 2326a8f4e..70977d245 100644 --- a/tests/integration/tests/suite/streaming_terminal_response.rs +++ b/tests/integration/tests/suite/streaming_terminal_response.rs @@ -11,7 +11,7 @@ use std::{ io::{Read as _, Write as _}, - net::TcpStream, + net::{Shutdown, TcpStream}, sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -336,6 +336,36 @@ fn streaming_h11_chunked_framing() { ); } +#[test] +fn streaming_h11_write_half_close_still_receives_response() { + let backend = start_echo_backend(); + let proxy_port = free_port(); + let config = Config::from_yaml(&custom_filter_yaml(proxy_port, backend.port(), "multi_chunk_streaming")).unwrap(); + let registry = registry_with("multi_chunk_streaming", || Box::new(MultiChunkStreamingFilter)); + let proxy = start_proxy_with_registry(&config, ®istry); + + let mut stream = TcpStream::connect(proxy.addr()).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .unwrap(); + stream.shutdown(Shutdown::Write).unwrap(); + let mut raw = Vec::new(); + stream.read_to_end(&mut raw).unwrap(); + let raw = String::from_utf8(raw).unwrap(); + + assert_eq!( + parse_status(&raw), + 200, + "a valid write half-close must preserve the response" + ); + assert_eq!( + parse_body(&raw), + "chunk1chunk2chunk3", + "the full stream should remain readable after the client half-closes writes" + ); +} + #[test] fn streaming_h10_close_delimited() { let backend = start_echo_backend();