From e9f3b0d15b1d55b962b18e38619d0180250e750e Mon Sep 17 00:00:00 2001
From: Ladislav Smola <lsmola@redhat.com>
Date: Fri, 14 Aug 2026 17:46:04 +0200
Subject: [PATCH 1/5] feat: add extraPortMappings support to ClusterSpec

Extend ClusterSpec with a ports field (reusing PortMapping) that renders
as extraPortMappings in the KIND cluster config. Required for macOS
where MetalLB LoadBalancer IPs are unreachable from the host.

Port mappings are applied to the first control-plane node. Validation
rejects zero ports and duplicate host port bindings.

Signed-off-by: Ladislav Smola <lsmola@redhat.com>
---
 src/cluster/kind.rs               | 114 +++++++++++++++++++++++++-----
 src/cluster/mod.rs                |  15 ++--
 src/command/up.rs                 |  12 ++--
 src/config/mod.rs                 |   7 ++
 src/config/validate.rs            |  81 +++++++++++++++++++++
 src/stack/engine.rs               |   1 +
 src/stack/mod.rs                  |   1 +
 tests/fixtures/port-mappings.yaml |  21 ++++++
 tests/integration.rs              |  19 +++++
 9 files changed, 244 insertions(+), 27 deletions(-)
 create mode 100644 tests/fixtures/port-mappings.yaml

diff --git a/src/cluster/kind.rs b/src/cluster/kind.rs
index 2ab8bb4..a449920 100644
--- a/src/cluster/kind.rs
+++ b/src/cluster/kind.rs
@@ -8,7 +8,7 @@ use std::collections::BTreeMap;
 
 use crate::{
     command::runner::{CommandOutput, CommandRunner, CommandSpec},
-    config::NodeConfig,
+    config::{NodeConfig, PortMapping},
     error::ForgeError,
 };
 
@@ -30,6 +30,18 @@ pub fn kubectl_context(kind_name: &str) -> String {
 // Lifecycle
 // ---------------------------------------------------------------
 
+/// Configuration for creating a KIND cluster.
+pub struct CreateClusterConfig<'a> {
+    /// Node layout for the cluster.
+    pub nodes: &'a NodeConfig,
+    /// Port mappings to add to the first control-plane node.
+    pub ports: &'a [PortMapping],
+    /// Directory where config files are written.
+    pub config_dir: &'a std::path::Path,
+    /// Optional Docker network to join via `KIND_EXPERIMENTAL_DOCKER_NETWORK`.
+    pub docker_network: Option<&'a str>,
+}
+
 /// Check whether a KIND cluster with the given name exists.
 ///
 /// # Errors
@@ -40,10 +52,7 @@ pub fn cluster_exists(runner: &dyn CommandRunner, kind_name: &str) -> Result<boo
     Ok(clusters.iter().any(|c| c == kind_name))
 }
 
-/// Create a KIND cluster with a generated config.
-///
-/// When `docker_network` is `Some`, the cluster nodes join that
-/// Docker network via `KIND_EXPERIMENTAL_DOCKER_NETWORK`.
+/// Create a Kind cluster with the given configuration.
 ///
 /// # Errors
 ///
@@ -51,13 +60,11 @@ pub fn cluster_exists(runner: &dyn CommandRunner, kind_name: &str) -> Result<boo
 pub fn create_cluster(
     runner: &dyn CommandRunner,
     kind_name: &str,
-    nodes: &NodeConfig,
-    config_dir: &std::path::Path,
-    docker_network: Option<&str>,
+    config: &CreateClusterConfig<'_>,
 ) -> Result<(), ForgeError> {
-    let config_yaml = generate_kind_config(nodes);
-    let config_path = write_kind_config(config_dir, kind_name, &config_yaml)?;
-    let result = run_create(runner, kind_name, &config_path, docker_network);
+    let config_yaml = generate_kind_config(config.nodes, config.ports);
+    let config_path = write_kind_config(config.config_dir, kind_name, &config_yaml)?;
+    let result = run_create(runner, kind_name, &config_path, config.docker_network);
     cleanup_kind_config(&config_path);
     result
 }
@@ -130,11 +137,31 @@ pub fn run_kubectl(
 // KIND config generation
 // ---------------------------------------------------------------
 
-/// Generate a KIND cluster config YAML from a [`NodeConfig`].
-pub fn generate_kind_config(nodes: &NodeConfig) -> String {
+/// Generate a Kind cluster config YAML string.
+///
+/// When `ports` is non-empty, `extraPortMappings` entries are added
+/// to the first control-plane node (Kind only supports port mappings
+/// on control-plane nodes).
+pub fn generate_kind_config(nodes: &NodeConfig, ports: &[PortMapping]) -> String {
+    use std::fmt::Write as _;
     let mut yaml = String::from("kind: Cluster\napiVersion: kind.x-k8s.io/v1alpha4\nnodes:\n");
-    for _ in 0..nodes.control_planes {
+    for i in 0..nodes.control_planes {
         yaml.push_str("  - role: control-plane\n");
+        if i == 0 && !ports.is_empty() {
+            yaml.push_str("    extraPortMappings:\n");
+            for pm in ports {
+                let _written = write!(
+                    &mut yaml,
+                    "      - hostPort: {}\n        containerPort: {}\n        protocol: {}\n",
+                    pm.host,
+                    pm.container,
+                    pm.protocol.to_uppercase(),
+                );
+                if let Some(addr) = &pm.bind_address {
+                    let _written = writeln!(&mut yaml, "        listenAddress: \"{addr}\"");
+                }
+            }
+        }
     }
     for _ in 0..nodes.workers {
         yaml.push_str("  - role: worker\n");
@@ -316,7 +343,7 @@ mod tests {
     #[test]
     fn generate_kind_config_default_nodes() {
         let nodes = NodeConfig::default();
-        let yaml = generate_kind_config(&nodes);
+        let yaml = generate_kind_config(&nodes, &[]);
         assert!(yaml.contains("control-plane"), "should have control-plane");
         let cp_count = yaml.matches("control-plane").count();
         assert_eq!(
@@ -332,7 +359,7 @@ mod tests {
             control_planes: 3,
             workers: 2,
         };
-        let yaml = generate_kind_config(&nodes);
+        let yaml = generate_kind_config(&nodes, &[]);
         let cp_count = yaml.matches("control-plane").count();
         let w_count = yaml.matches("worker").count();
         assert_eq!(cp_count, 3, "should have 3 control-planes, got {cp_count}");
@@ -439,4 +466,59 @@ mod tests {
         });
         assert!(!exists, "forge-missing should not exist");
     }
+
+    #[test]
+    fn generate_kind_config_with_port_mappings() {
+        let nodes = NodeConfig::default();
+        let ports = vec![
+            PortMapping {
+                bind_address: None,
+                host: 13000,
+                container: 30300,
+                protocol: "tcp".to_owned(),
+            },
+            PortMapping {
+                bind_address: None,
+                host: 19090,
+                container: 30909,
+                protocol: "tcp".to_owned(),
+            },
+        ];
+        let yaml = generate_kind_config(&nodes, &ports);
+        assert!(
+            yaml.contains("extraPortMappings:"),
+            "should have extraPortMappings"
+        );
+        assert!(yaml.contains("hostPort: 13000"), "should map Grafana port");
+        assert!(
+            yaml.contains("containerPort: 30300"),
+            "should map Grafana node port"
+        );
+        assert!(
+            yaml.contains("hostPort: 19090"),
+            "should map Prometheus port"
+        );
+        assert!(yaml.contains("protocol: TCP"), "should uppercase protocol");
+    }
+
+    #[test]
+    fn generate_kind_config_port_mappings_only_on_first_control_plane() {
+        let nodes = NodeConfig {
+            control_planes: 2,
+            workers: 0,
+        };
+        let ports = vec![PortMapping {
+            bind_address: None,
+            host: 8080,
+            container: 30080,
+            protocol: "tcp".to_owned(),
+        }];
+        let yaml = generate_kind_config(&nodes, &ports);
+        // extraPortMappings should appear exactly once
+        assert_eq!(
+            yaml.matches("extraPortMappings:").count(),
+            1,
+            "port mappings should only appear on the first control-plane node"
+        );
+    }
 }
diff --git a/src/cluster/mod.rs b/src/cluster/mod.rs
index a0db0c8..9120d9f 100644
--- a/src/cluster/mod.rs
+++ b/src/cluster/mod.rs
@@ -62,7 +62,7 @@ fn handle_create(
     }
     let _lock = lock::acquire(&ctx.state_dir)?;
     let mut state = state::load(&ctx.state_dir)?;
-    create_if_missing(ctx, &kind_name, &cluster.nodes, &mut state, name)?;
+    create_if_missing(ctx, &kind_name, cluster, &mut state)?;
     state::save(&ctx.state_dir, &state)?;
     report_created(writer, name, &kind_name, &ctx.format)
 }
@@ -168,15 +168,20 @@ fn cluster_kind_name(ctx: &ForgeContext<'_>, name: &str) -> String {
 fn create_if_missing(
     ctx: &ForgeContext<'_>,
     kind_name: &str,
-    nodes: &crate::config::NodeConfig,
+    cluster: &crate::config::ClusterSpec,
     st: &mut state::ForgeState,
-    name: &str,
 ) -> Result<(), ForgeError> {
     if kind_ops::cluster_exists(ctx.runner, kind_name)? {
         return Ok(());
     }
-    kind_ops::create_cluster(ctx.runner, kind_name, nodes, &ctx.state_dir, None)?;
-    upsert_cluster_state(st, name, kind_name, ClusterPhase::Running);
+    let config = kind_ops::CreateClusterConfig {
+        nodes: &cluster.nodes,
+        ports: &cluster.ports,
+        config_dir: &ctx.state_dir,
+        docker_network: None,
+    };
+    kind_ops::create_cluster(ctx.runner, kind_name, &config)?;
+    upsert_cluster_state(st, &cluster.name, kind_name, ClusterPhase::Running);
     Ok(())
 }
 
diff --git a/src/command/up.rs b/src/command/up.rs
index 490762a..6ef32bb 100644
--- a/src/command/up.rs
+++ b/src/command/up.rs
@@ -199,13 +199,13 @@ fn create_if_missing(
         ensure_state_entry(state, &cluster.name, kind_name, ClusterPhase::Running);
         return Ok(false);
     }
-    kind_ops::create_cluster(
-        ctx.runner,
-        kind_name,
-        &cluster.nodes,
-        &ctx.state_dir,
+    let cluster_config = kind_ops::CreateClusterConfig {
+        nodes: &cluster.nodes,
+        ports: &cluster.ports,
+        config_dir: &ctx.state_dir,
         docker_network,
-    )?;
+    };
+    kind_ops::create_cluster(ctx.runner, kind_name, &cluster_config)?;
     ensure_state_entry(state, &cluster.name, kind_name, ClusterPhase::Running);
     Ok(true)
 }
diff --git a/src/config/mod.rs b/src/config/mod.rs
index dba3752..344a8a3 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -153,6 +153,13 @@ pub struct ClusterSpec {
     /// Node layout for this Kind cluster.
     #[serde(default)]
     pub nodes: NodeConfig,
+    /// Host-to-node port mappings (Kind `extraPortMappings`).
+    ///
+    /// Maps host ports to Kind node ports via `extraPortMappings` in the
+    /// Kind cluster config. Required on macOS where `MetalLB` `LoadBalancer`
+    /// IPs are unreachable from the host.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub ports: Vec<PortMapping>,
     /// Stacks to apply to this cluster (must exist in `spec.stacks`).
     #[serde(default)]
     pub stacks: Vec<String>,
diff --git a/src/config/validate.rs b/src/config/validate.rs
index e5a45a6..fa234e9 100644
--- a/src/config/validate.rs
+++ b/src/config/validate.rs
@@ -25,6 +25,7 @@ pub fn validate(config: &ForgeConfig) -> Result<(), ForgeError> {
     check_network_name(&config.metadata.name, &config.spec)?;
     check_cluster_names(config)?;
     check_cluster_nodes(config)?;
+    check_cluster_ports(config)?;
     check_service_names(config)?;
     check_services(config)?;
     check_service_deps(config)?;
@@ -171,6 +172,35 @@ fn check_cluster_nodes(config: &ForgeConfig) -> Result<(), ForgeError> {
     Ok(())
 }
 
+/// Validate cluster-level port mappings.
+fn check_cluster_ports(config: &ForgeConfig) -> Result<(), ForgeError> {
+    for cluster in &config.spec.clusters {
+        let mut seen = BTreeSet::new();
+        for (i, pm) in cluster.ports.iter().enumerate() {
+            if pm.host == 0 {
+                return Err(ForgeError::Validation(format!(
+                    "cluster '{}' port mapping {}: host port must be non-zero",
+                    cluster.name, i,
+                )));
+            }
+            if pm.container == 0 {
+                return Err(ForgeError::Validation(format!(
+                    "cluster '{}' port mapping {}: container port must be non-zero",
+                    cluster.name, i,
+                )));
+            }
+            let key = (pm.host, pm.protocol.to_lowercase());
+            if !seen.insert(key) {
+                return Err(ForgeError::Validation(format!(
+                    "cluster '{}': duplicate host port {} ({})",
+                    cluster.name, pm.host, pm.protocol,
+                )));
+            }
+        }
+    }
+    Ok(())
+}
+
 /// Service names must be unique and DNS-label-valid.
 fn check_service_names(config: &ForgeConfig) -> Result<(), ForgeError> {
     let mut seen = BTreeSet::new();
@@ -1193,6 +1223,7 @@ mod tests {
         let cluster = ClusterSpec {
             name: "dupe".to_owned(),
             nodes: NodeConfig::default(),
+            ports: Vec::new(),
             stacks: Vec::new(),
             properties: BTreeMap::new(),
         };
@@ -1213,6 +1244,7 @@ mod tests {
         config.spec.clusters = vec![ClusterSpec {
             name: "c1".to_owned(),
             nodes: NodeConfig::default(),
+            ports: Vec::new(),
             stacks: vec!["nonexistent".to_owned()],
             properties: BTreeMap::new(),
         }];
@@ -1232,6 +1264,7 @@ mod tests {
         config.spec.clusters = vec![ClusterSpec {
             name: "c1".to_owned(),
             nodes: NodeConfig::default(),
+            ports: Vec::new(),
             stacks: Vec::new(),
             properties: BTreeMap::from([(
                 "model".to_owned(),
@@ -1263,6 +1296,7 @@ mod tests {
         config.spec.clusters = vec![ClusterSpec {
             name: "hub".to_owned(),
             nodes: NodeConfig::default(),
+            ports: Vec::new(),
             stacks: vec!["base".to_owned()],
             properties: BTreeMap::new(),
         }];
@@ -1299,6 +1333,7 @@ mod tests {
                 control_planes: 0,
                 workers: 1,
             },
+            ports: Vec::new(),
             stacks: Vec::new(),
             properties: BTreeMap::new(),
         }];
@@ -1312,6 +1347,52 @@ mod tests {
         );
     }
 
+    #[test]
+    fn cluster_port_zero_host_rejected() {
+        let mut config = base_config();
+        config.spec.clusters.push(ClusterSpec {
+            name: "test".to_owned(),
+            nodes: NodeConfig::default(),
+            ports: vec![PortMapping {
+                bind_address: None,
+                host: 0,
+                container: 30080,
+                protocol: "tcp".to_owned(),
+            }],
+            stacks: vec![],
+            properties: BTreeMap::new(),
+        });
+        let result = validate(&config);
+        assert!(result.is_err(), "zero host port should be rejected");
+    }
+
+    #[test]
+    fn cluster_duplicate_host_port_rejected() {
+        let mut config = base_config();
+        config.spec.clusters.push(ClusterSpec {
+            name: "test".to_owned(),
+            nodes: NodeConfig::default(),
+            ports: vec![
+                PortMapping {
+                    bind_address: None,
+                    host: 8080,
+                    container: 30080,
+                    protocol: "tcp".to_owned(),
+                },
+                PortMapping {
+                    bind_address: None,
+                    host: 8080,
+                    container: 30081,
+                    protocol: "tcp".to_owned(),
+                },
+            ],
+            stacks: vec![],
+            properties: BTreeMap::new(),
+        });
+        let result = validate(&config);
+        assert!(result.is_err(), "duplicate host port should be rejected");
+    }
+
     #[test]
     fn invalid_service_protocol_rejected() {
         let mut config = base_config();
diff --git a/src/stack/engine.rs b/src/stack/engine.rs
index b06dae0..021dce4 100644
--- a/src/stack/engine.rs
+++ b/src/stack/engine.rs
@@ -1161,6 +1161,7 @@ mod tests {
         let cluster = ClusterSpec {
             name: "provider-east".to_owned(),
             nodes: NodeConfig::default(),
+            ports: Vec::new(),
             stacks: Vec::new(),
             properties: BTreeMap::new(),
         };
diff --git a/src/stack/mod.rs b/src/stack/mod.rs
index b8ee565..91a5ed8 100644
--- a/src/stack/mod.rs
+++ b/src/stack/mod.rs
@@ -703,6 +703,7 @@ mod tests {
                 clusters: vec![ClusterSpec {
                     name: "hub".to_owned(),
                     nodes: NodeConfig::default(),
+                    ports: Vec::new(),
                     stacks: vec!["base".to_owned()],
                     properties: BTreeMap::new(),
                 }],
diff --git a/tests/fixtures/port-mappings.yaml b/tests/fixtures/port-mappings.yaml
new file mode 100644
index 0000000..0c3e8c1
--- /dev/null
+++ b/tests/fixtures/port-mappings.yaml
@@ -0,0 +1,21 @@
+apiVersion: forge.praxis.dev/v1alpha1
+kind: Environment
+metadata:
+  name: port-mapping-test
+spec:
+  runtime:
+    provider: docker
+    clusterPrefix: pm-test
+  network:
+    crossCluster: false
+    dnsZone: pm.test
+  clusters:
+    - name: local
+      ports:
+        - host: 8080
+          container: 30080
+        - host: 3000
+          container: 30300
+          protocol: tcp
+      stacks: []
+  stacks: {}
diff --git a/tests/integration.rs b/tests/integration.rs
index 0cb0579..0533602 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -313,3 +313,22 @@ fn cli_accepts_stack_status() {
     let result = Cli::try_parse_from(["praxis-forge", "stack", "status"]);
     assert!(result.is_ok(), "stack status should parse: {result:?}");
 }
+
+#[test]
+fn config_with_port_mappings_parses_and_validates() {
+    let yaml = std::fs::read_to_string("tests/fixtures/port-mappings.yaml").unwrap_or_else(|e| {
+        eprintln!("cannot read fixture: {e}");
+        std::process::abort()
+    });
+    let cfg: config::ForgeConfig = serde_yaml::from_str(&yaml).unwrap_or_else(|e| {
+        eprintln!("cannot parse: {e}");
+        std::process::abort()
+    });
+    validate::validate(&cfg).unwrap_or_else(|e| {
+        eprintln!("validation failed: {e}");
+        std::process::abort()
+    });
+    assert_eq!(cfg.spec.clusters[0].ports.len(), 2);
+    assert_eq!(cfg.spec.clusters[0].ports[0].host, 8080);
+    assert_eq!(cfg.spec.clusters[0].ports[0].container, 30080);
+}

From 77c52b3139ae11609851b9bfee8b3c825deef293 Mon Sep 17 00:00:00 2001
From: Ladislav Smola <lsmola@redhat.com>
Date: Fri, 14 Aug 2026 17:46:04 +0200
Subject: [PATCH 2/5] feat: add OTel observability benchmark demo

Add composable OTel benchmark example with:
- examples/stacks/observability/ (shared OTel Collector + dashboards)
- examples/stacks/mock-backends/ (Fortio echo + inference-sim)
- examples/otel-benchmark/ (forge.yaml, configs, scripts, README)

Deploys Prometheus, Grafana, Tempo, Loki, OTel Collector, MLflow,
and Praxis proxy on KIND. Includes A/B/C benchmark scripts for
measuring OTel tracing overhead.

Signed-off-by: Ladislav Smola <lsmola@redhat.com>
---
 .gitignore                                    |   1 +
 examples/otel-benchmark.yaml                  | 175 ++++++++++++++++++
 examples/otel-benchmark/README.md             | 135 ++++++++++++++
 examples/otel-benchmark/configs/baseline.yaml |  29 +++
 .../otel-benchmark/configs/otel-full.yaml     |  35 ++++
 .../otel-benchmark/configs/otel-noop.yaml     |  30 +++
 examples/otel-benchmark/manifests/praxis.yaml |  73 ++++++++
 .../manifests/servicemonitor.yaml             |  15 ++
 examples/otel-benchmark/scripts/benchmark.sh  | 121 ++++++++++++
 examples/otel-benchmark/scripts/report.sh     | 106 +++++++++++
 .../mock-backends/manifests/echo-backend.yaml |  40 ++++
 .../manifests/inference-sim.yaml              |  39 ++++
 .../dashboards/praxis-ai-golden-signals.json  | 173 +++++++++++++++++
 .../dashboards/praxis-benchmark.json          |  59 ++++++
 .../observability/dashboards/praxis-logs.json |  30 +++
 .../dashboards/praxis-proxy-overview.json     |  50 +++++
 .../dashboards/praxis-traces.json             |  97 ++++++++++
 .../manifests/otel-collector.yaml             |  96 ++++++++++
 tests/integration.rs                          |  27 +--
 19 files changed, 1319 insertions(+), 12 deletions(-)
 create mode 100644 examples/otel-benchmark.yaml
 create mode 100644 examples/otel-benchmark/README.md
 create mode 100644 examples/otel-benchmark/configs/baseline.yaml
 create mode 100644 examples/otel-benchmark/configs/otel-full.yaml
 create mode 100644 examples/otel-benchmark/configs/otel-noop.yaml
 create mode 100644 examples/otel-benchmark/manifests/praxis.yaml
 create mode 100644 examples/otel-benchmark/manifests/servicemonitor.yaml
 create mode 100755 examples/otel-benchmark/scripts/benchmark.sh
 create mode 100755 examples/otel-benchmark/scripts/report.sh
 create mode 100644 examples/stacks/mock-backends/manifests/echo-backend.yaml
 create mode 100644 examples/stacks/mock-backends/manifests/inference-sim.yaml
 create mode 100644 examples/stacks/observability/dashboards/praxis-ai-golden-signals.json
 create mode 100644 examples/stacks/observability/dashboards/praxis-benchmark.json
 create mode 100644 examples/stacks/observability/dashboards/praxis-logs.json
 create mode 100644 examples/stacks/observability/dashboards/praxis-proxy-overview.json
 create mode 100644 examples/stacks/observability/dashboards/praxis-traces.json
 create mode 100644 examples/stacks/observability/manifests/otel-collector.yaml

diff --git a/.gitignore b/.gitignore
index ad67955..369d0c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ target
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.forge/
diff --git a/examples/otel-benchmark.yaml b/examples/otel-benchmark.yaml
new file mode 100644
index 0000000..a500923
--- /dev/null
+++ b/examples/otel-benchmark.yaml
@@ -0,0 +1,175 @@
+apiVersion: forge.praxis.dev/v1alpha1
+kind: Environment
+metadata:
+  name: otel-benchmark
+spec:
+  runtime:
+    provider: docker
+    clusterPrefix: otel-bench
+  network:
+    crossCluster: false
+
+  clusters:
+    - name: local
+      ports:
+        - host: 18080
+          container: 30080    # Praxis proxy
+        - host: 18901
+          container: 30901    # Praxis admin/metrics
+        - host: 13000
+          container: 30300    # Grafana
+        - host: 19090
+          container: 30909    # Prometheus
+        - host: 15000
+          container: 30500    # MLflow
+      stacks:
+        - prometheus
+        - tempo
+        - loki
+        - otel-collector
+        - mlflow
+        - mock-backends
+        - praxis-images
+        - praxis-deploy
+        - dashboards
+        - datasources
+      properties:
+        # Pin chart versions for reproducibility
+        kubePrometheusVersion: "72.6.3"
+        tempoVersion: "1.24.4"
+        lokiStackVersion: "2.10.2"
+        mlflowVersion: "1.11.3"
+
+  stacks:
+    prometheus:
+      description: kube-prometheus-stack (Prometheus + Grafana)
+      steps:
+        - type: helm
+          release: monitoring
+          chart: prometheus-community/kube-prometheus-stack
+          version: "{{ cluster.properties.kubePrometheusVersion }}"
+          namespace: monitoring
+          values:
+            prometheus:
+              prometheusSpec:
+                serviceMonitorSelectorNilUsesHelmValues: false
+                additionalScrapeConfigs:
+                  - job_name: praxis-proxy
+                    metrics_path: /metrics
+                    scrape_interval: 15s
+                    static_configs:
+                      - targets: ["praxis-proxy.default.svc:9901"]
+              service:
+                type: NodePort
+                nodePort: 30909
+            grafana:
+              adminPassword: admin
+              image:
+                tag: 11.6.0
+              service:
+                type: NodePort
+                nodePort: 30300
+        - type: wait
+          resource: deployment/monitoring-grafana
+          namespace: monitoring
+          condition: available
+          timeout: "300s"
+
+    tempo:
+      description: Tempo trace storage
+      steps:
+        - type: helm
+          release: tempo
+          chart: grafana/tempo
+          version: "{{ cluster.properties.tempoVersion }}"
+          namespace: monitoring
+
+    loki:
+      description: Loki + Promtail log aggregation
+      steps:
+        - type: helm
+          release: loki
+          chart: grafana/loki-stack
+          version: "{{ cluster.properties.lokiStackVersion }}"
+          namespace: monitoring
+          values:
+            loki:
+              persistence:
+                enabled: false
+            promtail:
+              enabled: true
+
+    otel-collector:
+      description: OpenTelemetry Collector (OTLP → Tempo + MLflow)
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl create namespace otel --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: manifest
+          path: stacks/observability/manifests/otel-collector.yaml
+        - type: wait
+          resource: deployment/otel-collector
+          namespace: otel
+          condition: available
+          timeout: "60s"
+
+    mlflow:
+      description: MLflow experiment tracking
+      steps:
+        - type: helm
+          release: mlflow
+          chart: community-charts/mlflow
+          version: "{{ cluster.properties.mlflowVersion }}"
+          namespace: mlflow
+        - type: exec
+          command: [bash, -c, "kubectl -n mlflow patch svc mlflow --type=merge -p '{\"spec\":{\"type\":\"NodePort\"}}' && kubectl -n mlflow patch svc mlflow --type=json -p '[{\"op\":\"replace\",\"path\":\"/spec/ports/0/nodePort\",\"value\":30500}]'"]
+
+    mock-backends:
+      description: Fortio echo + llm-d inference-sim
+      steps:
+        - type: manifest
+          path: stacks/mock-backends/manifests/echo-backend.yaml
+        - type: manifest
+          path: stacks/mock-backends/manifests/inference-sim.yaml
+
+    praxis-images:
+      description: Build and load Praxis images into KIND
+      steps:
+        - type: exec
+          command: [bash, -c, "cd \"${PRAXIS_DIR:?Set PRAXIS_DIR to your praxis checkout}\" && docker build -t praxis:dev -f Containerfile ."]
+        - type: exec
+          command: [bash, -c, "cd \"${PRAXIS_DIR:?}\" && sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' Containerfile | docker build -t praxis:dev-otel -f - ."]
+        - type: exec
+          command: [bash, -c, "kind load docker-image praxis:dev --name otel-bench-local"]
+        - type: exec
+          command: [bash, -c, "kind load docker-image praxis:dev-otel --name otel-bench-local"]
+
+    praxis-deploy:
+      description: Deploy Praxis proxy with OTel config
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl create configmap praxis-config --from-file=config.yaml=examples/otel-benchmark/configs/otel-full.yaml -n default --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: manifest
+          path: otel-benchmark/manifests/praxis.yaml
+        - type: manifest
+          path: otel-benchmark/manifests/servicemonitor.yaml
+        - type: wait
+          resource: deployment/praxis-proxy
+          namespace: default
+          condition: available
+          timeout: "120s"
+
+    dashboards:
+      description: Load Grafana dashboards via ConfigMap
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl -n monitoring create configmap praxis-dashboards --from-file=examples/stacks/observability/dashboards/ --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: exec
+          command: [kubectl, -n, monitoring, label, configmap, praxis-dashboards, grafana_dashboard=1, --overwrite]
+
+    datasources:
+      description: Add Tempo + Loki datasources to Grafana
+      steps:
+        - type: exec
+          command: [bash, -c, "sleep 5 && curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Tempo\",\"type\":\"tempo\",\"access\":\"proxy\",\"url\":\"http://tempo.monitoring.svc:3200\",\"uid\":\"tempo\"}' || true"]
+        - type: exec
+          command: [bash, -c, "curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Loki\",\"type\":\"loki\",\"access\":\"proxy\",\"url\":\"http://loki.monitoring.svc:3100\",\"uid\":\"loki\"}' || true"]
diff --git a/examples/otel-benchmark/README.md b/examples/otel-benchmark/README.md
new file mode 100644
index 0000000..7c92b13
--- /dev/null
+++ b/examples/otel-benchmark/README.md
@@ -0,0 +1,135 @@
+# OTel Observability Benchmark
+
+Deploys a full observability stack on KIND for benchmarking Praxis proxy
+OTel tracing overhead.
+
+## Stack
+
+- **Prometheus + Grafana** (kube-prometheus-stack) — metrics + visualization
+- **Tempo** — distributed trace storage
+- **Loki + Promtail** — log aggregation
+- **OTel Collector** — trace pipeline (OTLP → Tempo + MLflow)
+- **MLflow** — experiment tracking
+- **Fortio echo** — mock HTTP backend
+- **llm-d inference-sim** — mock LLM backend
+- **Praxis proxy** — the proxy under test (baseline + OTel variants)
+
+## Prerequisites
+
+- Docker or Podman
+- [KIND](https://kind.sigs.k8s.io/)
+- [Helm](https://helm.sh/) with repos: `prometheus-community`, `grafana`, `community-charts`
+- [vegeta](https://github.com/tsenart/vegeta) (for benchmarks)
+- Praxis source checkout with OTel PRs (for image builds)
+
+## Quick Start
+
+```bash
+# 1. Add helm repos (one-time)
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo add community-charts https://community-charts.github.io/helm-charts
+helm repo update
+
+# 2. Set the path to your praxis repo checkout
+export PRAXIS_DIR=/path/to/praxis
+
+# 3. Build praxis images (if not already built)
+cd "$PRAXIS_DIR"
+docker build -t praxis:dev -f Containerfile .
+sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' \
+  Containerfile | docker build -t praxis:dev-otel -f - .
+
+# 4. Deploy the full stack
+cd /path/to/forge
+cargo run -- up --config examples/otel-benchmark.yaml
+
+# If forge up doesn't apply stacks automatically, apply them manually:
+for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-images praxis-deploy dashboards datasources; do
+  cargo run -- stack apply --config examples/otel-benchmark.yaml local "$stack"
+done
+
+# 5. Switch to OTel image (if deployed with baseline)
+kubectl --context kind-otel-bench-local set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel
+
+# 6. Verify
+curl http://localhost:18080/         # Praxis proxy
+open http://localhost:13000          # Grafana (admin/admin)
+open http://localhost:19090          # Prometheus
+open http://localhost:15000          # MLflow
+```
+
+## Run Benchmark
+
+> **Note:** This benchmark uses the core `praxis` proxy, which generates
+> a root request span, per-filter child spans, and an upstream exchange
+> span (10 spans per request). To see AI-specific routing spans
+> (`routing.select` with provider/cluster/site attributes), build from
+> the `praxis-proxy/ai` repo with `--features opentelemetry` and use
+> AI filters (intelligent_route, format classification) with
+> inference-sim as the backend. That is a separate demo configuration.
+
+```bash
+bash examples/otel-benchmark/scripts/benchmark.sh
+```
+
+Runs 3 configurations at 2000 RPS for 30s each:
+- **A: Baseline** — `praxis:dev` (no OTel feature)
+- **B: OTel noop** — `praxis:dev-otel` (spans created, not exported)
+- **C: OTel full** — `praxis:dev-otel` (spans exported to collector → Tempo)
+
+Generate the markdown report:
+```bash
+bash examples/otel-benchmark/scripts/report.sh <results-dir>
+```
+
+## Dashboards
+
+| Dashboard | URL | What it shows |
+|-----------|-----|------|
+| Praxis Proxy Overview | http://localhost:13000/d/praxis-proxy-overview | Request rate, latency P50/P99, requests by method |
+| OTel Traces | http://localhost:13000/d/praxis-traces | Searchable trace table with clickable Trace IDs |
+| Benchmark Results | http://localhost:13000/d/praxis-benchmark | CPU/memory for praxis + collector, RPS, latency |
+| AI/LLM Golden Signals | http://localhost:13000/d/praxis-ai-golden-signals | P95 latency stat, throughput, AI token metrics (future) |
+| Structured Logs | http://localhost:13000/d/praxis-logs | Log volume, error logs, all praxis access logs |
+
+### Explore views
+
+| View | URL |
+|------|-----|
+| Tempo trace search | http://localhost:13000/explore (select Tempo datasource → Search tab) |
+| Prometheus metrics | http://localhost:13000/explore (select Prometheus datasource) |
+| Loki log search | http://localhost:13000/explore (select Loki datasource) |
+
+### Other UIs
+
+| Service | URL |
+|---------|-----|
+| Prometheus | http://localhost:19090 |
+| MLflow | http://localhost:15000 |
+| Praxis proxy | http://localhost:18080 |
+| Praxis admin/metrics | http://localhost:18901/metrics |
+
+## Host Ports
+
+| Port | Service | KIND NodePort |
+|------|---------|------|
+| 18080 | Praxis proxy | 30080 |
+| 18901 | Praxis admin | 30901 |
+| 13000 | Grafana | 30300 |
+| 19090 | Prometheus | 30909 |
+| 15000 | MLflow | 30500 |
+
+## Known Issues
+
+- **Grafana version**: Must use 11.x (pinned via `grafana.image.tag`). Grafana 12.0 has rendering bugs with provisioned dashboards using `row`/`gauge` panel types.
+- **Datasources**: Prometheus and Tempo datasources are added via the `datasources` stack. If Grafana restarts, they need re-adding.
+- **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export. This fix is not yet in any upstream PR.
+
+## Teardown
+
+```bash
+cargo run -- down --config examples/otel-benchmark.yaml
+# or
+kind delete cluster --name otel-bench-local
+```
diff --git a/examples/otel-benchmark/configs/baseline.yaml b/examples/otel-benchmark/configs/baseline.yaml
new file mode 100644
index 0000000..70389c2
--- /dev/null
+++ b/examples/otel-benchmark/configs/baseline.yaml
@@ -0,0 +1,29 @@
+# Baseline benchmark config (no OTel).
+# Routes all traffic to the Fortio echo backend.
+
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+
+      - filter: router
+        routes:
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
diff --git a/examples/otel-benchmark/configs/otel-full.yaml b/examples/otel-benchmark/configs/otel-full.yaml
new file mode 100644
index 0000000..19b179d
--- /dev/null
+++ b/examples/otel-benchmark/configs/otel-full.yaml
@@ -0,0 +1,35 @@
+# OTel full export benchmark config.
+# Spans are created and exported via OTLP/gRPC to the collector.
+# The OTEL_EXPORTER_OTLP_ENDPOINT env var overrides this at runtime.
+
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+telemetry:
+  otlp_endpoint: "http://otel-collector.otel.svc:4317"
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+
+      - filter: access_log
+
+      - filter: router
+        routes:
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
diff --git a/examples/otel-benchmark/configs/otel-noop.yaml b/examples/otel-benchmark/configs/otel-noop.yaml
new file mode 100644
index 0000000..6d51b7c
--- /dev/null
+++ b/examples/otel-benchmark/configs/otel-noop.yaml
@@ -0,0 +1,30 @@
+# OTel noop benchmark config.
+# OTel feature is compiled in but no OTLP endpoint is configured,
+# so spans are created but never exported.
+
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+
+      - filter: router
+        routes:
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
diff --git a/examples/otel-benchmark/manifests/praxis.yaml b/examples/otel-benchmark/manifests/praxis.yaml
new file mode 100644
index 0000000..2279cb6
--- /dev/null
+++ b/examples/otel-benchmark/manifests/praxis.yaml
@@ -0,0 +1,73 @@
+# Praxis Deployment for benchmark mode.
+# ConfigMap is NOT included here — setup.sh creates it separately
+# to avoid overwriting with empty data.
+# No args are specified — the Containerfile ENTRYPOINT already
+# includes [-c, /etc/praxis/config.yaml].
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: praxis-proxy
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: praxis-proxy
+  template:
+    metadata:
+      labels:
+        app: praxis-proxy
+    spec:
+      containers:
+        - name: praxis-proxy
+          image: praxis:dev-otel
+          ports:
+            - containerPort: 8080
+              name: proxy
+            - containerPort: 9901
+              name: admin
+          volumeMounts:
+            - name: config
+              mountPath: /etc/praxis
+          resources:
+            requests:
+              cpu: 500m
+              memory: 128Mi
+            limits:
+              cpu: "2"
+              memory: 512Mi
+          readinessProbe:
+            httpGet:
+              path: /healthy
+              port: admin
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /healthy
+              port: admin
+            initialDelaySeconds: 5
+            periodSeconds: 10
+      volumes:
+        - name: config
+          configMap:
+            name: praxis-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: praxis-proxy
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app: praxis-proxy
+  ports:
+    - port: 8080
+      targetPort: 8080
+      nodePort: 30080
+      name: proxy
+    - port: 9901
+      targetPort: 9901
+      nodePort: 30901
+      name: admin
diff --git a/examples/otel-benchmark/manifests/servicemonitor.yaml b/examples/otel-benchmark/manifests/servicemonitor.yaml
new file mode 100644
index 0000000..312a057
--- /dev/null
+++ b/examples/otel-benchmark/manifests/servicemonitor.yaml
@@ -0,0 +1,15 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: praxis-proxy
+  namespace: default
+  labels:
+    app: praxis-proxy
+spec:
+  selector:
+    matchLabels:
+      app: praxis-proxy
+  endpoints:
+    - port: admin
+      interval: 15s
+      path: /metrics
diff --git a/examples/otel-benchmark/scripts/benchmark.sh b/examples/otel-benchmark/scripts/benchmark.sh
new file mode 100755
index 0000000..4901ae6
--- /dev/null
+++ b/examples/otel-benchmark/scripts/benchmark.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+KIND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+CLUSTER_NAME="${CLUSTER_NAME:-otel-bench-local}"
+CTX="kind-${CLUSTER_NAME}"
+GATEWAY_URL="http://localhost:18080"
+
+RATE="${RATE:-2000}"
+DURATION="${DURATION:-30s}"
+RUNS="${RUNS:-3}"
+
+RESULTS_DIR="${KIND_DIR}/results/$(date +%Y%m%d-%H%M%S)"
+mkdir -p "${RESULTS_DIR}"
+
+echo "=== Praxis OTel Benchmark ==="
+echo "Rate: ${RATE} RPS | Duration: ${DURATION} | Runs: ${RUNS}"
+echo "Results: ${RESULTS_DIR}"
+echo ""
+
+run_vegeta() {
+  local label="$1"
+  local run="$2"
+  echo "--- ${label} run ${run}/${RUNS} ---"
+  echo "GET ${GATEWAY_URL}/" | \
+    vegeta attack -rate="${RATE}" -duration="${DURATION}" -connections=200 | \
+    tee "${RESULTS_DIR}/${label}-run${run}.bin" | \
+    vegeta report -type=json > "${RESULTS_DIR}/${label}-run${run}.json"
+  vegeta report < "${RESULTS_DIR}/${label}-run${run}.bin"
+  # Capture resource snapshot
+  kubectl --context "${CTX}" top pod -n default --no-headers 2>/dev/null \
+    >> "${RESULTS_DIR}/${label}-resources.txt" || true
+  echo ""
+}
+
+# ---- Run A: Baseline (no OTel) ----
+echo "=========================================="
+echo "  Run A: Baseline (praxis:dev, no OTel)"
+echo "=========================================="
+
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${KIND_DIR}/configs/baseline.yaml" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
+kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev -n default
+kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "baseline" "${i}"
+  sleep 5
+done
+
+# ---- Run B: OTel noop (spans created, not exported) ----
+echo "=========================================="
+echo "  Run B: OTel noop (praxis:dev-otel, no endpoint)"
+echo "=========================================="
+
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${KIND_DIR}/configs/otel-noop.yaml" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
+kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel -n default
+kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "otel-noop" "${i}"
+  sleep 5
+done
+
+# ---- Run C: OTel full (spans exported to collector) ----
+echo "=========================================="
+echo "  Run C: OTel full (praxis:dev-otel, exporting)"
+echo "=========================================="
+
+# Set OTEL env var BEFORE restarting the pod
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${KIND_DIR}/configs/otel-full.yaml" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
+kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel -n default
+kubectl --context "${CTX}" set env deployment/praxis-proxy \
+  OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.otel.svc:4317 -n default
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "otel-full" "${i}"
+  sleep 5
+done
+
+echo "=========================================="
+echo "  Benchmark complete"
+echo "=========================================="
+echo "Results in: ${RESULTS_DIR}"
+echo ""
+echo "Generate report:"
+echo "  bash ${SCRIPT_DIR}/report.sh ${RESULTS_DIR}"
diff --git a/examples/otel-benchmark/scripts/report.sh b/examples/otel-benchmark/scripts/report.sh
new file mode 100755
index 0000000..dd2213d
--- /dev/null
+++ b/examples/otel-benchmark/scripts/report.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RESULTS_DIR="${1:?Usage: report.sh <results-dir>}"
+
+if [ ! -d "${RESULTS_DIR}" ]; then
+  echo "Error: ${RESULTS_DIR} does not exist"
+  exit 1
+fi
+
+BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
+COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+
+REPORT="${RESULTS_DIR}/report.md"
+
+# Use Python to generate the report (macOS has bash 3.2, no associative arrays)
+python3 << 'PYTHON_SCRIPT'
+import json
+import glob
+import sys
+import os
+from datetime import datetime
+
+results_dir = os.environ['RESULTS_DIR']
+branch = os.environ['BRANCH']
+commit = os.environ['COMMIT']
+
+configs = ['baseline', 'otel-noop', 'otel-full']
+stats = {}
+
+for config in configs:
+    total_p50 = 0
+    total_p99 = 0
+    total_rps = 0
+    count = 0
+
+    pattern = os.path.join(results_dir, f'{config}-run*.json')
+    for path in glob.glob(pattern):
+        try:
+            with open(path) as f:
+                data = json.load(f)
+                total_p50 += data['latencies']['50th']
+                total_p99 += data['latencies']['99th']
+                total_rps += data['throughput']
+                count += 1
+        except (json.JSONDecodeError, KeyError) as e:
+            print(f"Warning: Failed to parse {path}: {e}", file=sys.stderr)
+            continue
+
+    if count > 0:
+        stats[config] = {
+            'p50': int(total_p50 / count / 1000),  # Convert to microseconds
+            'p99': int(total_p99 / count / 1000),
+            'rps': int(total_rps / count),
+            'runs': count
+        }
+    else:
+        stats[config] = {'p50': 0, 'p99': 0, 'rps': 0, 'runs': 0}
+
+# Generate report
+report_path = os.path.join(results_dir, 'report.md')
+with open(report_path, 'w') as f:
+    f.write(f"# Praxis OTel Overhead Benchmark\n\n")
+    f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d')} | **Commit:** {commit} | **Branch:** {branch}\n")
+    f.write(f"**Platform:** KIND (1 node) | **Backend:** Fortio echo\n\n")
+    f.write(f"## Summary\n\n")
+    f.write(f"| Config | P50 (us) | P99 (us) | RPS | Runs | P50 delta | P99 delta |\n")
+    f.write(f"|--------|----------|----------|-----|------|-----------|-----------||\n")
+
+    base_p50 = stats['baseline']['p50']
+    base_p99 = stats['baseline']['p99']
+
+    for config in configs:
+        s = stats[config]
+        p50 = s['p50']
+        p99 = s['p99']
+        rps = s['rps']
+        runs = s['runs']
+
+        if config == 'baseline':
+            f.write(f"| Baseline | {p50} | {p99} | {rps} | {runs} | -- | -- |\n")
+        else:
+            label = 'OTel (noop)' if config == 'otel-noop' else 'OTel (full)'
+
+            if base_p50 > 0:
+                dp50 = f"{(p50 - base_p50) / base_p50 * 100:+.1f}%"
+                dp99 = f"{(p99 - base_p99) / base_p99 * 100:+.1f}%"
+            else:
+                dp50 = "N/A"
+                dp99 = "N/A"
+
+            f.write(f"| {label} | {p50} | {p99} | {rps} | {runs} | {dp50} | {dp99} |\n")
+
+    f.write(f"\n## Previous Results (reference)\n\n")
+    f.write(f"| Config | P50 (us) | P99 (us) | RPS | P50 delta | P99 delta |\n")
+    f.write(f"|--------|----------|----------|-----|-----------|-----------||\n")
+    f.write(f"| Baseline | 388 | 568 | 2,000 | -- | -- |\n")
+    f.write(f"| OTel noop | 386 | 558 | 2,000 | -0.5% | -1.8% |\n")
+    f.write(f"| OTel full | 392 | 592 | 2,000 | +1.0% | +4.2% |\n\n")
+    f.write(f"Note: Previous results were from the old branch without root spans.\n")
+
+print(f"Report written to: {report_path}")
+PYTHON_SCRIPT
+
+echo ""
+cat "${REPORT}"
diff --git a/examples/stacks/mock-backends/manifests/echo-backend.yaml b/examples/stacks/mock-backends/manifests/echo-backend.yaml
new file mode 100644
index 0000000..cd4b15b
--- /dev/null
+++ b/examples/stacks/mock-backends/manifests/echo-backend.yaml
@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: echo-backend
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: echo-backend
+  template:
+    metadata:
+      labels:
+        app: echo-backend
+    spec:
+      containers:
+        - name: fortio
+          image: fortio/fortio:latest
+          args: ["server", "-echo-server-default-params", "delay=0"]
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 100m
+              memory: 64Mi
+            limits:
+              cpu: "1"
+              memory: 256Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echo-backend
+  namespace: default
+spec:
+  selector:
+    app: echo-backend
+  ports:
+    - port: 8080
+      targetPort: 8080
diff --git a/examples/stacks/mock-backends/manifests/inference-sim.yaml b/examples/stacks/mock-backends/manifests/inference-sim.yaml
new file mode 100644
index 0000000..bfc1811
--- /dev/null
+++ b/examples/stacks/mock-backends/manifests/inference-sim.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: inference-sim
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: inference-sim
+  template:
+    metadata:
+      labels:
+        app: inference-sim
+    spec:
+      containers:
+        - name: sim
+          image: ghcr.io/llm-d/llm-d-inference-sim:latest
+          ports:
+            - containerPort: 8000
+          resources:
+            requests:
+              cpu: 100m
+              memory: 64Mi
+            limits:
+              cpu: "1"
+              memory: 256Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: inference-sim
+  namespace: default
+spec:
+  selector:
+    app: inference-sim
+  ports:
+    - port: 8000
+      targetPort: 8000
diff --git a/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json b/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json
new file mode 100644
index 0000000..295b635
--- /dev/null
+++ b/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json
@@ -0,0 +1,173 @@
+{
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.5
+              },
+              {
+                "color": "red",
+                "value": 2
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        }
+      },
+      "targets": [
+        {
+          "expr": "avg_over_time(praxis_http_request_duration_seconds{quantile=\"0.95\"}[5m])",
+          "legendFormat": "P95"
+        }
+      ],
+      "title": "P95 Request Latency",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 16,
+        "x": 8,
+        "y": 0
+      },
+      "id": 2,
+      "targets": [
+        {
+          "expr": "sum(rate(praxis_http_requests_total[5m]))",
+          "legendFormat": "Requests/sec"
+        }
+      ],
+      "title": "Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "noValue": "No AI metrics yet"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 3,
+      "targets": [
+        {
+          "expr": "sum(rate(praxis_ai_prompt_tokens_total[5m])) + sum(rate(praxis_ai_generation_tokens_total[5m]))",
+          "legendFormat": "Tokens/sec"
+        }
+      ],
+      "title": "Tokens/sec (requires ai#92)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "noValue": "No AI metrics yet",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 4,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(increase(praxis_ai_prompt_tokens_total[1h])) * 0.003/1000 + sum(increase(praxis_ai_generation_tokens_total[1h])) * 0.015/1000",
+          "legendFormat": "Hourly Cost"
+        }
+      ],
+      "title": "Estimated Hourly Cost (requires ai#92)",
+      "type": "stat"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": [
+    "praxis",
+    "ai"
+  ],
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Praxis AI/LLM Golden Signals",
+  "uid": "praxis-ai-golden-signals"
+}
diff --git a/examples/stacks/observability/dashboards/praxis-benchmark.json b/examples/stacks/observability/dashboards/praxis-benchmark.json
new file mode 100644
index 0000000..a4f6cf1
--- /dev/null
+++ b/examples/stacks/observability/dashboards/praxis-benchmark.json
@@ -0,0 +1,59 @@
+{
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "id": 1,
+      "targets": [{ "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"praxis-proxy.*\"}[1m]))", "legendFormat": "{{pod}}" }],
+      "title": "Praxis CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "id": 2,
+      "targets": [{ "expr": "container_memory_working_set_bytes{pod=~\"praxis-proxy.*\", container=\"praxis-proxy\"}", "legendFormat": "{{pod}}" }],
+      "title": "Praxis Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 3,
+      "targets": [{ "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"otel-collector.*\"}[1m]))", "legendFormat": "{{pod}}" }],
+      "title": "OTel Collector CPU",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "id": 4,
+      "targets": [{ "expr": "sum(rate(praxis_http_requests_total[1m]))", "legendFormat": "RPS" }],
+      "title": "Request Rate (1m)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+      "id": 5,
+      "targets": [
+        { "expr": "praxis_http_request_duration_seconds{quantile=\"0.5\"}", "legendFormat": "P50" },
+        { "expr": "praxis_http_request_duration_seconds{quantile=\"0.99\"}", "legendFormat": "P99" }
+      ],
+      "title": "Latency During Benchmark",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["praxis", "benchmark"],
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "Praxis Benchmark Results",
+  "uid": "praxis-benchmark"
+}
diff --git a/examples/stacks/observability/dashboards/praxis-logs.json b/examples/stacks/observability/dashboards/praxis-logs.json
new file mode 100644
index 0000000..70db91f
--- /dev/null
+++ b/examples/stacks/observability/dashboards/praxis-logs.json
@@ -0,0 +1,30 @@
+{
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "targets": [{ "expr": "sum(rate({container=\"praxis-proxy\"} [5m]))", "legendFormat": "log rate" }],
+      "title": "Log Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": {}, "overrides": [] },
+      "gridPos": { "h": 14, "w": 24, "x": 0, "y": 6 },
+      "id": 2,
+      "options": { "cellHeight": "sm", "showHeader": true },
+      "targets": [{ "expr": "{container=\"praxis-proxy\"}", "refId": "A" }],
+      "title": "All Praxis Logs",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["praxis", "logs"],
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "Praxis Structured Logs",
+  "uid": "praxis-logs"
+}
diff --git a/examples/stacks/observability/dashboards/praxis-proxy-overview.json b/examples/stacks/observability/dashboards/praxis-proxy-overview.json
new file mode 100644
index 0000000..35b3090
--- /dev/null
+++ b/examples/stacks/observability/dashboards/praxis-proxy-overview.json
@@ -0,0 +1,50 @@
+{
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "id": 1,
+      "targets": [{ "expr": "sum(rate(praxis_http_requests_total[5m]))", "legendFormat": "RPS" }],
+      "title": "Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "id": 2,
+      "targets": [
+        { "expr": "praxis_http_request_duration_seconds{quantile=\"0.5\"}", "legendFormat": "P50" },
+        { "expr": "praxis_http_request_duration_seconds{quantile=\"0.99\"}", "legendFormat": "P99" }
+      ],
+      "title": "Latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 3,
+      "targets": [{ "expr": "sum(rate(praxis_http_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }],
+      "title": "Requests by Method",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "id": 4,
+      "targets": [{ "expr": "rate(praxis_http_request_duration_seconds_sum[5m]) / rate(praxis_http_request_duration_seconds_count[5m])", "legendFormat": "Avg" }],
+      "title": "Average Latency",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["praxis"],
+  "time": { "from": "now-15m", "to": "now" },
+  "title": "Praxis Proxy Overview",
+  "uid": "praxis-proxy-overview"
+}
diff --git a/examples/stacks/observability/dashboards/praxis-traces.json b/examples/stacks/observability/dashboards/praxis-traces.json
new file mode 100644
index 0000000..1748ce4
--- /dev/null
+++ b/examples/stacks/observability/dashboards/praxis-traces.json
@@ -0,0 +1,97 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 100,
+      "title": "Trace Search",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "tempo", "uid": "tempo" },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": { "h": 12, "w": 24, "x": 0, "y": 1 },
+      "id": 1,
+      "options": {
+        "cellHeight": "sm",
+        "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false },
+        "showHeader": true,
+        "sortBy": [{ "desc": true, "displayName": "Start time" }]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "tempo", "uid": "tempo" },
+          "queryType": "nativeSearch",
+          "serviceName": "praxis",
+          "limit": 20
+        }
+      ],
+      "title": "Recent Traces",
+      "type": "table"
+    },
+    {
+      "datasource": { "type": "tempo", "uid": "tempo" },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": { "h": 10, "w": 12, "x": 0, "y": 13 },
+      "id": 2,
+      "options": {
+        "cellHeight": "sm",
+        "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "tempo", "uid": "tempo" },
+          "queryType": "nativeSearch",
+          "serviceName": "praxis",
+          "search": "status=error",
+          "limit": 20
+        }
+      ],
+      "title": "Error Traces",
+      "type": "table"
+    },
+    {
+      "datasource": { "type": "tempo", "uid": "tempo" },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": { "h": 10, "w": 12, "x": 12, "y": 13 },
+      "id": 3,
+      "options": {
+        "cellHeight": "sm",
+        "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "tempo", "uid": "tempo" },
+          "queryType": "nativeSearch",
+          "serviceName": "praxis",
+          "minDuration": "100ms",
+          "limit": 20
+        }
+      ],
+      "title": "Slow Traces (>100ms)",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["praxis", "traces"],
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "Praxis OTel Traces",
+  "uid": "praxis-traces"
+}
diff --git a/examples/stacks/observability/manifests/otel-collector.yaml b/examples/stacks/observability/manifests/otel-collector.yaml
new file mode 100644
index 0000000..bae9a06
--- /dev/null
+++ b/examples/stacks/observability/manifests/otel-collector.yaml
@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+  namespace: otel
+data:
+  config.yaml: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+
+    processors:
+      batch:
+        timeout: 5s
+        send_batch_size: 8192
+
+    exporters:
+      debug:
+        verbosity: basic
+      otlp/tempo:
+        endpoint: tempo.monitoring.svc:4317
+        tls:
+          insecure: true
+      otlphttp/mlflow:
+        endpoint: http://mlflow.mlflow.svc:5000
+        tls:
+          insecure: true
+        headers:
+          x-mlflow-experiment-id: "0"
+          Host: "localhost:5000"
+
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [debug, otlp/tempo, otlphttp/mlflow]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  namespace: otel
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: otel-collector
+  template:
+    metadata:
+      labels:
+        app: otel-collector
+    spec:
+      containers:
+        - name: collector
+          image: otel/opentelemetry-collector-contrib:0.108.0
+          args: ["--config=/etc/otelcol/config.yaml"]
+          ports:
+            - containerPort: 4317
+              name: otlp-grpc
+            - containerPort: 4318
+              name: otlp-http
+          volumeMounts:
+            - name: config
+              mountPath: /etc/otelcol
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+      volumes:
+        - name: config
+          configMap:
+            name: otel-collector-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: otel
+spec:
+  selector:
+    app: otel-collector
+  ports:
+    - port: 4317
+      targetPort: 4317
+      name: otlp-grpc
+    - port: 4318
+      targetPort: 4318
+      name: otlp-http
diff --git a/tests/integration.rs b/tests/integration.rs
index 0533602..3ede809 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -316,19 +316,22 @@ fn cli_accepts_stack_status() {
 
 #[test]
 fn config_with_port_mappings_parses_and_validates() {
-    let yaml = std::fs::read_to_string("tests/fixtures/port-mappings.yaml").unwrap_or_else(|e| {
-        eprintln!("cannot read fixture: {e}");
+    let Ok(yaml) = std::fs::read_to_string("tests/fixtures/port-mappings.yaml") else {
         std::process::abort()
-    });
-    let cfg: config::ForgeConfig = serde_yaml::from_str(&yaml).unwrap_or_else(|e| {
-        eprintln!("cannot parse: {e}");
+    };
+    let Ok(cfg) = serde_yaml::from_str::<config::ForgeConfig>(&yaml) else {
         std::process::abort()
-    });
-    validate::validate(&cfg).unwrap_or_else(|e| {
-        eprintln!("validation failed: {e}");
+    };
+    if validate::validate(&cfg).is_err() {
         std::process::abort()
-    });
-    assert_eq!(cfg.spec.clusters[0].ports.len(), 2);
-    assert_eq!(cfg.spec.clusters[0].ports[0].host, 8080);
-    assert_eq!(cfg.spec.clusters[0].ports[0].container, 30080);
+    }
+    let Some(cluster) = cfg.spec.clusters.first() else {
+        std::process::abort()
+    };
+    assert_eq!(cluster.ports.len(), 2);
+    let Some(port) = cluster.ports.first() else {
+        std::process::abort()
+    };
+    assert_eq!(port.host, 8080);
+    assert_eq!(port.container, 30080);
 }

From 74ddd95d8a17a22e3b1279f7ff0e5c1dae0c11b7 Mon Sep 17 00:00:00 2001
From: Ladislav Smola <lsmola@redhat.com>
Date: Fri, 14 Aug 2026 23:59:14 +0200
Subject: [PATCH 3/5] feat: add AI benchmark scenario and fix MLflow/collector
 config

- Add Scenario 2: AI benchmark with inference-sim (POST /v1/chat/completions)
  - benchmark-ai.sh: A/B/C comparison at 500 RPS (baseline/noop/full)
  - report-ai.sh: generate markdown report from AI benchmark results
  - ai-otel-full.yaml: config with /v1/* -> inference, / -> echo routing
- Fix OTel collector: remove broken MLflow exporter (protobuf vs JSON mismatch)
- Fix MLflow helm: enable databaseMigration, set service.type=NodePort
- Fix praxis manifest: correct ENTRYPOINT comment for AI vs core images
- Update README with both scenarios, span tree, build instructions

Signed-off-by: Ladislav Smola <lsmola@redhat.com>
---
 examples/otel-benchmark.yaml                  |   7 +-
 examples/otel-benchmark/README.md             | 125 +++++++++++++----
 .../otel-benchmark/configs/ai-otel-full.yaml  |  35 +++++
 examples/otel-benchmark/manifests/praxis.yaml |   4 +-
 .../otel-benchmark/scripts/ai-payload.json    |   1 +
 .../otel-benchmark/scripts/benchmark-ai.sh    | 128 ++++++++++++++++++
 examples/otel-benchmark/scripts/report-ai.sh  |  93 +++++++++++++
 .../manifests/inference-sim.yaml              |   1 +
 .../manifests/otel-collector.yaml             |  10 +-
 9 files changed, 363 insertions(+), 41 deletions(-)
 create mode 100644 examples/otel-benchmark/configs/ai-otel-full.yaml
 create mode 100644 examples/otel-benchmark/scripts/ai-payload.json
 create mode 100755 examples/otel-benchmark/scripts/benchmark-ai.sh
 create mode 100755 examples/otel-benchmark/scripts/report-ai.sh

diff --git a/examples/otel-benchmark.yaml b/examples/otel-benchmark.yaml
index a500923..2bb7db3 100644
--- a/examples/otel-benchmark.yaml
+++ b/examples/otel-benchmark.yaml
@@ -120,8 +120,13 @@ spec:
           chart: community-charts/mlflow
           version: "{{ cluster.properties.mlflowVersion }}"
           namespace: mlflow
+          values:
+            backendStore:
+              databaseMigration: true
+            service:
+              type: NodePort
         - type: exec
-          command: [bash, -c, "kubectl -n mlflow patch svc mlflow --type=merge -p '{\"spec\":{\"type\":\"NodePort\"}}' && kubectl -n mlflow patch svc mlflow --type=json -p '[{\"op\":\"replace\",\"path\":\"/spec/ports/0/nodePort\",\"value\":30500}]'"]
+          command: [bash, -c, "kubectl -n mlflow patch svc mlflow --type=json -p '[{\"op\":\"replace\",\"path\":\"/spec/ports/0/nodePort\",\"value\":30500}]'"]
 
     mock-backends:
       description: Fortio echo + llm-d inference-sim
diff --git a/examples/otel-benchmark/README.md b/examples/otel-benchmark/README.md
index 7c92b13..8001271 100644
--- a/examples/otel-benchmark/README.md
+++ b/examples/otel-benchmark/README.md
@@ -1,15 +1,15 @@
 # OTel Observability Benchmark
 
 Deploys a full observability stack on KIND for benchmarking Praxis proxy
-OTel tracing overhead.
+OTel tracing overhead across two scenarios: core proxy and AI proxy.
 
 ## Stack
 
-- **Prometheus + Grafana** (kube-prometheus-stack) — metrics + visualization
+- **Prometheus + Grafana 11.x** — metrics + visualization
 - **Tempo** — distributed trace storage
 - **Loki + Promtail** — log aggregation
-- **OTel Collector** — trace pipeline (OTLP → Tempo + MLflow)
-- **MLflow** — experiment tracking
+- **OTel Collector** — trace pipeline (OTLP -> Tempo)
+- **MLflow** — experiment tracking UI
 - **Fortio echo** — mock HTTP backend
 - **llm-d inference-sim** — mock LLM backend
 - **Praxis proxy** — the proxy under test (baseline + OTel variants)
@@ -20,7 +20,7 @@ OTel tracing overhead.
 - [KIND](https://kind.sigs.k8s.io/)
 - [Helm](https://helm.sh/) with repos: `prometheus-community`, `grafana`, `community-charts`
 - [vegeta](https://github.com/tsenart/vegeta) (for benchmarks)
-- Praxis source checkout with OTel PRs (for image builds)
+- Praxis source checkouts (for image builds)
 
 ## Quick Start
 
@@ -31,43 +31,44 @@ helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add community-charts https://community-charts.github.io/helm-charts
 helm repo update
 
-# 2. Set the path to your praxis repo checkout
-export PRAXIS_DIR=/path/to/praxis
+# 2. Set paths to source checkouts
+export PRAXIS_DIR=/path/to/praxis    # praxis core repo (with OTel PRs)
+export AI_DIR=/path/to/ai            # praxis AI repo
 
-# 3. Build praxis images (if not already built)
+# 3. Build images
+# Core: baseline + OTel
 cd "$PRAXIS_DIR"
 docker build -t praxis:dev -f Containerfile .
 sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' \
   Containerfile | docker build -t praxis:dev-otel -f - .
 
+# AI: baseline (from upstream/main, no patches)
+cd "$AI_DIR"
+docker build -t praxis-ai:dev -f Containerfile .
+
+# AI: OTel (from otel-fixes branch with praxis core patches)
+# See "Building praxis-ai:dev-otel" section below.
+
 # 4. Deploy the full stack
 cd /path/to/forge
 cargo run -- up --config examples/otel-benchmark.yaml
 
-# If forge up doesn't apply stacks automatically, apply them manually:
+# If forge up doesn't apply stacks automatically:
 for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-images praxis-deploy dashboards datasources; do
   cargo run -- stack apply --config examples/otel-benchmark.yaml local "$stack"
 done
 
-# 5. Switch to OTel image (if deployed with baseline)
-kubectl --context kind-otel-bench-local set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel
-
-# 6. Verify
-curl http://localhost:18080/         # Praxis proxy
-open http://localhost:13000          # Grafana (admin/admin)
-open http://localhost:19090          # Prometheus
-open http://localhost:15000          # MLflow
+# 5. Verify
+curl http://localhost:18080/
+open http://localhost:13000    # Grafana (admin/admin)
+open http://localhost:19090    # Prometheus
+open http://localhost:15000    # MLflow
 ```
 
-## Run Benchmark
+## Scenario 1: Core Praxis OTel Benchmark
 
-> **Note:** This benchmark uses the core `praxis` proxy, which generates
-> a root request span, per-filter child spans, and an upstream exchange
-> span (10 spans per request). To see AI-specific routing spans
-> (`routing.select` with provider/cluster/site attributes), build from
-> the `praxis-proxy/ai` repo with `--features opentelemetry` and use
-> AI filters (intelligent_route, format classification) with
-> inference-sim as the backend. That is a separate demo configuration.
+Measures OTel tracing overhead on the core proxy with echo backend.
+Generates 10 spans per request (root + 8 per-filter + upstream_exchange).
 
 ```bash
 bash examples/otel-benchmark/scripts/benchmark.sh
@@ -76,13 +77,78 @@ bash examples/otel-benchmark/scripts/benchmark.sh
 Runs 3 configurations at 2000 RPS for 30s each:
 - **A: Baseline** — `praxis:dev` (no OTel feature)
 - **B: OTel noop** — `praxis:dev-otel` (spans created, not exported)
-- **C: OTel full** — `praxis:dev-otel` (spans exported to collector → Tempo)
+- **C: OTel full** — `praxis:dev-otel` (spans exported to collector -> Tempo)
 
-Generate the markdown report:
+Generate the report:
 ```bash
 bash examples/otel-benchmark/scripts/report.sh <results-dir>
 ```
 
+## Scenario 2: AI Praxis with Inference Sim
+
+Measures OTel overhead on the AI proxy with mock LLM backend.
+Generates 11 spans per request (core spans + response_body phase).
+Sends `POST /v1/chat/completions` to inference-sim.
+
+```bash
+bash examples/otel-benchmark/scripts/benchmark-ai.sh
+```
+
+Runs 3 configurations at 500 RPS for 30s each:
+- **A: AI Baseline** — `praxis-ai:dev` (no OTel feature)
+- **B: AI OTel noop** — `praxis-ai:dev-otel` (spans created, not exported)
+- **C: AI OTel full** — `praxis-ai:dev-otel` (spans exported to collector -> Tempo)
+
+Generate the report:
+```bash
+bash examples/otel-benchmark/scripts/report-ai.sh <results-dir>
+```
+
+### Span tree (AI request)
+
+```
+POST /v1/chat/completions -> inference-sim  (root)
+  |-- filter:request_id:request
+  |-- filter:access_log:request
+  |-- filter:router:request          -> routes /v1/* to inference cluster
+  |-- filter:load_balancer:request
+  |-- filter:load_balancer:response
+  |-- filter:router:response
+  |-- filter:access_log:response
+  |-- filter:request_id:response
+  |-- filter:access_log:response_body
+  +-- upstream_exchange [inference-sim:8000]
+```
+
+## Building praxis-ai:dev-otel
+
+The AI OTel image requires patched praxis core (for OTel spans) and the
+AI OTel fixes. Build with both repos side by side:
+
+```bash
+BUILD_DIR=$(mktemp -d)
+rsync -a --exclude='.git' --exclude='target' "$AI_DIR/" "$BUILD_DIR/ai/"
+for crate in core filter protocol tls server; do
+    rsync -a --exclude='target' "$PRAXIS_DIR/$crate/" "$BUILD_DIR/praxis/$crate/"
+done
+cp "$PRAXIS_DIR/Cargo.toml" "$PRAXIS_DIR/Cargo.lock" "$BUILD_DIR/praxis/"
+
+# Add patch.crates-io to ai/Cargo.toml pointing to ../praxis/*
+cat >> "$BUILD_DIR/ai/Cargo.toml" << 'PATCH'
+
+[patch.crates-io]
+praxis-proxy-core = { path = "../praxis/core" }
+praxis-proxy-filter = { path = "../praxis/filter" }
+praxis-proxy-protocol = { path = "../praxis/protocol" }
+praxis-proxy-tls = { path = "../praxis/tls" }
+praxis-proxy = { path = "../praxis/server" }
+PATCH
+
+# Build with OTel features enabled
+# (requires custom Containerfile that copies both repos)
+docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR"
+```
+
 ## Dashboards
 
 | Dashboard | URL | What it shows |
@@ -97,7 +163,7 @@ bash examples/otel-benchmark/scripts/report.sh <results-dir>
 
 | View | URL |
 |------|-----|
-| Tempo trace search | http://localhost:13000/explore (select Tempo datasource → Search tab) |
+| Tempo trace search | http://localhost:13000/explore (select Tempo datasource) |
 | Prometheus metrics | http://localhost:13000/explore (select Prometheus datasource) |
 | Loki log search | http://localhost:13000/explore (select Loki datasource) |
 
@@ -124,7 +190,8 @@ bash examples/otel-benchmark/scripts/report.sh <results-dir>
 
 - **Grafana version**: Must use 11.x (pinned via `grafana.image.tag`). Grafana 12.0 has rendering bugs with provisioned dashboards using `row`/`gauge` panel types.
 - **Datasources**: Prometheus and Tempo datasources are added via the `datasources` stack. If Grafana restarts, they need re-adding.
-- **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export. This fix is not yet in any upstream PR.
+- **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export.
+- **MLflow trace ingestion**: The OTel collector v0.108 sends protobuf to MLflow, but MLflow 3.x only accepts JSON OTLP. Traces go to Tempo (primary store). MLflow shows the experiment tracking UI.
 
 ## Teardown
 
diff --git a/examples/otel-benchmark/configs/ai-otel-full.yaml b/examples/otel-benchmark/configs/ai-otel-full.yaml
new file mode 100644
index 0000000..3f26e9c
--- /dev/null
+++ b/examples/otel-benchmark/configs/ai-otel-full.yaml
@@ -0,0 +1,35 @@
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+telemetry:
+  otlp_endpoint: "http://otel-collector.otel.svc:4317"
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+      - filter: access_log
+
+      - filter: router
+        routes:
+          - path_prefix: "/v1/"
+            cluster: inference
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
+          - name: inference
+            endpoints:
+              - "inference-sim.default.svc:8000"
diff --git a/examples/otel-benchmark/manifests/praxis.yaml b/examples/otel-benchmark/manifests/praxis.yaml
index 2279cb6..0ac64f8 100644
--- a/examples/otel-benchmark/manifests/praxis.yaml
+++ b/examples/otel-benchmark/manifests/praxis.yaml
@@ -1,8 +1,8 @@
 # Praxis Deployment for benchmark mode.
 # ConfigMap is NOT included here — setup.sh creates it separately
 # to avoid overwriting with empty data.
-# No args are specified — the Containerfile ENTRYPOINT already
-# includes [-c, /etc/praxis/config.yaml].
+# The core praxis Containerfile includes [-c, /etc/praxis/config.yaml]
+# in ENTRYPOINT but the AI Containerfile does not — args are set below.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
diff --git a/examples/otel-benchmark/scripts/ai-payload.json b/examples/otel-benchmark/scripts/ai-payload.json
new file mode 100644
index 0000000..9f03b09
--- /dev/null
+++ b/examples/otel-benchmark/scripts/ai-payload.json
@@ -0,0 +1 @@
+{"model":"test-model","messages":[{"role":"user","content":"hello"}]}
diff --git a/examples/otel-benchmark/scripts/benchmark-ai.sh b/examples/otel-benchmark/scripts/benchmark-ai.sh
new file mode 100755
index 0000000..e986875
--- /dev/null
+++ b/examples/otel-benchmark/scripts/benchmark-ai.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+KIND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+CLUSTER_NAME="${CLUSTER_NAME:-otel-bench-local}"
+CTX="kind-${CLUSTER_NAME}"
+GATEWAY_URL="http://localhost:18080"
+
+RATE="${RATE:-500}"
+DURATION="${DURATION:-30s}"
+RUNS="${RUNS:-3}"
+
+RESULTS_DIR="${KIND_DIR}/results/ai-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "${RESULTS_DIR}"
+
+echo "=== Praxis AI OTel Benchmark ==="
+echo "Rate: ${RATE} RPS | Duration: ${DURATION} | Runs: ${RUNS}"
+echo "Results: ${RESULTS_DIR}"
+echo ""
+
+run_vegeta() {
+  local label="$1"
+  local run="$2"
+  echo "--- ${label} run ${run}/${RUNS} ---"
+  printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \
+    "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \
+    vegeta attack -rate="${RATE}" -duration="${DURATION}" -connections=100 | \
+    tee "${RESULTS_DIR}/${label}-run${run}.bin" | \
+    vegeta report -type=json > "${RESULTS_DIR}/${label}-run${run}.json"
+  vegeta report < "${RESULTS_DIR}/${label}-run${run}.bin"
+  kubectl --context "${CTX}" top pod -n default --no-headers 2>/dev/null \
+    >> "${RESULTS_DIR}/${label}-resources.txt" || true
+  echo ""
+}
+
+AI_CONFIG="${KIND_DIR}/configs/ai-otel-full.yaml"
+
+# ---- Run A: AI Baseline (no OTel) ----
+echo "=========================================="
+echo "  Run A: AI Baseline (praxis-ai:dev, no OTel)"
+echo "=========================================="
+
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${AI_CONFIG}" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
+# praxis-ai:dev has ENTRYPOINT [praxis-ai] — needs -c arg
+kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \
+  -p '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"praxis-ai:dev"},{"op":"replace","path":"/spec/template/spec/containers/0/args","value":["-c","/etc/praxis/config.yaml"]}]'
+kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \
+  "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \
+  vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "ai-baseline" "${i}"
+  sleep 5
+done
+
+# ---- Run B: AI OTel noop (spans created, not exported) ----
+echo "=========================================="
+echo "  Run B: AI OTel noop (praxis-ai:dev-otel, no endpoint)"
+echo "=========================================="
+
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${AI_CONFIG}" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
+# praxis-ai:dev-otel has ENTRYPOINT [praxis -c /etc/praxis/config.yaml] — clear args
+kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \
+  -p '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"praxis-ai:dev-otel"},{"op":"replace","path":"/spec/template/spec/containers/0/args","value":[]}]'
+kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \
+  "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \
+  vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "ai-otel-noop" "${i}"
+  sleep 5
+done
+
+# ---- Run C: AI OTel full (spans exported to collector) ----
+echo "=========================================="
+echo "  Run C: AI OTel full (praxis-ai:dev-otel, exporting)"
+echo "=========================================="
+
+kubectl --context "${CTX}" set env deployment/praxis-proxy \
+  OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.otel.svc:4317 -n default
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
+sleep 3
+kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default
+kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s
+sleep 5
+
+echo "Warmup..."
+printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \
+  "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \
+  vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true
+sleep 2
+
+for i in $(seq 1 "${RUNS}"); do
+  run_vegeta "ai-otel-full" "${i}"
+  sleep 5
+done
+
+echo "=========================================="
+echo "  AI Benchmark complete"
+echo "=========================================="
+echo "Results in: ${RESULTS_DIR}"
+echo ""
+echo "Generate report:"
+echo "  bash ${SCRIPT_DIR}/report-ai.sh ${RESULTS_DIR}"
diff --git a/examples/otel-benchmark/scripts/report-ai.sh b/examples/otel-benchmark/scripts/report-ai.sh
new file mode 100755
index 0000000..555517c
--- /dev/null
+++ b/examples/otel-benchmark/scripts/report-ai.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RESULTS_DIR="${1:?Usage: report-ai.sh <results-dir>}"
+
+if [ ! -d "${RESULTS_DIR}" ]; then
+  echo "Error: ${RESULTS_DIR} does not exist"
+  exit 1
+fi
+
+export BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
+export COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+export RESULTS_DIR
+
+python3 << 'PYTHON_SCRIPT'
+import json
+import glob
+import os
+from datetime import datetime
+
+results_dir = os.environ['RESULTS_DIR']
+branch = os.environ.get('BRANCH', 'unknown')
+commit = os.environ.get('COMMIT', 'unknown')
+
+configs = ['ai-baseline', 'ai-otel-noop', 'ai-otel-full']
+stats = {}
+
+for config in configs:
+    total_p50 = 0
+    total_p99 = 0
+    total_rps = 0
+    count = 0
+
+    pattern = os.path.join(results_dir, f'{config}-run*.json')
+    for path in glob.glob(pattern):
+        try:
+            with open(path) as f:
+                data = json.load(f)
+                total_p50 += data['latencies']['50th']
+                total_p99 += data['latencies']['99th']
+                total_rps += data['throughput']
+                count += 1
+        except (json.JSONDecodeError, KeyError) as e:
+            print(f"Warning: Failed to parse {path}: {e}")
+            continue
+
+    if count > 0:
+        stats[config] = {
+            'p50': int(total_p50 / count / 1000),
+            'p99': int(total_p99 / count / 1000),
+            'rps': int(total_rps / count),
+            'runs': count
+        }
+    else:
+        stats[config] = {'p50': 0, 'p99': 0, 'rps': 0, 'runs': 0}
+
+report_path = os.path.join(results_dir, 'report.md')
+with open(report_path, 'w') as f:
+    f.write(f"# Praxis AI OTel Overhead Benchmark\n\n")
+    f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d')} | **Commit:** {commit} | **Branch:** {branch}\n")
+    f.write(f"**Platform:** KIND (1 node) | **Backend:** inference-sim (mock LLM)\n\n")
+    f.write(f"## Summary\n\n")
+    f.write(f"| Config | P50 (us) | P99 (us) | RPS | Runs | P50 delta | P99 delta |\n")
+    f.write(f"|--------|----------|----------|-----|------|-----------|-----------|\n")
+
+    base_p50 = stats['ai-baseline']['p50']
+    base_p99 = stats['ai-baseline']['p99']
+
+    for config in configs:
+        s = stats[config]
+        p50 = s['p50']
+        p99 = s['p99']
+        rps = s['rps']
+        runs = s['runs']
+
+        if config == 'ai-baseline':
+            f.write(f"| AI Baseline | {p50} | {p99} | {rps} | {runs} | -- | -- |\n")
+        else:
+            label = 'AI OTel (noop)' if config == 'ai-otel-noop' else 'AI OTel (full)'
+
+            if base_p50 > 0:
+                dp50 = f"{(p50 - base_p50) / base_p50 * 100:+.1f}%"
+                dp99 = f"{(p99 - base_p99) / base_p99 * 100:+.1f}%"
+            else:
+                dp50 = "N/A"
+                dp99 = "N/A"
+
+            f.write(f"| {label} | {p50} | {p99} | {rps} | {runs} | {dp50} | {dp99} |\n")
+
+print(f"Report written to: {report_path}")
+PYTHON_SCRIPT
+
+cat "${RESULTS_DIR}/report.md"
diff --git a/examples/stacks/mock-backends/manifests/inference-sim.yaml b/examples/stacks/mock-backends/manifests/inference-sim.yaml
index bfc1811..035de75 100644
--- a/examples/stacks/mock-backends/manifests/inference-sim.yaml
+++ b/examples/stacks/mock-backends/manifests/inference-sim.yaml
@@ -16,6 +16,7 @@ spec:
       containers:
         - name: sim
           image: ghcr.io/llm-d/llm-d-inference-sim:latest
+          args: ["--port", "8000", "--model", "test-model", "--mode", "random"]
           ports:
             - containerPort: 8000
           resources:
diff --git a/examples/stacks/observability/manifests/otel-collector.yaml b/examples/stacks/observability/manifests/otel-collector.yaml
index bae9a06..452cb67 100644
--- a/examples/stacks/observability/manifests/otel-collector.yaml
+++ b/examples/stacks/observability/manifests/otel-collector.yaml
@@ -25,20 +25,12 @@ data:
         endpoint: tempo.monitoring.svc:4317
         tls:
           insecure: true
-      otlphttp/mlflow:
-        endpoint: http://mlflow.mlflow.svc:5000
-        tls:
-          insecure: true
-        headers:
-          x-mlflow-experiment-id: "0"
-          Host: "localhost:5000"
-
     service:
       pipelines:
         traces:
           receivers: [otlp]
           processors: [batch]
-          exporters: [debug, otlp/tempo, otlphttp/mlflow]
+          exporters: [debug, otlp/tempo]
 ---
 apiVersion: apps/v1
 kind: Deployment

From 53155ddef02a8517aab9f45e13730bb0e3526e21 Mon Sep 17 00:00:00 2001
From: Ladislav Smola <lsmola@redhat.com>
Date: Sat, 15 Aug 2026 09:52:12 +0200
Subject: [PATCH 4/5] fix: update README to use praxis-forge CLI and document
 stack apply workflow

Signed-off-by: Ladislav Smola <lsmola@redhat.com>
---
 examples/otel-benchmark/README.md | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/examples/otel-benchmark/README.md b/examples/otel-benchmark/README.md
index 8001271..b05ff9a 100644
--- a/examples/otel-benchmark/README.md
+++ b/examples/otel-benchmark/README.md
@@ -51,12 +51,19 @@ docker build -t praxis-ai:dev -f Containerfile .
 
 # 4. Deploy the full stack
 cd /path/to/forge
-cargo run -- up --config examples/otel-benchmark.yaml
+praxis-forge doctor                                            # check tools
+praxis-forge plan --config examples/otel-benchmark.yaml        # preview
+praxis-forge up --config examples/otel-benchmark.yaml          # create cluster
 
-# If forge up doesn't apply stacks automatically:
-for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-images praxis-deploy dashboards datasources; do
-  cargo run -- stack apply --config examples/otel-benchmark.yaml local "$stack"
+# Load pre-built images into KIND
+kind load docker-image praxis:dev praxis:dev-otel --name otel-bench-local
+kind load docker-image praxis-ai:dev praxis-ai:dev-otel --name otel-bench-local
+
+# Apply stacks (skip praxis-images if images are already loaded)
+for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-deploy dashboards datasources; do
+  praxis-forge apply --config examples/otel-benchmark.yaml local "$stack"
 done
+praxis-forge status --config examples/otel-benchmark.yaml      # check status
 
 # 5. Verify
 curl http://localhost:18080/
@@ -196,7 +203,7 @@ docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR"
 ## Teardown
 
 ```bash
-cargo run -- down --config examples/otel-benchmark.yaml
+praxis-forge down --config examples/otel-benchmark.yaml
 # or
 kind delete cluster --name otel-bench-local
 ```

From 23d5772c42eb40c66654ff521fec17490c477707 Mon Sep 17 00:00:00 2001
From: Ladislav Smola <lsmola@redhat.com>
Date: Sat, 15 Aug 2026 10:43:47 +0200
Subject: [PATCH 5/5] fix: align benchmark configs, README accuracy, and image
 pinning

- Add access_log filter to baseline and otel-noop configs so all three
  core benchmark runs use identical filter chains (4 filters, 10 spans)
- Create ai-baseline.yaml and ai-otel-noop.yaml for per-run config
  isolation in the AI benchmark (prevents noop from exporting via
  hardcoded otlp_endpoint in ai-otel-full.yaml)
- Update benchmark-ai.sh to use per-run config files
- Fix README dashboard titles to match actual Grafana JSON titles
- Fix Known Issues: collector exports to Tempo only, not MLflow
- Fix otel-collector stack description (remove "+ MLflow")
- Fix praxis.yaml stale comments (setup.sh reference, phantom args)
- Add labels to Service metadata so ServiceMonitor selector matches
- Pin mock backend images (fortio:1.75.2, inference-sim:v0.10.2)

Signed-off-by: Ladislav Smola <lsmola@redhat.com>
---
 examples/otel-benchmark.yaml                  |  2 +-
 examples/otel-benchmark/README.md             | 10 ++---
 .../otel-benchmark/configs/ai-baseline.yaml   | 35 ++++++++++++++++++
 .../otel-benchmark/configs/ai-otel-noop.yaml  | 37 +++++++++++++++++++
 examples/otel-benchmark/configs/baseline.yaml |  2 +
 .../otel-benchmark/configs/otel-noop.yaml     |  2 +
 examples/otel-benchmark/manifests/praxis.yaml |  8 ++--
 .../otel-benchmark/scripts/benchmark-ai.sh    |  9 +++--
 .../mock-backends/manifests/echo-backend.yaml |  2 +-
 .../manifests/inference-sim.yaml              |  2 +-
 10 files changed, 93 insertions(+), 16 deletions(-)
 create mode 100644 examples/otel-benchmark/configs/ai-baseline.yaml
 create mode 100644 examples/otel-benchmark/configs/ai-otel-noop.yaml

diff --git a/examples/otel-benchmark.yaml b/examples/otel-benchmark.yaml
index 2bb7db3..b1a07b8 100644
--- a/examples/otel-benchmark.yaml
+++ b/examples/otel-benchmark.yaml
@@ -100,7 +100,7 @@ spec:
               enabled: true
 
     otel-collector:
-      description: OpenTelemetry Collector (OTLP → Tempo + MLflow)
+      description: OpenTelemetry Collector (OTLP → Tempo)
       steps:
         - type: exec
           command: [bash, -c, "kubectl create namespace otel --dry-run=client -o yaml | kubectl apply -f -"]
diff --git a/examples/otel-benchmark/README.md b/examples/otel-benchmark/README.md
index b05ff9a..f835f27 100644
--- a/examples/otel-benchmark/README.md
+++ b/examples/otel-benchmark/README.md
@@ -161,10 +161,10 @@ docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR"
 | Dashboard | URL | What it shows |
 |-----------|-----|------|
 | Praxis Proxy Overview | http://localhost:13000/d/praxis-proxy-overview | Request rate, latency P50/P99, requests by method |
-| OTel Traces | http://localhost:13000/d/praxis-traces | Searchable trace table with clickable Trace IDs |
-| Benchmark Results | http://localhost:13000/d/praxis-benchmark | CPU/memory for praxis + collector, RPS, latency |
-| AI/LLM Golden Signals | http://localhost:13000/d/praxis-ai-golden-signals | P95 latency stat, throughput, AI token metrics (future) |
-| Structured Logs | http://localhost:13000/d/praxis-logs | Log volume, error logs, all praxis access logs |
+| Praxis OTel Traces | http://localhost:13000/d/praxis-traces | Searchable trace table with clickable Trace IDs |
+| Praxis Benchmark Results | http://localhost:13000/d/praxis-benchmark | CPU/memory for praxis + collector, RPS, latency |
+| Praxis AI/LLM Golden Signals | http://localhost:13000/d/praxis-ai-golden-signals | P95 latency stat, throughput, AI token metrics (future) |
+| Praxis Structured Logs | http://localhost:13000/d/praxis-logs | Log volume, error logs, all praxis access logs |
 
 ### Explore views
 
@@ -198,7 +198,7 @@ docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR"
 - **Grafana version**: Must use 11.x (pinned via `grafana.image.tag`). Grafana 12.0 has rendering bugs with provisioned dashboards using `row`/`gauge` panel types.
 - **Datasources**: Prometheus and Tempo datasources are added via the `datasources` stack. If Grafana restarts, they need re-adding.
 - **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export.
-- **MLflow trace ingestion**: The OTel collector v0.108 sends protobuf to MLflow, but MLflow 3.x only accepts JSON OTLP. Traces go to Tempo (primary store). MLflow shows the experiment tracking UI.
+- **MLflow trace ingestion**: The OTel collector exports traces to Tempo only. MLflow is deployed for experiment tracking UI but does not receive trace data from the collector pipeline.
 
 ## Teardown
 
diff --git a/examples/otel-benchmark/configs/ai-baseline.yaml b/examples/otel-benchmark/configs/ai-baseline.yaml
new file mode 100644
index 0000000..f8d21ac
--- /dev/null
+++ b/examples/otel-benchmark/configs/ai-baseline.yaml
@@ -0,0 +1,35 @@
+# AI baseline benchmark config (no OTel).
+# Routes /v1/* to inference-sim, everything else to Fortio echo.
+
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+      - filter: access_log
+
+      - filter: router
+        routes:
+          - path_prefix: "/v1/"
+            cluster: inference
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
+          - name: inference
+            endpoints:
+              - "inference-sim.default.svc:8000"
diff --git a/examples/otel-benchmark/configs/ai-otel-noop.yaml b/examples/otel-benchmark/configs/ai-otel-noop.yaml
new file mode 100644
index 0000000..ae73660
--- /dev/null
+++ b/examples/otel-benchmark/configs/ai-otel-noop.yaml
@@ -0,0 +1,37 @@
+# AI OTel noop benchmark config.
+# OTel feature is compiled in but no OTLP endpoint is configured,
+# so spans are created but never exported.
+# Routes /v1/* to inference-sim, everything else to Fortio echo.
+
+admin:
+  address: "0.0.0.0:9901"
+
+insecure_options:
+  allow_public_admin: true
+
+listeners:
+  - name: default
+    address: "0.0.0.0:8080"
+    filter_chains: [main]
+
+filter_chains:
+  - name: main
+    filters:
+      - filter: request_id
+      - filter: access_log
+
+      - filter: router
+        routes:
+          - path_prefix: "/v1/"
+            cluster: inference
+          - path_prefix: "/"
+            cluster: echo
+
+      - filter: load_balancer
+        clusters:
+          - name: echo
+            endpoints:
+              - "echo-backend.default.svc:8080"
+          - name: inference
+            endpoints:
+              - "inference-sim.default.svc:8000"
diff --git a/examples/otel-benchmark/configs/baseline.yaml b/examples/otel-benchmark/configs/baseline.yaml
index 70389c2..856b1d9 100644
--- a/examples/otel-benchmark/configs/baseline.yaml
+++ b/examples/otel-benchmark/configs/baseline.yaml
@@ -17,6 +17,8 @@ filter_chains:
     filters:
       - filter: request_id
 
+      - filter: access_log
+
       - filter: router
         routes:
           - path_prefix: "/"
diff --git a/examples/otel-benchmark/configs/otel-noop.yaml b/examples/otel-benchmark/configs/otel-noop.yaml
index 6d51b7c..8a34b70 100644
--- a/examples/otel-benchmark/configs/otel-noop.yaml
+++ b/examples/otel-benchmark/configs/otel-noop.yaml
@@ -18,6 +18,8 @@ filter_chains:
     filters:
       - filter: request_id
 
+      - filter: access_log
+
       - filter: router
         routes:
           - path_prefix: "/"
diff --git a/examples/otel-benchmark/manifests/praxis.yaml b/examples/otel-benchmark/manifests/praxis.yaml
index 0ac64f8..022dc0a 100644
--- a/examples/otel-benchmark/manifests/praxis.yaml
+++ b/examples/otel-benchmark/manifests/praxis.yaml
@@ -1,8 +1,6 @@
 # Praxis Deployment for benchmark mode.
-# ConfigMap is NOT included here — setup.sh creates it separately
-# to avoid overwriting with empty data.
-# The core praxis Containerfile includes [-c, /etc/praxis/config.yaml]
-# in ENTRYPOINT but the AI Containerfile does not — args are set below.
+# ConfigMap is created by the praxis-deploy stack (otel-benchmark.yaml)
+# or by benchmark.sh/benchmark-ai.sh when swapping configs between runs.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -58,6 +56,8 @@ kind: Service
 metadata:
   name: praxis-proxy
   namespace: default
+  labels:
+    app: praxis-proxy
 spec:
   type: NodePort
   selector:
diff --git a/examples/otel-benchmark/scripts/benchmark-ai.sh b/examples/otel-benchmark/scripts/benchmark-ai.sh
index e986875..51e62ce 100755
--- a/examples/otel-benchmark/scripts/benchmark-ai.sh
+++ b/examples/otel-benchmark/scripts/benchmark-ai.sh
@@ -35,15 +35,13 @@ run_vegeta() {
   echo ""
 }
 
-AI_CONFIG="${KIND_DIR}/configs/ai-otel-full.yaml"
-
 # ---- Run A: AI Baseline (no OTel) ----
 echo "=========================================="
 echo "  Run A: AI Baseline (praxis-ai:dev, no OTel)"
 echo "=========================================="
 
 kubectl --context "${CTX}" create configmap praxis-config \
-  --from-file=config.yaml="${AI_CONFIG}" \
+  --from-file=config.yaml="${KIND_DIR}/configs/ai-baseline.yaml" \
   -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
 # praxis-ai:dev has ENTRYPOINT [praxis-ai] — needs -c arg
 kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \
@@ -72,7 +70,7 @@ echo "  Run B: AI OTel noop (praxis-ai:dev-otel, no endpoint)"
 echo "=========================================="
 
 kubectl --context "${CTX}" create configmap praxis-config \
-  --from-file=config.yaml="${AI_CONFIG}" \
+  --from-file=config.yaml="${KIND_DIR}/configs/ai-otel-noop.yaml" \
   -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
 # praxis-ai:dev-otel has ENTRYPOINT [praxis -c /etc/praxis/config.yaml] — clear args
 kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \
@@ -100,6 +98,9 @@ echo "=========================================="
 echo "  Run C: AI OTel full (praxis-ai:dev-otel, exporting)"
 echo "=========================================="
 
+kubectl --context "${CTX}" create configmap praxis-config \
+  --from-file=config.yaml="${KIND_DIR}/configs/ai-otel-full.yaml" \
+  -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f -
 kubectl --context "${CTX}" set env deployment/praxis-proxy \
   OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.otel.svc:4317 -n default
 kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default
diff --git a/examples/stacks/mock-backends/manifests/echo-backend.yaml b/examples/stacks/mock-backends/manifests/echo-backend.yaml
index cd4b15b..5eeae10 100644
--- a/examples/stacks/mock-backends/manifests/echo-backend.yaml
+++ b/examples/stacks/mock-backends/manifests/echo-backend.yaml
@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
         - name: fortio
-          image: fortio/fortio:latest
+          image: fortio/fortio:1.75.2
           args: ["server", "-echo-server-default-params", "delay=0"]
           ports:
             - containerPort: 8080
diff --git a/examples/stacks/mock-backends/manifests/inference-sim.yaml b/examples/stacks/mock-backends/manifests/inference-sim.yaml
index 035de75..16badc7 100644
--- a/examples/stacks/mock-backends/manifests/inference-sim.yaml
+++ b/examples/stacks/mock-backends/manifests/inference-sim.yaml
@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
         - name: sim
-          image: ghcr.io/llm-d/llm-d-inference-sim:latest
+          image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
           args: ["--port", "8000", "--model", "test-model", "--mode", "random"]
           ports:
             - containerPort: 8000