feat: add OTel observability benchmark demo

.gitignore

@@ -19,3 +19,4 @@ target
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.forge/

examples/otel-benchmark.yaml

@@ -0,0 +1,180 @@
+apiVersion: forge.praxis.dev/v1alpha1
+kind: Environment
+metadata:
+  name: otel-benchmark
+spec:
+  runtime:
+    provider: docker
+    clusterPrefix: otel-bench
+  network:
+    crossCluster: false
+
+  clusters:
+    - name: local
+      ports:
+        - host: 18080
+          container: 30080    # Praxis proxy
+        - host: 18901
+          container: 30901    # Praxis admin/metrics
+        - host: 13000
+          container: 30300    # Grafana
+        - host: 19090
+          container: 30909    # Prometheus
+        - host: 15000
+          container: 30500    # MLflow
+      stacks:
+        - prometheus
+        - tempo
+        - loki
+        - otel-collector
+        - mlflow
+        - mock-backends
+        - praxis-images
+        - praxis-deploy
+        - dashboards
+        - datasources
+      properties:
+        # Pin chart versions for reproducibility
+        kubePrometheusVersion: "72.6.3"
+        tempoVersion: "1.24.4"
+        lokiStackVersion: "2.10.2"
+        mlflowVersion: "1.11.3"
+
+  stacks:
+    prometheus:
+      description: kube-prometheus-stack (Prometheus + Grafana)
+      steps:
+        - type: helm
+          release: monitoring
+          chart: prometheus-community/kube-prometheus-stack
+          version: "{{ cluster.properties.kubePrometheusVersion }}"
+          namespace: monitoring
+          values:
+            prometheus:
+              prometheusSpec:
+                serviceMonitorSelectorNilUsesHelmValues: false
+                additionalScrapeConfigs:
+                  - job_name: praxis-proxy
+                    metrics_path: /metrics
+                    scrape_interval: 15s
+                    static_configs:
+                      - targets: ["praxis-proxy.default.svc:9901"]
+              service:
+                type: NodePort
+                nodePort: 30909
+            grafana:
+              adminPassword: admin
+              image:
+                tag: 11.6.0
+              service:
+                type: NodePort
+                nodePort: 30300
+        - type: wait
+          resource: deployment/monitoring-grafana
+          namespace: monitoring
+          condition: available
+          timeout: "300s"
+
+    tempo:
+      description: Tempo trace storage
+      steps:
+        - type: helm
+          release: tempo
+          chart: grafana/tempo
+          version: "{{ cluster.properties.tempoVersion }}"
+          namespace: monitoring
+
+    loki:
+      description: Loki + Promtail log aggregation
+      steps:
+        - type: helm
+          release: loki
+          chart: grafana/loki-stack
+          version: "{{ cluster.properties.lokiStackVersion }}"
+          namespace: monitoring
+          values:
+            loki:
+              persistence:
+                enabled: false
+            promtail:
+              enabled: true
+
+    otel-collector:
+      description: OpenTelemetry Collector (OTLP → Tempo)
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl create namespace otel --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: manifest
+          path: stacks/observability/manifests/otel-collector.yaml
+        - type: wait
+          resource: deployment/otel-collector
+          namespace: otel
+          condition: available
+          timeout: "60s"
+
+    mlflow:
+      description: MLflow experiment tracking
+      steps:
+        - type: helm
+          release: mlflow
+          chart: community-charts/mlflow
+          version: "{{ cluster.properties.mlflowVersion }}"
+          namespace: mlflow
+          values:
+            backendStore:
+              databaseMigration: true
+            service:
+              type: NodePort
+        - type: exec
+          command: [bash, -c, "kubectl -n mlflow patch svc mlflow --type=json -p '[{\"op\":\"replace\",\"path\":\"/spec/ports/0/nodePort\",\"value\":30500}]'"]
+
+    mock-backends:
+      description: Fortio echo + llm-d inference-sim
+      steps:
+        - type: manifest
+          path: stacks/mock-backends/manifests/echo-backend.yaml
+        - type: manifest
+          path: stacks/mock-backends/manifests/inference-sim.yaml
+
+    praxis-images:
+      description: Build and load Praxis images into KIND
+      steps:
+        - type: exec
+          command: [bash, -c, "cd \"${PRAXIS_DIR:?Set PRAXIS_DIR to your praxis checkout}\" && docker build -t praxis:dev -f Containerfile ."]
+        - type: exec
+          command: [bash, -c, "cd \"${PRAXIS_DIR:?}\" && sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' Containerfile | docker build -t praxis:dev-otel -f - ."]
+        - type: exec
+          command: [bash, -c, "kind load docker-image praxis:dev --name otel-bench-local"]
+        - type: exec
+          command: [bash, -c, "kind load docker-image praxis:dev-otel --name otel-bench-local"]
+
+    praxis-deploy:
+      description: Deploy Praxis proxy with OTel config
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl create configmap praxis-config --from-file=config.yaml=examples/otel-benchmark/configs/otel-full.yaml -n default --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: manifest
+          path: otel-benchmark/manifests/praxis.yaml
+        - type: manifest
+          path: otel-benchmark/manifests/servicemonitor.yaml
+        - type: wait
+          resource: deployment/praxis-proxy
+          namespace: default
+          condition: available
+          timeout: "120s"
+
+    dashboards:
+      description: Load Grafana dashboards via ConfigMap
+      steps:
+        - type: exec
+          command: [bash, -c, "kubectl -n monitoring create configmap praxis-dashboards --from-file=examples/stacks/observability/dashboards/ --dry-run=client -o yaml | kubectl apply -f -"]
+        - type: exec
+          command: [kubectl, -n, monitoring, label, configmap, praxis-dashboards, grafana_dashboard=1, --overwrite]
+
+    datasources:
+      description: Add Tempo + Loki datasources to Grafana
+      steps:
+        - type: exec
+          command: [bash, -c, "sleep 5 && curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Tempo\",\"type\":\"tempo\",\"access\":\"proxy\",\"url\":\"http://tempo.monitoring.svc:3200\",\"uid\":\"tempo\"}' || true"]
+        - type: exec
+          command: [bash, -c, "curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Loki\",\"type\":\"loki\",\"access\":\"proxy\",\"url\":\"http://loki.monitoring.svc:3100\",\"uid\":\"loki\"}' || true"]

examples/otel-benchmark/README.md

@@ -0,0 +1,209 @@
+# OTel Observability Benchmark
+
+Deploys a full observability stack on KIND for benchmarking Praxis proxy
+OTel tracing overhead across two scenarios: core proxy and AI proxy.
+
+## Stack
+
+- **Prometheus + Grafana 11.x** — metrics + visualization
+- **Tempo** — distributed trace storage
+- **Loki + Promtail** — log aggregation
+- **OTel Collector** — trace pipeline (OTLP -> Tempo)
+- **MLflow** — experiment tracking UI
+- **Fortio echo** — mock HTTP backend
+- **llm-d inference-sim** — mock LLM backend
+- **Praxis proxy** — the proxy under test (baseline + OTel variants)
+
+## Prerequisites
+
+- Docker or Podman
+- [KIND](https://kind.sigs.k8s.io/)
+- [Helm](https://helm.sh/) with repos: `prometheus-community`, `grafana`, `community-charts`
+- [vegeta](https://github.com/tsenart/vegeta) (for benchmarks)
+- Praxis source checkouts (for image builds)
+
+## Quick Start
+
+```bash
+# 1. Add helm repos (one-time)
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo add community-charts https://community-charts.github.io/helm-charts
+helm repo update
+
+# 2. Set paths to source checkouts
+export PRAXIS_DIR=/path/to/praxis    # praxis core repo (with OTel PRs)
+export AI_DIR=/path/to/ai            # praxis AI repo
+
+# 3. Build images
+# Core: baseline + OTel
+cd "$PRAXIS_DIR"
+docker build -t praxis:dev -f Containerfile .
+sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' \
+  Containerfile | docker build -t praxis:dev-otel -f - .
+
+# AI: baseline (from upstream/main, no patches)
+cd "$AI_DIR"
+docker build -t praxis-ai:dev -f Containerfile .
+
+# AI: OTel (from otel-fixes branch with praxis core patches)
+# See "Building praxis-ai:dev-otel" section below.
+
+# 4. Deploy the full stack
+cd /path/to/forge
+praxis-forge doctor                                            # check tools
+praxis-forge plan --config examples/otel-benchmark.yaml        # preview
+praxis-forge up --config examples/otel-benchmark.yaml          # create cluster
+
+# Load pre-built images into KIND
+kind load docker-image praxis:dev praxis:dev-otel --name otel-bench-local
+kind load docker-image praxis-ai:dev praxis-ai:dev-otel --name otel-bench-local
+
+# Apply stacks (skip praxis-images if images are already loaded)
+for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-deploy dashboards datasources; do
+  praxis-forge apply --config examples/otel-benchmark.yaml local "$stack"
+done
+praxis-forge status --config examples/otel-benchmark.yaml      # check status
+
+# 5. Verify
+curl http://localhost:18080/
+open http://localhost:13000    # Grafana (admin/admin)
+open http://localhost:19090    # Prometheus
+open http://localhost:15000    # MLflow
+```
+
+## Scenario 1: Core Praxis OTel Benchmark
+
+Measures OTel tracing overhead on the core proxy with echo backend.
+Generates 10 spans per request (root + 8 per-filter + upstream_exchange).
+
+```bash
+bash examples/otel-benchmark/scripts/benchmark.sh
+```
+
+Runs 3 configurations at 2000 RPS for 30s each:
+- **A: Baseline** — `praxis:dev` (no OTel feature)
+- **B: OTel noop** — `praxis:dev-otel` (spans created, not exported)
+- **C: OTel full** — `praxis:dev-otel` (spans exported to collector -> Tempo)
+
+Generate the report:
+```bash
+bash examples/otel-benchmark/scripts/report.sh <results-dir>
+```
+
+## Scenario 2: AI Praxis with Inference Sim
+
+Measures OTel overhead on the AI proxy with mock LLM backend.
+Generates 11 spans per request (core spans + response_body phase).
+Sends `POST /v1/chat/completions` to inference-sim.
+
+```bash
+bash examples/otel-benchmark/scripts/benchmark-ai.sh
+```
+
+Runs 3 configurations at 500 RPS for 30s each:
+- **A: AI Baseline** — `praxis-ai:dev` (no OTel feature)
+- **B: AI OTel noop** — `praxis-ai:dev-otel` (spans created, not exported)
+- **C: AI OTel full** — `praxis-ai:dev-otel` (spans exported to collector -> Tempo)
+
+Generate the report:
+```bash
+bash examples/otel-benchmark/scripts/report-ai.sh <results-dir>
+```
+
+### Span tree (AI request)
+
+```
+POST /v1/chat/completions -> inference-sim  (root)
+  |-- filter:request_id:request
+  |-- filter:access_log:request
+  |-- filter:router:request          -> routes /v1/* to inference cluster
+  |-- filter:load_balancer:request
+  |-- filter:load_balancer:response
+  |-- filter:router:response
+  |-- filter:access_log:response
+  |-- filter:request_id:response
+  |-- filter:access_log:response_body
+  +-- upstream_exchange [inference-sim:8000]
+```
+
+## Building praxis-ai:dev-otel
+
+The AI OTel image requires patched praxis core (for OTel spans) and the
+AI OTel fixes. Build with both repos side by side:
+
+```bash
+BUILD_DIR=$(mktemp -d)
+rsync -a --exclude='.git' --exclude='target' "$AI_DIR/" "$BUILD_DIR/ai/"
+for crate in core filter protocol tls server; do
+    rsync -a --exclude='target' "$PRAXIS_DIR/$crate/" "$BUILD_DIR/praxis/$crate/"
+done
+cp "$PRAXIS_DIR/Cargo.toml" "$PRAXIS_DIR/Cargo.lock" "$BUILD_DIR/praxis/"
+
+# Add patch.crates-io to ai/Cargo.toml pointing to ../praxis/*
+cat >> "$BUILD_DIR/ai/Cargo.toml" << 'PATCH'
+
+[patch.crates-io]
+praxis-proxy-core = { path = "../praxis/core" }
+praxis-proxy-filter = { path = "../praxis/filter" }
+praxis-proxy-protocol = { path = "../praxis/protocol" }
+praxis-proxy-tls = { path = "../praxis/tls" }
+praxis-proxy = { path = "../praxis/server" }
+PATCH
+
+# Build with OTel features enabled
+# (requires custom Containerfile that copies both repos)
+docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR"
+```
+
+## Dashboards
+
+| Dashboard | URL | What it shows |
+|-----------|-----|------|
+| Praxis Proxy Overview | http://localhost:13000/d/praxis-proxy-overview | Request rate, latency P50/P99, requests by method |
+| Praxis OTel Traces | http://localhost:13000/d/praxis-traces | Searchable trace table with clickable Trace IDs |
+| Praxis Benchmark Results | http://localhost:13000/d/praxis-benchmark | CPU/memory for praxis + collector, RPS, latency |
+| Praxis AI/LLM Golden Signals | http://localhost:13000/d/praxis-ai-golden-signals | P95 latency stat, throughput, AI token metrics (future) |
+| Praxis Structured Logs | http://localhost:13000/d/praxis-logs | Log volume, error logs, all praxis access logs |
+
+### Explore views
+
+| View | URL |
+|------|-----|
+| Tempo trace search | http://localhost:13000/explore (select Tempo datasource) |
+| Prometheus metrics | http://localhost:13000/explore (select Prometheus datasource) |
+| Loki log search | http://localhost:13000/explore (select Loki datasource) |
+
+### Other UIs
+
+| Service | URL |
+|---------|-----|
+| Prometheus | http://localhost:19090 |
+| MLflow | http://localhost:15000 |
+| Praxis proxy | http://localhost:18080 |
+| Praxis admin/metrics | http://localhost:18901/metrics |
+
+## Host Ports
+
+| Port | Service | KIND NodePort |
+|------|---------|------|
+| 18080 | Praxis proxy | 30080 |
+| 18901 | Praxis admin | 30901 |
+| 13000 | Grafana | 30300 |
+| 19090 | Prometheus | 30909 |
+| 15000 | MLflow | 30500 |
+
+## Known Issues
+
+- **Grafana version**: Must use 11.x (pinned via `grafana.image.tag`). Grafana 12.0 has rendering bugs with provisioned dashboards using `row`/`gauge` panel types.
+- **Datasources**: Prometheus and Tempo datasources are added via the `datasources` stack. If Grafana restarts, they need re-adding.
+- **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export.
+- **MLflow trace ingestion**: The OTel collector exports traces to Tempo only. MLflow is deployed for experiment tracking UI but does not receive trace data from the collector pipeline.
+
+## Teardown
+
+```bash
+praxis-forge down --config examples/otel-benchmark.yaml
+# or
+kind delete cluster --name otel-bench-local
+```