diff --git a/.gitignore b/.gitignore index ad67955..369d0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +.forge/ diff --git a/examples/otel-benchmark.yaml b/examples/otel-benchmark.yaml new file mode 100644 index 0000000..b1a07b8 --- /dev/null +++ b/examples/otel-benchmark.yaml @@ -0,0 +1,180 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment +metadata: + name: otel-benchmark +spec: + runtime: + provider: docker + clusterPrefix: otel-bench + network: + crossCluster: false + + clusters: + - name: local + ports: + - host: 18080 + container: 30080 # Praxis proxy + - host: 18901 + container: 30901 # Praxis admin/metrics + - host: 13000 + container: 30300 # Grafana + - host: 19090 + container: 30909 # Prometheus + - host: 15000 + container: 30500 # MLflow + stacks: + - prometheus + - tempo + - loki + - otel-collector + - mlflow + - mock-backends + - praxis-images + - praxis-deploy + - dashboards + - datasources + properties: + # Pin chart versions for reproducibility + kubePrometheusVersion: "72.6.3" + tempoVersion: "1.24.4" + lokiStackVersion: "2.10.2" + mlflowVersion: "1.11.3" + + stacks: + prometheus: + description: kube-prometheus-stack (Prometheus + Grafana) + steps: + - type: helm + release: monitoring + chart: prometheus-community/kube-prometheus-stack + version: "{{ cluster.properties.kubePrometheusVersion }}" + namespace: monitoring + values: + prometheus: + prometheusSpec: + serviceMonitorSelectorNilUsesHelmValues: false + additionalScrapeConfigs: + - job_name: praxis-proxy + metrics_path: /metrics + scrape_interval: 15s + static_configs: + - targets: ["praxis-proxy.default.svc:9901"] + service: + type: NodePort + nodePort: 30909 + grafana: + adminPassword: admin + image: + tag: 11.6.0 + service: + type: NodePort + nodePort: 30300 + - type: wait + resource: deployment/monitoring-grafana + namespace: monitoring + condition: available + timeout: "300s" + + tempo: + description: Tempo trace storage + steps: + - type: helm + release: tempo + chart: grafana/tempo + version: "{{ cluster.properties.tempoVersion }}" + namespace: monitoring + + loki: + description: Loki + Promtail log aggregation + steps: + - type: helm + release: loki + chart: grafana/loki-stack + version: "{{ cluster.properties.lokiStackVersion }}" + namespace: monitoring + values: + loki: + persistence: + enabled: false + promtail: + enabled: true + + otel-collector: + description: OpenTelemetry Collector (OTLP → Tempo) + steps: + - type: exec + command: [bash, -c, "kubectl create namespace otel --dry-run=client -o yaml | kubectl apply -f -"] + - type: manifest + path: stacks/observability/manifests/otel-collector.yaml + - type: wait + resource: deployment/otel-collector + namespace: otel + condition: available + timeout: "60s" + + mlflow: + description: MLflow experiment tracking + steps: + - type: helm + release: mlflow + chart: community-charts/mlflow + version: "{{ cluster.properties.mlflowVersion }}" + namespace: mlflow + values: + backendStore: + databaseMigration: true + service: + type: NodePort + - type: exec + command: [bash, -c, "kubectl -n mlflow patch svc mlflow --type=json -p '[{\"op\":\"replace\",\"path\":\"/spec/ports/0/nodePort\",\"value\":30500}]'"] + + mock-backends: + description: Fortio echo + llm-d inference-sim + steps: + - type: manifest + path: stacks/mock-backends/manifests/echo-backend.yaml + - type: manifest + path: stacks/mock-backends/manifests/inference-sim.yaml + + praxis-images: + description: Build and load Praxis images into KIND + steps: + - type: exec + command: [bash, -c, "cd \"${PRAXIS_DIR:?Set PRAXIS_DIR to your praxis checkout}\" && docker build -t praxis:dev -f Containerfile ."] + - type: exec + command: [bash, -c, "cd \"${PRAXIS_DIR:?}\" && sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' Containerfile | docker build -t praxis:dev-otel -f - ."] + - type: exec + command: [bash, -c, "kind load docker-image praxis:dev --name otel-bench-local"] + - type: exec + command: [bash, -c, "kind load docker-image praxis:dev-otel --name otel-bench-local"] + + praxis-deploy: + description: Deploy Praxis proxy with OTel config + steps: + - type: exec + command: [bash, -c, "kubectl create configmap praxis-config --from-file=config.yaml=examples/otel-benchmark/configs/otel-full.yaml -n default --dry-run=client -o yaml | kubectl apply -f -"] + - type: manifest + path: otel-benchmark/manifests/praxis.yaml + - type: manifest + path: otel-benchmark/manifests/servicemonitor.yaml + - type: wait + resource: deployment/praxis-proxy + namespace: default + condition: available + timeout: "120s" + + dashboards: + description: Load Grafana dashboards via ConfigMap + steps: + - type: exec + command: [bash, -c, "kubectl -n monitoring create configmap praxis-dashboards --from-file=examples/stacks/observability/dashboards/ --dry-run=client -o yaml | kubectl apply -f -"] + - type: exec + command: [kubectl, -n, monitoring, label, configmap, praxis-dashboards, grafana_dashboard=1, --overwrite] + + datasources: + description: Add Tempo + Loki datasources to Grafana + steps: + - type: exec + command: [bash, -c, "sleep 5 && curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Tempo\",\"type\":\"tempo\",\"access\":\"proxy\",\"url\":\"http://tempo.monitoring.svc:3200\",\"uid\":\"tempo\"}' || true"] + - type: exec + command: [bash, -c, "curl -sf -u admin:admin -X POST http://localhost:13000/api/datasources -H 'Content-Type: application/json' -d '{\"name\":\"Loki\",\"type\":\"loki\",\"access\":\"proxy\",\"url\":\"http://loki.monitoring.svc:3100\",\"uid\":\"loki\"}' || true"] diff --git a/examples/otel-benchmark/README.md b/examples/otel-benchmark/README.md new file mode 100644 index 0000000..f835f27 --- /dev/null +++ b/examples/otel-benchmark/README.md @@ -0,0 +1,209 @@ +# OTel Observability Benchmark + +Deploys a full observability stack on KIND for benchmarking Praxis proxy +OTel tracing overhead across two scenarios: core proxy and AI proxy. + +## Stack + +- **Prometheus + Grafana 11.x** — metrics + visualization +- **Tempo** — distributed trace storage +- **Loki + Promtail** — log aggregation +- **OTel Collector** — trace pipeline (OTLP -> Tempo) +- **MLflow** — experiment tracking UI +- **Fortio echo** — mock HTTP backend +- **llm-d inference-sim** — mock LLM backend +- **Praxis proxy** — the proxy under test (baseline + OTel variants) + +## Prerequisites + +- Docker or Podman +- [KIND](https://kind.sigs.k8s.io/) +- [Helm](https://helm.sh/) with repos: `prometheus-community`, `grafana`, `community-charts` +- [vegeta](https://github.com/tsenart/vegeta) (for benchmarks) +- Praxis source checkouts (for image builds) + +## Quick Start + +```bash +# 1. Add helm repos (one-time) +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add grafana https://grafana.github.io/helm-charts +helm repo add community-charts https://community-charts.github.io/helm-charts +helm repo update + +# 2. Set paths to source checkouts +export PRAXIS_DIR=/path/to/praxis # praxis core repo (with OTel PRs) +export AI_DIR=/path/to/ai # praxis AI repo + +# 3. Build images +# Core: baseline + OTel +cd "$PRAXIS_DIR" +docker build -t praxis:dev -f Containerfile . +sed 's|cargo build --release -p praxis-proxy|cargo build --release -p praxis-proxy --features otel|g' \ + Containerfile | docker build -t praxis:dev-otel -f - . + +# AI: baseline (from upstream/main, no patches) +cd "$AI_DIR" +docker build -t praxis-ai:dev -f Containerfile . + +# AI: OTel (from otel-fixes branch with praxis core patches) +# See "Building praxis-ai:dev-otel" section below. + +# 4. Deploy the full stack +cd /path/to/forge +praxis-forge doctor # check tools +praxis-forge plan --config examples/otel-benchmark.yaml # preview +praxis-forge up --config examples/otel-benchmark.yaml # create cluster + +# Load pre-built images into KIND +kind load docker-image praxis:dev praxis:dev-otel --name otel-bench-local +kind load docker-image praxis-ai:dev praxis-ai:dev-otel --name otel-bench-local + +# Apply stacks (skip praxis-images if images are already loaded) +for stack in prometheus tempo loki otel-collector mlflow mock-backends praxis-deploy dashboards datasources; do + praxis-forge apply --config examples/otel-benchmark.yaml local "$stack" +done +praxis-forge status --config examples/otel-benchmark.yaml # check status + +# 5. Verify +curl http://localhost:18080/ +open http://localhost:13000 # Grafana (admin/admin) +open http://localhost:19090 # Prometheus +open http://localhost:15000 # MLflow +``` + +## Scenario 1: Core Praxis OTel Benchmark + +Measures OTel tracing overhead on the core proxy with echo backend. +Generates 10 spans per request (root + 8 per-filter + upstream_exchange). + +```bash +bash examples/otel-benchmark/scripts/benchmark.sh +``` + +Runs 3 configurations at 2000 RPS for 30s each: +- **A: Baseline** — `praxis:dev` (no OTel feature) +- **B: OTel noop** — `praxis:dev-otel` (spans created, not exported) +- **C: OTel full** — `praxis:dev-otel` (spans exported to collector -> Tempo) + +Generate the report: +```bash +bash examples/otel-benchmark/scripts/report.sh +``` + +## Scenario 2: AI Praxis with Inference Sim + +Measures OTel overhead on the AI proxy with mock LLM backend. +Generates 11 spans per request (core spans + response_body phase). +Sends `POST /v1/chat/completions` to inference-sim. + +```bash +bash examples/otel-benchmark/scripts/benchmark-ai.sh +``` + +Runs 3 configurations at 500 RPS for 30s each: +- **A: AI Baseline** — `praxis-ai:dev` (no OTel feature) +- **B: AI OTel noop** — `praxis-ai:dev-otel` (spans created, not exported) +- **C: AI OTel full** — `praxis-ai:dev-otel` (spans exported to collector -> Tempo) + +Generate the report: +```bash +bash examples/otel-benchmark/scripts/report-ai.sh +``` + +### Span tree (AI request) + +``` +POST /v1/chat/completions -> inference-sim (root) + |-- filter:request_id:request + |-- filter:access_log:request + |-- filter:router:request -> routes /v1/* to inference cluster + |-- filter:load_balancer:request + |-- filter:load_balancer:response + |-- filter:router:response + |-- filter:access_log:response + |-- filter:request_id:response + |-- filter:access_log:response_body + +-- upstream_exchange [inference-sim:8000] +``` + +## Building praxis-ai:dev-otel + +The AI OTel image requires patched praxis core (for OTel spans) and the +AI OTel fixes. Build with both repos side by side: + +```bash +BUILD_DIR=$(mktemp -d) +rsync -a --exclude='.git' --exclude='target' "$AI_DIR/" "$BUILD_DIR/ai/" +for crate in core filter protocol tls server; do + rsync -a --exclude='target' "$PRAXIS_DIR/$crate/" "$BUILD_DIR/praxis/$crate/" +done +cp "$PRAXIS_DIR/Cargo.toml" "$PRAXIS_DIR/Cargo.lock" "$BUILD_DIR/praxis/" + +# Add patch.crates-io to ai/Cargo.toml pointing to ../praxis/* +cat >> "$BUILD_DIR/ai/Cargo.toml" << 'PATCH' + +[patch.crates-io] +praxis-proxy-core = { path = "../praxis/core" } +praxis-proxy-filter = { path = "../praxis/filter" } +praxis-proxy-protocol = { path = "../praxis/protocol" } +praxis-proxy-tls = { path = "../praxis/tls" } +praxis-proxy = { path = "../praxis/server" } +PATCH + +# Build with OTel features enabled +# (requires custom Containerfile that copies both repos) +docker build -t praxis-ai:dev-otel -f Containerfile "$BUILD_DIR" +``` + +## Dashboards + +| Dashboard | URL | What it shows | +|-----------|-----|------| +| Praxis Proxy Overview | http://localhost:13000/d/praxis-proxy-overview | Request rate, latency P50/P99, requests by method | +| Praxis OTel Traces | http://localhost:13000/d/praxis-traces | Searchable trace table with clickable Trace IDs | +| Praxis Benchmark Results | http://localhost:13000/d/praxis-benchmark | CPU/memory for praxis + collector, RPS, latency | +| Praxis AI/LLM Golden Signals | http://localhost:13000/d/praxis-ai-golden-signals | P95 latency stat, throughput, AI token metrics (future) | +| Praxis Structured Logs | http://localhost:13000/d/praxis-logs | Log volume, error logs, all praxis access logs | + +### Explore views + +| View | URL | +|------|-----| +| Tempo trace search | http://localhost:13000/explore (select Tempo datasource) | +| Prometheus metrics | http://localhost:13000/explore (select Prometheus datasource) | +| Loki log search | http://localhost:13000/explore (select Loki datasource) | + +### Other UIs + +| Service | URL | +|---------|-----| +| Prometheus | http://localhost:19090 | +| MLflow | http://localhost:15000 | +| Praxis proxy | http://localhost:18080 | +| Praxis admin/metrics | http://localhost:18901/metrics | + +## Host Ports + +| Port | Service | KIND NodePort | +|------|---------|------| +| 18080 | Praxis proxy | 30080 | +| 18901 | Praxis admin | 30901 | +| 13000 | Grafana | 30300 | +| 19090 | Prometheus | 30909 | +| 15000 | MLflow | 30500 | + +## Known Issues + +- **Grafana version**: Must use 11.x (pinned via `grafana.image.tag`). Grafana 12.0 has rendering bugs with provisioned dashboards using `row`/`gauge` panel types. +- **Datasources**: Prometheus and Tempo datasources are added via the `datasources` stack. If Grafana restarts, they need re-adding. +- **Tokio runtime fix**: The praxis `otel` feature requires a persistent Tokio runtime in `core/src/logging.rs` for the `BatchSpanProcessor` to drive tonic's async gRPC export. +- **MLflow trace ingestion**: The OTel collector exports traces to Tempo only. MLflow is deployed for experiment tracking UI but does not receive trace data from the collector pipeline. + +## Teardown + +```bash +praxis-forge down --config examples/otel-benchmark.yaml +# or +kind delete cluster --name otel-bench-local +``` diff --git a/examples/otel-benchmark/configs/ai-baseline.yaml b/examples/otel-benchmark/configs/ai-baseline.yaml new file mode 100644 index 0000000..f8d21ac --- /dev/null +++ b/examples/otel-benchmark/configs/ai-baseline.yaml @@ -0,0 +1,35 @@ +# AI baseline benchmark config (no OTel). +# Routes /v1/* to inference-sim, everything else to Fortio echo. + +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + - filter: access_log + + - filter: router + routes: + - path_prefix: "/v1/" + cluster: inference + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" + - name: inference + endpoints: + - "inference-sim.default.svc:8000" diff --git a/examples/otel-benchmark/configs/ai-otel-full.yaml b/examples/otel-benchmark/configs/ai-otel-full.yaml new file mode 100644 index 0000000..3f26e9c --- /dev/null +++ b/examples/otel-benchmark/configs/ai-otel-full.yaml @@ -0,0 +1,35 @@ +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +telemetry: + otlp_endpoint: "http://otel-collector.otel.svc:4317" + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + - filter: access_log + + - filter: router + routes: + - path_prefix: "/v1/" + cluster: inference + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" + - name: inference + endpoints: + - "inference-sim.default.svc:8000" diff --git a/examples/otel-benchmark/configs/ai-otel-noop.yaml b/examples/otel-benchmark/configs/ai-otel-noop.yaml new file mode 100644 index 0000000..ae73660 --- /dev/null +++ b/examples/otel-benchmark/configs/ai-otel-noop.yaml @@ -0,0 +1,37 @@ +# AI OTel noop benchmark config. +# OTel feature is compiled in but no OTLP endpoint is configured, +# so spans are created but never exported. +# Routes /v1/* to inference-sim, everything else to Fortio echo. + +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + - filter: access_log + + - filter: router + routes: + - path_prefix: "/v1/" + cluster: inference + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" + - name: inference + endpoints: + - "inference-sim.default.svc:8000" diff --git a/examples/otel-benchmark/configs/baseline.yaml b/examples/otel-benchmark/configs/baseline.yaml new file mode 100644 index 0000000..856b1d9 --- /dev/null +++ b/examples/otel-benchmark/configs/baseline.yaml @@ -0,0 +1,31 @@ +# Baseline benchmark config (no OTel). +# Routes all traffic to the Fortio echo backend. + +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + + - filter: access_log + + - filter: router + routes: + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" diff --git a/examples/otel-benchmark/configs/otel-full.yaml b/examples/otel-benchmark/configs/otel-full.yaml new file mode 100644 index 0000000..19b179d --- /dev/null +++ b/examples/otel-benchmark/configs/otel-full.yaml @@ -0,0 +1,35 @@ +# OTel full export benchmark config. +# Spans are created and exported via OTLP/gRPC to the collector. +# The OTEL_EXPORTER_OTLP_ENDPOINT env var overrides this at runtime. + +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +telemetry: + otlp_endpoint: "http://otel-collector.otel.svc:4317" + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + + - filter: access_log + + - filter: router + routes: + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" diff --git a/examples/otel-benchmark/configs/otel-noop.yaml b/examples/otel-benchmark/configs/otel-noop.yaml new file mode 100644 index 0000000..8a34b70 --- /dev/null +++ b/examples/otel-benchmark/configs/otel-noop.yaml @@ -0,0 +1,32 @@ +# OTel noop benchmark config. +# OTel feature is compiled in but no OTLP endpoint is configured, +# so spans are created but never exported. + +admin: + address: "0.0.0.0:9901" + +insecure_options: + allow_public_admin: true + +listeners: + - name: default + address: "0.0.0.0:8080" + filter_chains: [main] + +filter_chains: + - name: main + filters: + - filter: request_id + + - filter: access_log + + - filter: router + routes: + - path_prefix: "/" + cluster: echo + + - filter: load_balancer + clusters: + - name: echo + endpoints: + - "echo-backend.default.svc:8080" diff --git a/examples/otel-benchmark/manifests/praxis.yaml b/examples/otel-benchmark/manifests/praxis.yaml new file mode 100644 index 0000000..022dc0a --- /dev/null +++ b/examples/otel-benchmark/manifests/praxis.yaml @@ -0,0 +1,73 @@ +# Praxis Deployment for benchmark mode. +# ConfigMap is created by the praxis-deploy stack (otel-benchmark.yaml) +# or by benchmark.sh/benchmark-ai.sh when swapping configs between runs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: praxis-proxy + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: praxis-proxy + template: + metadata: + labels: + app: praxis-proxy + spec: + containers: + - name: praxis-proxy + image: praxis:dev-otel + ports: + - containerPort: 8080 + name: proxy + - containerPort: 9901 + name: admin + volumeMounts: + - name: config + mountPath: /etc/praxis + resources: + requests: + cpu: 500m + memory: 128Mi + limits: + cpu: "2" + memory: 512Mi + readinessProbe: + httpGet: + path: /healthy + port: admin + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthy + port: admin + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: praxis-config +--- +apiVersion: v1 +kind: Service +metadata: + name: praxis-proxy + namespace: default + labels: + app: praxis-proxy +spec: + type: NodePort + selector: + app: praxis-proxy + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30080 + name: proxy + - port: 9901 + targetPort: 9901 + nodePort: 30901 + name: admin diff --git a/examples/otel-benchmark/manifests/servicemonitor.yaml b/examples/otel-benchmark/manifests/servicemonitor.yaml new file mode 100644 index 0000000..312a057 --- /dev/null +++ b/examples/otel-benchmark/manifests/servicemonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: praxis-proxy + namespace: default + labels: + app: praxis-proxy +spec: + selector: + matchLabels: + app: praxis-proxy + endpoints: + - port: admin + interval: 15s + path: /metrics diff --git a/examples/otel-benchmark/scripts/ai-payload.json b/examples/otel-benchmark/scripts/ai-payload.json new file mode 100644 index 0000000..9f03b09 --- /dev/null +++ b/examples/otel-benchmark/scripts/ai-payload.json @@ -0,0 +1 @@ +{"model":"test-model","messages":[{"role":"user","content":"hello"}]} diff --git a/examples/otel-benchmark/scripts/benchmark-ai.sh b/examples/otel-benchmark/scripts/benchmark-ai.sh new file mode 100755 index 0000000..51e62ce --- /dev/null +++ b/examples/otel-benchmark/scripts/benchmark-ai.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +KIND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +CLUSTER_NAME="${CLUSTER_NAME:-otel-bench-local}" +CTX="kind-${CLUSTER_NAME}" +GATEWAY_URL="http://localhost:18080" + +RATE="${RATE:-500}" +DURATION="${DURATION:-30s}" +RUNS="${RUNS:-3}" + +RESULTS_DIR="${KIND_DIR}/results/ai-$(date +%Y%m%d-%H%M%S)" +mkdir -p "${RESULTS_DIR}" + +echo "=== Praxis AI OTel Benchmark ===" +echo "Rate: ${RATE} RPS | Duration: ${DURATION} | Runs: ${RUNS}" +echo "Results: ${RESULTS_DIR}" +echo "" + +run_vegeta() { + local label="$1" + local run="$2" + echo "--- ${label} run ${run}/${RUNS} ---" + printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \ + "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \ + vegeta attack -rate="${RATE}" -duration="${DURATION}" -connections=100 | \ + tee "${RESULTS_DIR}/${label}-run${run}.bin" | \ + vegeta report -type=json > "${RESULTS_DIR}/${label}-run${run}.json" + vegeta report < "${RESULTS_DIR}/${label}-run${run}.bin" + kubectl --context "${CTX}" top pod -n default --no-headers 2>/dev/null \ + >> "${RESULTS_DIR}/${label}-resources.txt" || true + echo "" +} + +# ---- Run A: AI Baseline (no OTel) ---- +echo "==========================================" +echo " Run A: AI Baseline (praxis-ai:dev, no OTel)" +echo "==========================================" + +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/ai-baseline.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +# praxis-ai:dev has ENTRYPOINT [praxis-ai] — needs -c arg +kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \ + -p '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"praxis-ai:dev"},{"op":"replace","path":"/spec/template/spec/containers/0/args","value":["-c","/etc/praxis/config.yaml"]}]' +kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \ + "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \ + vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "ai-baseline" "${i}" + sleep 5 +done + +# ---- Run B: AI OTel noop (spans created, not exported) ---- +echo "==========================================" +echo " Run B: AI OTel noop (praxis-ai:dev-otel, no endpoint)" +echo "==========================================" + +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/ai-otel-noop.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +# praxis-ai:dev-otel has ENTRYPOINT [praxis -c /etc/praxis/config.yaml] — clear args +kubectl --context "${CTX}" patch deployment praxis-proxy -n default --type=json \ + -p '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"praxis-ai:dev-otel"},{"op":"replace","path":"/spec/template/spec/containers/0/args","value":[]}]' +kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \ + "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \ + vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "ai-otel-noop" "${i}" + sleep 5 +done + +# ---- Run C: AI OTel full (spans exported to collector) ---- +echo "==========================================" +echo " Run C: AI OTel full (praxis-ai:dev-otel, exporting)" +echo "==========================================" + +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/ai-otel-full.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +kubectl --context "${CTX}" set env deployment/praxis-proxy \ + OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.otel.svc:4317 -n default +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +printf 'POST %s/v1/chat/completions\nContent-Type: application/json\n@%s\n' \ + "${GATEWAY_URL}" "${SCRIPT_DIR}/ai-payload.json" | \ + vegeta attack -rate=100 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "ai-otel-full" "${i}" + sleep 5 +done + +echo "==========================================" +echo " AI Benchmark complete" +echo "==========================================" +echo "Results in: ${RESULTS_DIR}" +echo "" +echo "Generate report:" +echo " bash ${SCRIPT_DIR}/report-ai.sh ${RESULTS_DIR}" diff --git a/examples/otel-benchmark/scripts/benchmark.sh b/examples/otel-benchmark/scripts/benchmark.sh new file mode 100755 index 0000000..4901ae6 --- /dev/null +++ b/examples/otel-benchmark/scripts/benchmark.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +KIND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +CLUSTER_NAME="${CLUSTER_NAME:-otel-bench-local}" +CTX="kind-${CLUSTER_NAME}" +GATEWAY_URL="http://localhost:18080" + +RATE="${RATE:-2000}" +DURATION="${DURATION:-30s}" +RUNS="${RUNS:-3}" + +RESULTS_DIR="${KIND_DIR}/results/$(date +%Y%m%d-%H%M%S)" +mkdir -p "${RESULTS_DIR}" + +echo "=== Praxis OTel Benchmark ===" +echo "Rate: ${RATE} RPS | Duration: ${DURATION} | Runs: ${RUNS}" +echo "Results: ${RESULTS_DIR}" +echo "" + +run_vegeta() { + local label="$1" + local run="$2" + echo "--- ${label} run ${run}/${RUNS} ---" + echo "GET ${GATEWAY_URL}/" | \ + vegeta attack -rate="${RATE}" -duration="${DURATION}" -connections=200 | \ + tee "${RESULTS_DIR}/${label}-run${run}.bin" | \ + vegeta report -type=json > "${RESULTS_DIR}/${label}-run${run}.json" + vegeta report < "${RESULTS_DIR}/${label}-run${run}.bin" + # Capture resource snapshot + kubectl --context "${CTX}" top pod -n default --no-headers 2>/dev/null \ + >> "${RESULTS_DIR}/${label}-resources.txt" || true + echo "" +} + +# ---- Run A: Baseline (no OTel) ---- +echo "==========================================" +echo " Run A: Baseline (praxis:dev, no OTel)" +echo "==========================================" + +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/baseline.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev -n default +kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "baseline" "${i}" + sleep 5 +done + +# ---- Run B: OTel noop (spans created, not exported) ---- +echo "==========================================" +echo " Run B: OTel noop (praxis:dev-otel, no endpoint)" +echo "==========================================" + +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/otel-noop.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel -n default +kubectl --context "${CTX}" set env deployment/praxis-proxy OTEL_EXPORTER_OTLP_ENDPOINT- -n default 2>/dev/null || true +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "otel-noop" "${i}" + sleep 5 +done + +# ---- Run C: OTel full (spans exported to collector) ---- +echo "==========================================" +echo " Run C: OTel full (praxis:dev-otel, exporting)" +echo "==========================================" + +# Set OTEL env var BEFORE restarting the pod +kubectl --context "${CTX}" create configmap praxis-config \ + --from-file=config.yaml="${KIND_DIR}/configs/otel-full.yaml" \ + -n default --dry-run=client -o yaml | kubectl --context "${CTX}" apply -f - +kubectl --context "${CTX}" set image deployment/praxis-proxy praxis-proxy=praxis:dev-otel -n default +kubectl --context "${CTX}" set env deployment/praxis-proxy \ + OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.otel.svc:4317 -n default +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=0 -n default +sleep 3 +kubectl --context "${CTX}" scale deployment/praxis-proxy --replicas=1 -n default +kubectl --context "${CTX}" -n default wait --for=condition=Available deployment/praxis-proxy --timeout 60s +sleep 5 + +echo "Warmup..." +echo "GET ${GATEWAY_URL}/" | vegeta attack -rate=500 -duration=10s > /dev/null 2>&1 || true +sleep 2 + +for i in $(seq 1 "${RUNS}"); do + run_vegeta "otel-full" "${i}" + sleep 5 +done + +echo "==========================================" +echo " Benchmark complete" +echo "==========================================" +echo "Results in: ${RESULTS_DIR}" +echo "" +echo "Generate report:" +echo " bash ${SCRIPT_DIR}/report.sh ${RESULTS_DIR}" diff --git a/examples/otel-benchmark/scripts/report-ai.sh b/examples/otel-benchmark/scripts/report-ai.sh new file mode 100755 index 0000000..555517c --- /dev/null +++ b/examples/otel-benchmark/scripts/report-ai.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +RESULTS_DIR="${1:?Usage: report-ai.sh }" + +if [ ! -d "${RESULTS_DIR}" ]; then + echo "Error: ${RESULTS_DIR} does not exist" + exit 1 +fi + +export BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +export COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") +export RESULTS_DIR + +python3 << 'PYTHON_SCRIPT' +import json +import glob +import os +from datetime import datetime + +results_dir = os.environ['RESULTS_DIR'] +branch = os.environ.get('BRANCH', 'unknown') +commit = os.environ.get('COMMIT', 'unknown') + +configs = ['ai-baseline', 'ai-otel-noop', 'ai-otel-full'] +stats = {} + +for config in configs: + total_p50 = 0 + total_p99 = 0 + total_rps = 0 + count = 0 + + pattern = os.path.join(results_dir, f'{config}-run*.json') + for path in glob.glob(pattern): + try: + with open(path) as f: + data = json.load(f) + total_p50 += data['latencies']['50th'] + total_p99 += data['latencies']['99th'] + total_rps += data['throughput'] + count += 1 + except (json.JSONDecodeError, KeyError) as e: + print(f"Warning: Failed to parse {path}: {e}") + continue + + if count > 0: + stats[config] = { + 'p50': int(total_p50 / count / 1000), + 'p99': int(total_p99 / count / 1000), + 'rps': int(total_rps / count), + 'runs': count + } + else: + stats[config] = {'p50': 0, 'p99': 0, 'rps': 0, 'runs': 0} + +report_path = os.path.join(results_dir, 'report.md') +with open(report_path, 'w') as f: + f.write(f"# Praxis AI OTel Overhead Benchmark\n\n") + f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d')} | **Commit:** {commit} | **Branch:** {branch}\n") + f.write(f"**Platform:** KIND (1 node) | **Backend:** inference-sim (mock LLM)\n\n") + f.write(f"## Summary\n\n") + f.write(f"| Config | P50 (us) | P99 (us) | RPS | Runs | P50 delta | P99 delta |\n") + f.write(f"|--------|----------|----------|-----|------|-----------|-----------|\n") + + base_p50 = stats['ai-baseline']['p50'] + base_p99 = stats['ai-baseline']['p99'] + + for config in configs: + s = stats[config] + p50 = s['p50'] + p99 = s['p99'] + rps = s['rps'] + runs = s['runs'] + + if config == 'ai-baseline': + f.write(f"| AI Baseline | {p50} | {p99} | {rps} | {runs} | -- | -- |\n") + else: + label = 'AI OTel (noop)' if config == 'ai-otel-noop' else 'AI OTel (full)' + + if base_p50 > 0: + dp50 = f"{(p50 - base_p50) / base_p50 * 100:+.1f}%" + dp99 = f"{(p99 - base_p99) / base_p99 * 100:+.1f}%" + else: + dp50 = "N/A" + dp99 = "N/A" + + f.write(f"| {label} | {p50} | {p99} | {rps} | {runs} | {dp50} | {dp99} |\n") + +print(f"Report written to: {report_path}") +PYTHON_SCRIPT + +cat "${RESULTS_DIR}/report.md" diff --git a/examples/otel-benchmark/scripts/report.sh b/examples/otel-benchmark/scripts/report.sh new file mode 100755 index 0000000..dd2213d --- /dev/null +++ b/examples/otel-benchmark/scripts/report.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +RESULTS_DIR="${1:?Usage: report.sh }" + +if [ ! -d "${RESULTS_DIR}" ]; then + echo "Error: ${RESULTS_DIR} does not exist" + exit 1 +fi + +BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +REPORT="${RESULTS_DIR}/report.md" + +# Use Python to generate the report (macOS has bash 3.2, no associative arrays) +python3 << 'PYTHON_SCRIPT' +import json +import glob +import sys +import os +from datetime import datetime + +results_dir = os.environ['RESULTS_DIR'] +branch = os.environ['BRANCH'] +commit = os.environ['COMMIT'] + +configs = ['baseline', 'otel-noop', 'otel-full'] +stats = {} + +for config in configs: + total_p50 = 0 + total_p99 = 0 + total_rps = 0 + count = 0 + + pattern = os.path.join(results_dir, f'{config}-run*.json') + for path in glob.glob(pattern): + try: + with open(path) as f: + data = json.load(f) + total_p50 += data['latencies']['50th'] + total_p99 += data['latencies']['99th'] + total_rps += data['throughput'] + count += 1 + except (json.JSONDecodeError, KeyError) as e: + print(f"Warning: Failed to parse {path}: {e}", file=sys.stderr) + continue + + if count > 0: + stats[config] = { + 'p50': int(total_p50 / count / 1000), # Convert to microseconds + 'p99': int(total_p99 / count / 1000), + 'rps': int(total_rps / count), + 'runs': count + } + else: + stats[config] = {'p50': 0, 'p99': 0, 'rps': 0, 'runs': 0} + +# Generate report +report_path = os.path.join(results_dir, 'report.md') +with open(report_path, 'w') as f: + f.write(f"# Praxis OTel Overhead Benchmark\n\n") + f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d')} | **Commit:** {commit} | **Branch:** {branch}\n") + f.write(f"**Platform:** KIND (1 node) | **Backend:** Fortio echo\n\n") + f.write(f"## Summary\n\n") + f.write(f"| Config | P50 (us) | P99 (us) | RPS | Runs | P50 delta | P99 delta |\n") + f.write(f"|--------|----------|----------|-----|------|-----------|-----------||\n") + + base_p50 = stats['baseline']['p50'] + base_p99 = stats['baseline']['p99'] + + for config in configs: + s = stats[config] + p50 = s['p50'] + p99 = s['p99'] + rps = s['rps'] + runs = s['runs'] + + if config == 'baseline': + f.write(f"| Baseline | {p50} | {p99} | {rps} | {runs} | -- | -- |\n") + else: + label = 'OTel (noop)' if config == 'otel-noop' else 'OTel (full)' + + if base_p50 > 0: + dp50 = f"{(p50 - base_p50) / base_p50 * 100:+.1f}%" + dp99 = f"{(p99 - base_p99) / base_p99 * 100:+.1f}%" + else: + dp50 = "N/A" + dp99 = "N/A" + + f.write(f"| {label} | {p50} | {p99} | {rps} | {runs} | {dp50} | {dp99} |\n") + + f.write(f"\n## Previous Results (reference)\n\n") + f.write(f"| Config | P50 (us) | P99 (us) | RPS | P50 delta | P99 delta |\n") + f.write(f"|--------|----------|----------|-----|-----------|-----------||\n") + f.write(f"| Baseline | 388 | 568 | 2,000 | -- | -- |\n") + f.write(f"| OTel noop | 386 | 558 | 2,000 | -0.5% | -1.8% |\n") + f.write(f"| OTel full | 392 | 592 | 2,000 | +1.0% | +4.2% |\n\n") + f.write(f"Note: Previous results were from the old branch without root spans.\n") + +print(f"Report written to: {report_path}") +PYTHON_SCRIPT + +echo "" +cat "${REPORT}" diff --git a/examples/stacks/mock-backends/manifests/echo-backend.yaml b/examples/stacks/mock-backends/manifests/echo-backend.yaml new file mode 100644 index 0000000..5eeae10 --- /dev/null +++ b/examples/stacks/mock-backends/manifests/echo-backend.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: echo-backend + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: echo-backend + template: + metadata: + labels: + app: echo-backend + spec: + containers: + - name: fortio + image: fortio/fortio:1.75.2 + args: ["server", "-echo-server-default-params", "delay=0"] + ports: + - containerPort: 8080 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: echo-backend + namespace: default +spec: + selector: + app: echo-backend + ports: + - port: 8080 + targetPort: 8080 diff --git a/examples/stacks/mock-backends/manifests/inference-sim.yaml b/examples/stacks/mock-backends/manifests/inference-sim.yaml new file mode 100644 index 0000000..16badc7 --- /dev/null +++ b/examples/stacks/mock-backends/manifests/inference-sim.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inference-sim + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: inference-sim + template: + metadata: + labels: + app: inference-sim + spec: + containers: + - name: sim + image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 + args: ["--port", "8000", "--model", "test-model", "--mode", "random"] + ports: + - containerPort: 8000 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: inference-sim + namespace: default +spec: + selector: + app: inference-sim + ports: + - port: 8000 + targetPort: 8000 diff --git a/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json b/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json new file mode 100644 index 0000000..295b635 --- /dev/null +++ b/examples/stacks/observability/dashboards/praxis-ai-golden-signals.json @@ -0,0 +1,173 @@ +{ + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "expr": "avg_over_time(praxis_http_request_duration_seconds{quantile=\"0.95\"}[5m])", + "legendFormat": "P95" + } + ], + "title": "P95 Request Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 0 + }, + "id": 2, + "targets": [ + { + "expr": "sum(rate(praxis_http_requests_total[5m]))", + "legendFormat": "Requests/sec" + } + ], + "title": "Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No AI metrics yet" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 3, + "targets": [ + { + "expr": "sum(rate(praxis_ai_prompt_tokens_total[5m])) + sum(rate(praxis_ai_generation_tokens_total[5m]))", + "legendFormat": "Tokens/sec" + } + ], + "title": "Tokens/sec (requires ai#92)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "noValue": "No AI metrics yet", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "expr": "sum(increase(praxis_ai_prompt_tokens_total[1h])) * 0.003/1000 + sum(increase(praxis_ai_generation_tokens_total[1h])) * 0.015/1000", + "legendFormat": "Hourly Cost" + } + ], + "title": "Estimated Hourly Cost (requires ai#92)", + "type": "stat" + } + ], + "schemaVersion": 39, + "tags": [ + "praxis", + "ai" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Praxis AI/LLM Golden Signals", + "uid": "praxis-ai-golden-signals" +} diff --git a/examples/stacks/observability/dashboards/praxis-benchmark.json b/examples/stacks/observability/dashboards/praxis-benchmark.json new file mode 100644 index 0000000..a4f6cf1 --- /dev/null +++ b/examples/stacks/observability/dashboards/praxis-benchmark.json @@ -0,0 +1,59 @@ +{ + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "targets": [{ "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"praxis-proxy.*\"}[1m]))", "legendFormat": "{{pod}}" }], + "title": "Praxis CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "targets": [{ "expr": "container_memory_working_set_bytes{pod=~\"praxis-proxy.*\", container=\"praxis-proxy\"}", "legendFormat": "{{pod}}" }], + "title": "Praxis Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 3, + "targets": [{ "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"otel-collector.*\"}[1m]))", "legendFormat": "{{pod}}" }], + "title": "OTel Collector CPU", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 4, + "targets": [{ "expr": "sum(rate(praxis_http_requests_total[1m]))", "legendFormat": "RPS" }], + "title": "Request Rate (1m)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 5, + "targets": [ + { "expr": "praxis_http_request_duration_seconds{quantile=\"0.5\"}", "legendFormat": "P50" }, + { "expr": "praxis_http_request_duration_seconds{quantile=\"0.99\"}", "legendFormat": "P99" } + ], + "title": "Latency During Benchmark", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["praxis", "benchmark"], + "time": { "from": "now-1h", "to": "now" }, + "title": "Praxis Benchmark Results", + "uid": "praxis-benchmark" +} diff --git a/examples/stacks/observability/dashboards/praxis-logs.json b/examples/stacks/observability/dashboards/praxis-logs.json new file mode 100644 index 0000000..70db91f --- /dev/null +++ b/examples/stacks/observability/dashboards/praxis-logs.json @@ -0,0 +1,30 @@ +{ + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "targets": [{ "expr": "sum(rate({container=\"praxis-proxy\"} [5m]))", "legendFormat": "log rate" }], + "title": "Log Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 14, "w": 24, "x": 0, "y": 6 }, + "id": 2, + "options": { "cellHeight": "sm", "showHeader": true }, + "targets": [{ "expr": "{container=\"praxis-proxy\"}", "refId": "A" }], + "title": "All Praxis Logs", + "type": "table" + } + ], + "schemaVersion": 39, + "tags": ["praxis", "logs"], + "time": { "from": "now-1h", "to": "now" }, + "title": "Praxis Structured Logs", + "uid": "praxis-logs" +} diff --git a/examples/stacks/observability/dashboards/praxis-proxy-overview.json b/examples/stacks/observability/dashboards/praxis-proxy-overview.json new file mode 100644 index 0000000..35b3090 --- /dev/null +++ b/examples/stacks/observability/dashboards/praxis-proxy-overview.json @@ -0,0 +1,50 @@ +{ + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "targets": [{ "expr": "sum(rate(praxis_http_requests_total[5m]))", "legendFormat": "RPS" }], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "targets": [ + { "expr": "praxis_http_request_duration_seconds{quantile=\"0.5\"}", "legendFormat": "P50" }, + { "expr": "praxis_http_request_duration_seconds{quantile=\"0.99\"}", "legendFormat": "P99" } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 3, + "targets": [{ "expr": "sum(rate(praxis_http_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }], + "title": "Requests by Method", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 4, + "targets": [{ "expr": "rate(praxis_http_request_duration_seconds_sum[5m]) / rate(praxis_http_request_duration_seconds_count[5m])", "legendFormat": "Avg" }], + "title": "Average Latency", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["praxis"], + "time": { "from": "now-15m", "to": "now" }, + "title": "Praxis Proxy Overview", + "uid": "praxis-proxy-overview" +} diff --git a/examples/stacks/observability/dashboards/praxis-traces.json b/examples/stacks/observability/dashboards/praxis-traces.json new file mode 100644 index 0000000..1748ce4 --- /dev/null +++ b/examples/stacks/observability/dashboards/praxis-traces.json @@ -0,0 +1,97 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Trace Search", + "type": "row" + }, + { + "datasource": { "type": "tempo", "uid": "tempo" }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 1 }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Start time" }] + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "tempo", "uid": "tempo" }, + "queryType": "nativeSearch", + "serviceName": "praxis", + "limit": 20 + } + ], + "title": "Recent Traces", + "type": "table" + }, + { + "datasource": { "type": "tempo", "uid": "tempo" }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 13 }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "tempo", "uid": "tempo" }, + "queryType": "nativeSearch", + "serviceName": "praxis", + "search": "status=error", + "limit": 20 + } + ], + "title": "Error Traces", + "type": "table" + }, + { + "datasource": { "type": "tempo", "uid": "tempo" }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 13 }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "tempo", "uid": "tempo" }, + "queryType": "nativeSearch", + "serviceName": "praxis", + "minDuration": "100ms", + "limit": 20 + } + ], + "title": "Slow Traces (>100ms)", + "type": "table" + } + ], + "schemaVersion": 39, + "tags": ["praxis", "traces"], + "time": { "from": "now-1h", "to": "now" }, + "title": "Praxis OTel Traces", + "uid": "praxis-traces" +} diff --git a/examples/stacks/observability/manifests/otel-collector.yaml b/examples/stacks/observability/manifests/otel-collector.yaml new file mode 100644 index 0000000..452cb67 --- /dev/null +++ b/examples/stacks/observability/manifests/otel-collector.yaml @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + namespace: otel +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + processors: + batch: + timeout: 5s + send_batch_size: 8192 + + exporters: + debug: + verbosity: basic + otlp/tempo: + endpoint: tempo.monitoring.svc:4317 + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/tempo] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + namespace: otel +spec: + replicas: 1 + selector: + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.108.0 + args: ["--config=/etc/otelcol/config.yaml"] + ports: + - containerPort: 4317 + name: otlp-grpc + - containerPort: 4318 + name: otlp-http + volumeMounts: + - name: config + mountPath: /etc/otelcol + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumes: + - name: config + configMap: + name: otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: otel +spec: + selector: + app: otel-collector + ports: + - port: 4317 + targetPort: 4317 + name: otlp-grpc + - port: 4318 + targetPort: 4318 + name: otlp-http diff --git a/src/cluster/kind.rs b/src/cluster/kind.rs index 2ab8bb4..a449920 100644 --- a/src/cluster/kind.rs +++ b/src/cluster/kind.rs @@ -8,7 +8,7 @@ use std::collections::BTreeMap; use crate::{ command::runner::{CommandOutput, CommandRunner, CommandSpec}, - config::NodeConfig, + config::{NodeConfig, PortMapping}, error::ForgeError, }; @@ -30,6 +30,18 @@ pub fn kubectl_context(kind_name: &str) -> String { // Lifecycle // --------------------------------------------------------------- +/// Configuration for creating a KIND cluster. +pub struct CreateClusterConfig<'a> { + /// Node layout for the cluster. + pub nodes: &'a NodeConfig, + /// Port mappings to add to the first control-plane node. + pub ports: &'a [PortMapping], + /// Directory where config files are written. + pub config_dir: &'a std::path::Path, + /// Optional Docker network to join via `KIND_EXPERIMENTAL_DOCKER_NETWORK`. + pub docker_network: Option<&'a str>, +} + /// Check whether a KIND cluster with the given name exists. /// /// # Errors @@ -40,10 +52,7 @@ pub fn cluster_exists(runner: &dyn CommandRunner, kind_name: &str) -> Result Result, + config: &CreateClusterConfig<'_>, ) -> Result<(), ForgeError> { - let config_yaml = generate_kind_config(nodes); - let config_path = write_kind_config(config_dir, kind_name, &config_yaml)?; - let result = run_create(runner, kind_name, &config_path, docker_network); + let config_yaml = generate_kind_config(config.nodes, config.ports); + let config_path = write_kind_config(config.config_dir, kind_name, &config_yaml)?; + let result = run_create(runner, kind_name, &config_path, config.docker_network); cleanup_kind_config(&config_path); result } @@ -130,11 +137,31 @@ pub fn run_kubectl( // KIND config generation // --------------------------------------------------------------- -/// Generate a KIND cluster config YAML from a [`NodeConfig`]. -pub fn generate_kind_config(nodes: &NodeConfig) -> String { +/// Generate a Kind cluster config YAML string. +/// +/// When `ports` is non-empty, `extraPortMappings` entries are added +/// to the first control-plane node (Kind only supports port mappings +/// on control-plane nodes). +pub fn generate_kind_config(nodes: &NodeConfig, ports: &[PortMapping]) -> String { + use std::fmt::Write as _; let mut yaml = String::from("kind: Cluster\napiVersion: kind.x-k8s.io/v1alpha4\nnodes:\n"); - for _ in 0..nodes.control_planes { + for i in 0..nodes.control_planes { yaml.push_str(" - role: control-plane\n"); + if i == 0 && !ports.is_empty() { + yaml.push_str(" extraPortMappings:\n"); + for pm in ports { + let _written = write!( + &mut yaml, + " - hostPort: {}\n containerPort: {}\n protocol: {}\n", + pm.host, + pm.container, + pm.protocol.to_uppercase(), + ); + if let Some(addr) = &pm.bind_address { + let _written = writeln!(&mut yaml, " listenAddress: \"{addr}\""); + } + } + } } for _ in 0..nodes.workers { yaml.push_str(" - role: worker\n"); @@ -316,7 +343,7 @@ mod tests { #[test] fn generate_kind_config_default_nodes() { let nodes = NodeConfig::default(); - let yaml = generate_kind_config(&nodes); + let yaml = generate_kind_config(&nodes, &[]); assert!(yaml.contains("control-plane"), "should have control-plane"); let cp_count = yaml.matches("control-plane").count(); assert_eq!( @@ -332,7 +359,7 @@ mod tests { control_planes: 3, workers: 2, }; - let yaml = generate_kind_config(&nodes); + let yaml = generate_kind_config(&nodes, &[]); let cp_count = yaml.matches("control-plane").count(); let w_count = yaml.matches("worker").count(); assert_eq!(cp_count, 3, "should have 3 control-planes, got {cp_count}"); @@ -439,4 +466,59 @@ mod tests { }); assert!(!exists, "forge-missing should not exist"); } + + #[test] + fn generate_kind_config_with_port_mappings() { + let nodes = NodeConfig::default(); + let ports = vec![ + PortMapping { + bind_address: None, + host: 13000, + container: 30300, + protocol: "tcp".to_owned(), + }, + PortMapping { + bind_address: None, + host: 19090, + container: 30909, + protocol: "tcp".to_owned(), + }, + ]; + let yaml = generate_kind_config(&nodes, &ports); + assert!( + yaml.contains("extraPortMappings:"), + "should have extraPortMappings" + ); + assert!(yaml.contains("hostPort: 13000"), "should map Grafana port"); + assert!( + yaml.contains("containerPort: 30300"), + "should map Grafana node port" + ); + assert!( + yaml.contains("hostPort: 19090"), + "should map Prometheus port" + ); + assert!(yaml.contains("protocol: TCP"), "should uppercase protocol"); + } + + #[test] + fn generate_kind_config_port_mappings_only_on_first_control_plane() { + let nodes = NodeConfig { + control_planes: 2, + workers: 0, + }; + let ports = vec![PortMapping { + bind_address: None, + host: 8080, + container: 30080, + protocol: "tcp".to_owned(), + }]; + let yaml = generate_kind_config(&nodes, &ports); + // extraPortMappings should appear exactly once + assert_eq!( + yaml.matches("extraPortMappings:").count(), + 1, + "port mappings should only appear on the first control-plane node" + ); + } } diff --git a/src/cluster/mod.rs b/src/cluster/mod.rs index a0db0c8..9120d9f 100644 --- a/src/cluster/mod.rs +++ b/src/cluster/mod.rs @@ -62,7 +62,7 @@ fn handle_create( } let _lock = lock::acquire(&ctx.state_dir)?; let mut state = state::load(&ctx.state_dir)?; - create_if_missing(ctx, &kind_name, &cluster.nodes, &mut state, name)?; + create_if_missing(ctx, &kind_name, cluster, &mut state)?; state::save(&ctx.state_dir, &state)?; report_created(writer, name, &kind_name, &ctx.format) } @@ -168,15 +168,20 @@ fn cluster_kind_name(ctx: &ForgeContext<'_>, name: &str) -> String { fn create_if_missing( ctx: &ForgeContext<'_>, kind_name: &str, - nodes: &crate::config::NodeConfig, + cluster: &crate::config::ClusterSpec, st: &mut state::ForgeState, - name: &str, ) -> Result<(), ForgeError> { if kind_ops::cluster_exists(ctx.runner, kind_name)? { return Ok(()); } - kind_ops::create_cluster(ctx.runner, kind_name, nodes, &ctx.state_dir, None)?; - upsert_cluster_state(st, name, kind_name, ClusterPhase::Running); + let config = kind_ops::CreateClusterConfig { + nodes: &cluster.nodes, + ports: &cluster.ports, + config_dir: &ctx.state_dir, + docker_network: None, + }; + kind_ops::create_cluster(ctx.runner, kind_name, &config)?; + upsert_cluster_state(st, &cluster.name, kind_name, ClusterPhase::Running); Ok(()) } diff --git a/src/command/up.rs b/src/command/up.rs index 490762a..6ef32bb 100644 --- a/src/command/up.rs +++ b/src/command/up.rs @@ -199,13 +199,13 @@ fn create_if_missing( ensure_state_entry(state, &cluster.name, kind_name, ClusterPhase::Running); return Ok(false); } - kind_ops::create_cluster( - ctx.runner, - kind_name, - &cluster.nodes, - &ctx.state_dir, + let cluster_config = kind_ops::CreateClusterConfig { + nodes: &cluster.nodes, + ports: &cluster.ports, + config_dir: &ctx.state_dir, docker_network, - )?; + }; + kind_ops::create_cluster(ctx.runner, kind_name, &cluster_config)?; ensure_state_entry(state, &cluster.name, kind_name, ClusterPhase::Running); Ok(true) } diff --git a/src/config/mod.rs b/src/config/mod.rs index dba3752..344a8a3 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -153,6 +153,13 @@ pub struct ClusterSpec { /// Node layout for this Kind cluster. #[serde(default)] pub nodes: NodeConfig, + /// Host-to-node port mappings (Kind `extraPortMappings`). + /// + /// Maps host ports to Kind node ports via `extraPortMappings` in the + /// Kind cluster config. Required on macOS where `MetalLB` `LoadBalancer` + /// IPs are unreachable from the host. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ports: Vec, /// Stacks to apply to this cluster (must exist in `spec.stacks`). #[serde(default)] pub stacks: Vec, diff --git a/src/config/validate.rs b/src/config/validate.rs index e5a45a6..fa234e9 100644 --- a/src/config/validate.rs +++ b/src/config/validate.rs @@ -25,6 +25,7 @@ pub fn validate(config: &ForgeConfig) -> Result<(), ForgeError> { check_network_name(&config.metadata.name, &config.spec)?; check_cluster_names(config)?; check_cluster_nodes(config)?; + check_cluster_ports(config)?; check_service_names(config)?; check_services(config)?; check_service_deps(config)?; @@ -171,6 +172,35 @@ fn check_cluster_nodes(config: &ForgeConfig) -> Result<(), ForgeError> { Ok(()) } +/// Validate cluster-level port mappings. +fn check_cluster_ports(config: &ForgeConfig) -> Result<(), ForgeError> { + for cluster in &config.spec.clusters { + let mut seen = BTreeSet::new(); + for (i, pm) in cluster.ports.iter().enumerate() { + if pm.host == 0 { + return Err(ForgeError::Validation(format!( + "cluster '{}' port mapping {}: host port must be non-zero", + cluster.name, i, + ))); + } + if pm.container == 0 { + return Err(ForgeError::Validation(format!( + "cluster '{}' port mapping {}: container port must be non-zero", + cluster.name, i, + ))); + } + let key = (pm.host, pm.protocol.to_lowercase()); + if !seen.insert(key) { + return Err(ForgeError::Validation(format!( + "cluster '{}': duplicate host port {} ({})", + cluster.name, pm.host, pm.protocol, + ))); + } + } + } + Ok(()) +} + /// Service names must be unique and DNS-label-valid. fn check_service_names(config: &ForgeConfig) -> Result<(), ForgeError> { let mut seen = BTreeSet::new(); @@ -1193,6 +1223,7 @@ mod tests { let cluster = ClusterSpec { name: "dupe".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: Vec::new(), properties: BTreeMap::new(), }; @@ -1213,6 +1244,7 @@ mod tests { config.spec.clusters = vec![ClusterSpec { name: "c1".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: vec!["nonexistent".to_owned()], properties: BTreeMap::new(), }]; @@ -1232,6 +1264,7 @@ mod tests { config.spec.clusters = vec![ClusterSpec { name: "c1".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: Vec::new(), properties: BTreeMap::from([( "model".to_owned(), @@ -1263,6 +1296,7 @@ mod tests { config.spec.clusters = vec![ClusterSpec { name: "hub".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: vec!["base".to_owned()], properties: BTreeMap::new(), }]; @@ -1299,6 +1333,7 @@ mod tests { control_planes: 0, workers: 1, }, + ports: Vec::new(), stacks: Vec::new(), properties: BTreeMap::new(), }]; @@ -1312,6 +1347,52 @@ mod tests { ); } + #[test] + fn cluster_port_zero_host_rejected() { + let mut config = base_config(); + config.spec.clusters.push(ClusterSpec { + name: "test".to_owned(), + nodes: NodeConfig::default(), + ports: vec![PortMapping { + bind_address: None, + host: 0, + container: 30080, + protocol: "tcp".to_owned(), + }], + stacks: vec![], + properties: BTreeMap::new(), + }); + let result = validate(&config); + assert!(result.is_err(), "zero host port should be rejected"); + } + + #[test] + fn cluster_duplicate_host_port_rejected() { + let mut config = base_config(); + config.spec.clusters.push(ClusterSpec { + name: "test".to_owned(), + nodes: NodeConfig::default(), + ports: vec![ + PortMapping { + bind_address: None, + host: 8080, + container: 30080, + protocol: "tcp".to_owned(), + }, + PortMapping { + bind_address: None, + host: 8080, + container: 30081, + protocol: "tcp".to_owned(), + }, + ], + stacks: vec![], + properties: BTreeMap::new(), + }); + let result = validate(&config); + assert!(result.is_err(), "duplicate host port should be rejected"); + } + #[test] fn invalid_service_protocol_rejected() { let mut config = base_config(); diff --git a/src/stack/engine.rs b/src/stack/engine.rs index b06dae0..021dce4 100644 --- a/src/stack/engine.rs +++ b/src/stack/engine.rs @@ -1161,6 +1161,7 @@ mod tests { let cluster = ClusterSpec { name: "provider-east".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: Vec::new(), properties: BTreeMap::new(), }; diff --git a/src/stack/mod.rs b/src/stack/mod.rs index b8ee565..91a5ed8 100644 --- a/src/stack/mod.rs +++ b/src/stack/mod.rs @@ -703,6 +703,7 @@ mod tests { clusters: vec![ClusterSpec { name: "hub".to_owned(), nodes: NodeConfig::default(), + ports: Vec::new(), stacks: vec!["base".to_owned()], properties: BTreeMap::new(), }], diff --git a/tests/fixtures/port-mappings.yaml b/tests/fixtures/port-mappings.yaml new file mode 100644 index 0000000..0c3e8c1 --- /dev/null +++ b/tests/fixtures/port-mappings.yaml @@ -0,0 +1,21 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment +metadata: + name: port-mapping-test +spec: + runtime: + provider: docker + clusterPrefix: pm-test + network: + crossCluster: false + dnsZone: pm.test + clusters: + - name: local + ports: + - host: 8080 + container: 30080 + - host: 3000 + container: 30300 + protocol: tcp + stacks: [] + stacks: {} diff --git a/tests/integration.rs b/tests/integration.rs index 0cb0579..3ede809 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -313,3 +313,25 @@ fn cli_accepts_stack_status() { let result = Cli::try_parse_from(["praxis-forge", "stack", "status"]); assert!(result.is_ok(), "stack status should parse: {result:?}"); } + +#[test] +fn config_with_port_mappings_parses_and_validates() { + let Ok(yaml) = std::fs::read_to_string("tests/fixtures/port-mappings.yaml") else { + std::process::abort() + }; + let Ok(cfg) = serde_yaml::from_str::(&yaml) else { + std::process::abort() + }; + if validate::validate(&cfg).is_err() { + std::process::abort() + } + let Some(cluster) = cfg.spec.clusters.first() else { + std::process::abort() + }; + assert_eq!(cluster.ports.len(), 2); + let Some(port) = cluster.ports.first() else { + std::process::abort() + }; + assert_eq!(port.host, 8080); + assert_eq!(port.container, 30080); +}