feat(telemetry): opt-in OpenTelemetry trace export over OTLP

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- OpenTelemetry distributed tracing (opt-in) via `[server.telemetry.traces]`: `ring server start` exports spans over OTLP/gRPC — one per HTTP request (stable HTTP-server semantic-convention attributes) and one per productive scheduler cycle. Endpoint, `service.name` and sampler are configurable, standard `OTEL_*` env vars override the TOML, and an unreachable collector degrades gracefully instead of failing the server
+
 ## [0.10.0] - 2026-07-04
 
 ### Added

Cargo.toml

@@ -61,7 +61,7 @@ sysinfo = "0.35.1"
 libc = "0.2"
 thiserror = "2.0.18"
 tracing = "0.1.44"
-tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt"] }
+tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "registry"] }
 tokio-vsock = "0.7"
 rust-embed = "8"
 mime_guess = "2"
@@ -85,6 +85,17 @@ prost = "0.14"
 prost-types = "0.14"
 tokio-stream = "0.1"
 docker_credential = "1.4.0"
+# OpenTelemetry distributed tracing. Spans from `tracing` are bridged to OTel
+# via `tracing-opentelemetry` and exported over OTLP/gRPC (tonic transport,
+# already vendored above for containerd — one tonic across the tree). The whole
+# pipeline is opt-in: with `[server.telemetry.traces]` disabled no exporter is
+# built and there is zero runtime cost. Pinned in lockstep on 0.32 (the core
+# crates release together); `tracing-opentelemetry` 0.33 targets opentelemetry
+# 0.32.
+opentelemetry = "0.32"
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+tracing-opentelemetry = "0.33"
 
 [dev-dependencies]
 axum-test = "17.3.0"

documentation/reference/config-toml.md

@@ -15,6 +15,7 @@ A context describes one client→server connection; it has no business deciding
 [server]                                  # daemon config (shared)
 [server.scheduler]                        # optional
 [server.dashboard]                        # optional
+[server.telemetry.traces]                 # opt-in: enabled = true
 [server.runtime.docker]                   # opt-in: enabled = true
 [server.runtime.cloud_hypervisor]         # opt-in: enabled = true
 
@@ -75,6 +76,21 @@ The daemon's own configuration, shared by every context in the file. All subsect
 | `enabled` | bool | no | `false` | Spawn the embedded dashboard. Also flippable via `--dashboard` / `RING_DASHBOARD` |
 | `listen_address` | string | no | `"127.0.0.1:3031"` | `host:port` the dashboard binds to. Override with `RING_DASHBOARD_LISTEN` |
 
+### `[server.telemetry.traces]`
+
+Opt-in OpenTelemetry span export over OTLP/gRPC. Off by default: with `enabled = false` no exporter is built and the server runs exactly as before. Only `ring server start` exports traces; the CLI commands stay console-only.
+
+| Field | Type | Required | Default | Purpose |
+|---|---|---|---|---|
+| `enabled` | bool | no | `false` | Export spans to an OTLP/gRPC collector |
+| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) |
+| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` |
+| `sampler` | string | no | `"parent_based_always_on"` | `always_on`, `always_off`, `parent_based_always_on`, `parent_based_always_off`, or `ratio:<0..1>` (e.g. `ratio:0.1` for 10% of roots) |
+
+Standard `OTEL_*` environment variables take precedence over the TOML values, so a deployment can point Ring at its collector without editing the file. If the exporter fails to initialise (unreachable endpoint at startup, etc.) Ring logs the error and continues without traces rather than failing.
+
+Ring emits one span per HTTP request (named `{method} {route}` with the stable HTTP-server semantic-convention attributes) and one `scheduler.cycle` span per reconciliation cycle that has work to do — idle scheduler ticks produce no spans.
+
 ### `[server.runtime.docker]`
 
 | Field | Type | Required | Default | Purpose |

src/api/server.rs

@@ -2,7 +2,7 @@ use axum::{
     Router,
     error_handling::HandleErrorLayer,
     http::{HeaderValue, Method, StatusCode, header},
-    middleware::from_fn_with_state,
+    middleware::{from_fn, from_fn_with_state},
     routing::{delete, get, post, put},
 };
 use axum_macros::FromRef;
@@ -176,7 +176,10 @@ pub(crate) fn router(state: AppState) -> Router {
         .merge(public_routes)
         .merge(streaming_routes)
         .merge(api_routes)
-        .with_state(state);
+        .with_state(state)
+        // One tracing span per request (OTel HTTP-server semconv). Outermost so
+        // it wraps every route; a no-op when trace export is disabled.
+        .layer(from_fn(crate::telemetry::http_trace_middleware));
 
     if !cors_origins.is_empty() {
         let origins: Vec<HeaderValue> = cors_origins

src/config/server.rs

@@ -20,6 +20,8 @@ pub(crate) struct ServerConfig {
     pub(crate) runtime: RuntimesConfig,
     #[serde(default)]
     pub(crate) dashboard: DashboardConfig,
+    #[serde(default)]
+    pub(crate) telemetry: TelemetryConfig,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -272,3 +274,117 @@ impl Default for DashboardConfig {
         }
     }
 }
+
+/// `[server.telemetry]` — OpenTelemetry export. One sub-block per signal so the
+/// table can grow without breaking existing configs: `traces` ships now; `logs`
+/// and `metrics` are reserved for later phases and are intentionally absent from
+/// the struct until implemented (an unknown `[server.telemetry.logs]` table is
+/// simply ignored by serde today, so adding it later is backward-compatible).
+#[derive(Deserialize, Debug, Clone, Default)]
+pub(crate) struct TelemetryConfig {
+    #[serde(default)]
+    pub(crate) traces: TracesConfig,
+}
+
+/// `[server.telemetry.traces]` — OTLP/gRPC span export. Opt-in: with `enabled`
+/// false (the default) no exporter is built and Ring runs exactly as before.
+///
+/// Standard `OTEL_*` environment variables override the TOML values so a
+/// deployment can point Ring at its collector without editing the file:
+/// `OTEL_EXPORTER_OTLP_ENDPOINT` overrides `endpoint`, `OTEL_SERVICE_NAME`
+/// overrides `service_name`. Resolution order is env > TOML > built-in default.
+#[derive(Deserialize, Debug, Clone)]
+pub(crate) struct TracesConfig {
+    #[serde(default)]
+    pub(crate) enabled: bool,
+    /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`.
+    #[serde(default = "default_traces_endpoint")]
+    pub(crate) endpoint: String,
+    /// `service.name` resource attribute reported to the collector.
+    #[serde(default = "default_traces_service_name")]
+    pub(crate) service_name: String,
+    /// Sampler: `parent_based_always_on` (default) follows an upstream decision
+    /// and samples roots; `always_on`, `always_off`, or `ratio:<0..1>` (e.g.
+    /// `ratio:0.1` for 10%, wrapped in parent-based).
+    #[serde(default = "default_traces_sampler")]
+    pub(crate) sampler: String,
+}
+
+fn default_traces_endpoint() -> String {
+    "http://127.0.0.1:4317".to_string()
+}
+
+fn default_traces_service_name() -> String {
+    "ring-server".to_string()
+}
+
+fn default_traces_sampler() -> String {
+    "parent_based_always_on".to_string()
+}
+
+impl Default for TracesConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            endpoint: default_traces_endpoint(),
+            service_name: default_traces_service_name(),
+            sampler: default_traces_sampler(),
+        }
+    }
+}
+
+impl TracesConfig {
+    /// Effective collector endpoint: `OTEL_EXPORTER_OTLP_ENDPOINT` (or the
+    /// traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, which wins per the
+    /// OTLP spec) overrides the configured value.
+    pub(crate) fn resolved_endpoint(&self) -> String {
+        std::env::var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+            .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT"))
+            .unwrap_or_else(|_| self.endpoint.clone())
+    }
+
+    /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured
+    /// value.
+    pub(crate) fn resolved_service_name(&self) -> String {
+        std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone())
+    }
+}
+
+#[cfg(test)]
+mod telemetry_tests {
+    use super::*;
+
+    #[test]
+    fn traces_disabled_by_default() {
+        let c = TracesConfig::default();
+        assert!(!c.enabled);
+        assert_eq!(c.endpoint, "http://127.0.0.1:4317");
+        assert_eq!(c.service_name, "ring-server");
+        assert_eq!(c.sampler, "parent_based_always_on");
+    }
+
+    #[test]
+    fn telemetry_absent_from_toml_yields_disabled_traces() {
+        // A `[server]` table with no telemetry block must not enable anything.
+        let cfg: ServerConfig = toml::from_str("").unwrap();
+        assert!(!cfg.telemetry.traces.enabled);
+    }
+
+    #[test]
+    fn traces_block_parses_from_toml() {
+        let cfg: ServerConfig = toml::from_str(
+            r#"
+            [telemetry.traces]
+            enabled = true
+            endpoint = "http://collector:4317"
+            service_name = "ring-prod"
+            sampler = "ratio:0.25"
+            "#,
+        )
+        .unwrap();
+        assert!(cfg.telemetry.traces.enabled);
+        assert_eq!(cfg.telemetry.traces.endpoint, "http://collector:4317");
+        assert_eq!(cfg.telemetry.traces.service_name, "ring-prod");
+        assert_eq!(cfg.telemetry.traces.sampler, "ratio:0.25");
+    }
+}