diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a36ec..3f23baa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- OpenTelemetry distributed tracing (opt-in) via `[server.telemetry.traces]`: `ring server start` exports spans over OTLP/gRPC — one per HTTP request (stable HTTP-server semantic-convention attributes) and one per productive scheduler cycle. Endpoint, `service.name` and sampler are configurable, standard `OTEL_*` env vars override the TOML, and an unreachable collector degrades gracefully instead of failing the server + ## [0.10.0] - 2026-07-04 ### Added diff --git a/Cargo.lock b/Cargo.lock index 48c8a88..cbcfbf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2021,6 +2021,68 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.4", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -2157,6 +2219,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2515,6 +2583,9 @@ dependencies = [ "mime_guess", "nix", "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "owo-colors", "prost", "prost-types", @@ -2539,6 +2610,7 @@ dependencies = [ "tower", "tower-http", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "url", "uuid", @@ -3566,6 +3638,17 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3659,6 +3742,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -4009,6 +4108,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "whoami" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index d712790..0ac33f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ sysinfo = "0.35.1" libc = "0.2" thiserror = "2.0.18" tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt"] } +tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "registry"] } tokio-vsock = "0.7" rust-embed = "8" mime_guess = "2" @@ -85,6 +85,17 @@ prost = "0.14" prost-types = "0.14" tokio-stream = "0.1" docker_credential = "1.4.0" +# OpenTelemetry distributed tracing. Spans from `tracing` are bridged to OTel +# via `tracing-opentelemetry` and exported over OTLP/gRPC (tonic transport, +# already vendored above for containerd — one tonic across the tree). The whole +# pipeline is opt-in: with `[server.telemetry.traces]` disabled no exporter is +# built and there is zero runtime cost. Pinned in lockstep on 0.32 (the core +# crates release together); `tracing-opentelemetry` 0.33 targets opentelemetry +# 0.32. +opentelemetry = "0.32" +opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] } +tracing-opentelemetry = "0.33" [dev-dependencies] axum-test = "17.3.0" diff --git a/documentation/reference/config-toml.md b/documentation/reference/config-toml.md index 203f815..6c54b58 100644 --- a/documentation/reference/config-toml.md +++ b/documentation/reference/config-toml.md @@ -15,6 +15,7 @@ A context describes one client→server connection; it has no business deciding [server] # daemon config (shared) [server.scheduler] # optional [server.dashboard] # optional +[server.telemetry.traces] # opt-in: enabled = true [server.runtime.docker] # opt-in: enabled = true [server.runtime.cloud_hypervisor] # opt-in: enabled = true @@ -75,6 +76,21 @@ The daemon's own configuration, shared by every context in the file. All subsect | `enabled` | bool | no | `false` | Spawn the embedded dashboard. Also flippable via `--dashboard` / `RING_DASHBOARD` | | `listen_address` | string | no | `"127.0.0.1:3031"` | `host:port` the dashboard binds to. Override with `RING_DASHBOARD_LISTEN` | +### `[server.telemetry.traces]` + +Opt-in OpenTelemetry span export over OTLP/gRPC. Off by default: with `enabled = false` no exporter is built and the server runs exactly as before. Only `ring server start` exports traces; the CLI commands stay console-only. + +| Field | Type | Required | Default | Purpose | +|---|---|---|---|---| +| `enabled` | bool | no | `false` | Export spans to an OTLP/gRPC collector | +| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) | +| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` | +| `sampler` | string | no | `"parent_based_always_on"` | `always_on`, `always_off`, `parent_based_always_on`, `parent_based_always_off`, or `ratio:<0..1>` (e.g. `ratio:0.1` for 10% of roots) | + +Standard `OTEL_*` environment variables take precedence over the TOML values, so a deployment can point Ring at its collector without editing the file. If the exporter fails to initialise (unreachable endpoint at startup, etc.) Ring logs the error and continues without traces rather than failing. + +Ring emits one span per HTTP request (named `{method} {route}` with the stable HTTP-server semantic-convention attributes) and one `scheduler.cycle` span per reconciliation cycle that has work to do — idle scheduler ticks produce no spans. + ### `[server.runtime.docker]` | Field | Type | Required | Default | Purpose | diff --git a/src/api/server.rs b/src/api/server.rs index e99ef07..78e98ac 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -2,7 +2,7 @@ use axum::{ Router, error_handling::HandleErrorLayer, http::{HeaderValue, Method, StatusCode, header}, - middleware::from_fn_with_state, + middleware::{from_fn, from_fn_with_state}, routing::{delete, get, post, put}, }; use axum_macros::FromRef; @@ -176,7 +176,10 @@ pub(crate) fn router(state: AppState) -> Router { .merge(public_routes) .merge(streaming_routes) .merge(api_routes) - .with_state(state); + .with_state(state) + // One tracing span per request (OTel HTTP-server semconv). Outermost so + // it wraps every route; a no-op when trace export is disabled. + .layer(from_fn(crate::telemetry::http_trace_middleware)); if !cors_origins.is_empty() { let origins: Vec = cors_origins diff --git a/src/config/server.rs b/src/config/server.rs index 0037100..16bd31d 100644 --- a/src/config/server.rs +++ b/src/config/server.rs @@ -20,6 +20,8 @@ pub(crate) struct ServerConfig { pub(crate) runtime: RuntimesConfig, #[serde(default)] pub(crate) dashboard: DashboardConfig, + #[serde(default)] + pub(crate) telemetry: TelemetryConfig, } #[derive(Deserialize, Debug, Clone)] @@ -272,3 +274,117 @@ impl Default for DashboardConfig { } } } + +/// `[server.telemetry]` — OpenTelemetry export. One sub-block per signal so the +/// table can grow without breaking existing configs: `traces` ships now; `logs` +/// and `metrics` are reserved for later phases and are intentionally absent from +/// the struct until implemented (an unknown `[server.telemetry.logs]` table is +/// simply ignored by serde today, so adding it later is backward-compatible). +#[derive(Deserialize, Debug, Clone, Default)] +pub(crate) struct TelemetryConfig { + #[serde(default)] + pub(crate) traces: TracesConfig, +} + +/// `[server.telemetry.traces]` — OTLP/gRPC span export. Opt-in: with `enabled` +/// false (the default) no exporter is built and Ring runs exactly as before. +/// +/// Standard `OTEL_*` environment variables override the TOML values so a +/// deployment can point Ring at its collector without editing the file: +/// `OTEL_EXPORTER_OTLP_ENDPOINT` overrides `endpoint`, `OTEL_SERVICE_NAME` +/// overrides `service_name`. Resolution order is env > TOML > built-in default. +#[derive(Deserialize, Debug, Clone)] +pub(crate) struct TracesConfig { + #[serde(default)] + pub(crate) enabled: bool, + /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`. + #[serde(default = "default_traces_endpoint")] + pub(crate) endpoint: String, + /// `service.name` resource attribute reported to the collector. + #[serde(default = "default_traces_service_name")] + pub(crate) service_name: String, + /// Sampler: `parent_based_always_on` (default) follows an upstream decision + /// and samples roots; `always_on`, `always_off`, or `ratio:<0..1>` (e.g. + /// `ratio:0.1` for 10%, wrapped in parent-based). + #[serde(default = "default_traces_sampler")] + pub(crate) sampler: String, +} + +fn default_traces_endpoint() -> String { + "http://127.0.0.1:4317".to_string() +} + +fn default_traces_service_name() -> String { + "ring-server".to_string() +} + +fn default_traces_sampler() -> String { + "parent_based_always_on".to_string() +} + +impl Default for TracesConfig { + fn default() -> Self { + Self { + enabled: false, + endpoint: default_traces_endpoint(), + service_name: default_traces_service_name(), + sampler: default_traces_sampler(), + } + } +} + +impl TracesConfig { + /// Effective collector endpoint: `OTEL_EXPORTER_OTLP_ENDPOINT` (or the + /// traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, which wins per the + /// OTLP spec) overrides the configured value. + pub(crate) fn resolved_endpoint(&self) -> String { + std::env::var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")) + .unwrap_or_else(|_| self.endpoint.clone()) + } + + /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured + /// value. + pub(crate) fn resolved_service_name(&self) -> String { + std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone()) + } +} + +#[cfg(test)] +mod telemetry_tests { + use super::*; + + #[test] + fn traces_disabled_by_default() { + let c = TracesConfig::default(); + assert!(!c.enabled); + assert_eq!(c.endpoint, "http://127.0.0.1:4317"); + assert_eq!(c.service_name, "ring-server"); + assert_eq!(c.sampler, "parent_based_always_on"); + } + + #[test] + fn telemetry_absent_from_toml_yields_disabled_traces() { + // A `[server]` table with no telemetry block must not enable anything. + let cfg: ServerConfig = toml::from_str("").unwrap(); + assert!(!cfg.telemetry.traces.enabled); + } + + #[test] + fn traces_block_parses_from_toml() { + let cfg: ServerConfig = toml::from_str( + r#" + [telemetry.traces] + enabled = true + endpoint = "http://collector:4317" + service_name = "ring-prod" + sampler = "ratio:0.25" + "#, + ) + .unwrap(); + assert!(cfg.telemetry.traces.enabled); + assert_eq!(cfg.telemetry.traces.endpoint, "http://collector:4317"); + assert_eq!(cfg.telemetry.traces.service_name, "ring-prod"); + assert_eq!(cfg.telemetry.traces.sampler, "ratio:0.25"); + } +} diff --git a/src/main.rs b/src/main.rs index 4a18d3d..57f20fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,20 +25,59 @@ mod dashboard; mod database; mod exit_code; mod serializer; +mod telemetry; mod utils; #[cfg(test)] mod fixtures; +/// Build a fresh `EnvFilter` from `RUST_LOG` (falling back to `info`). Called +/// per layer so each layer filters independently rather than relying on the +/// `.with()` order — an `EnvFilter` added as a bare layer would otherwise act +/// globally over the whole registry. +fn env_filter() -> tracing_subscriber::EnvFilter { + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")) +} + +/// Initialise `tracing`. Always installs the console `fmt` layer; when +/// `traces` is `Some` and enabled, also attaches the OpenTelemetry span layer +/// and returns its guard (which must be kept alive for the process). If the +/// OTLP exporter can't be built, logs the error and continues console-only. +/// +/// Called exactly once, after the config is loaded, so `server start` can turn +/// tracing on while every other command stays console-only at zero cost. +fn init_tracing(traces: Option<&config::server::TracesConfig>) -> Option { + use tracing_subscriber::layer::{Layer, SubscriberExt}; + use tracing_subscriber::util::SubscriberInitExt; + + let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter()); + let registry = tracing_subscriber::registry().with(fmt_layer); + + match traces { + Some(cfg) if cfg.enabled => match telemetry::build_tracer(cfg) { + Ok((otel_layer, guard)) => { + registry.with(otel_layer.with_filter(env_filter())).init(); + info!("telemetry: OTLP trace export enabled ({})", cfg.endpoint); + Some(guard) + } + Err(e) => { + registry.init(); + error!( + "telemetry: failed to initialise OTLP exporter, continuing without traces: {e}" + ); + None + } + }, + _ => { + registry.init(); + None + } + } +} + #[tokio::main] async fn main() { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - let app = Command::new("ring") .version(env!("CARGO_PKG_VERSION")) .author("Mlanawo Mbechezi ") @@ -152,6 +191,17 @@ async fn main() { let subcommand_name = matches.subcommand(); let config = config::config::load_config(context, config_file); + + // OTLP trace export is a server concern: enable it only for `server start`, + // where the daemon runs long enough for batching to matter. Every other + // command initialises console logging only. The guard lives for the whole + // of `main` so spans flush on exit. + let is_server_start = matches!( + subcommand_name, + Some(("server", sub)) if sub.subcommand().map(|(n, _)| n).unwrap_or("start") == "start" + ); + let _telemetry_guard = init_tracing(is_server_start.then_some(&config.server.telemetry.traces)); + let client = reqwest::Client::builder() .default_headers({ let mut headers = reqwest::header::HeaderMap::new(); diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 4d6cb74..cb71ded 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -19,6 +19,7 @@ use std::env; use std::sync::Arc; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep}; +use tracing::Instrument as _; async fn resolve_environment(deployment: &mut Deployment, pool: &SqlitePool) -> Result<(), String> { let mut resolved = HashMap::new(); @@ -243,6 +244,22 @@ async fn prepare_deployment(pool: &SqlitePool, deployment: &Deployment) -> Optio Some(resolved) } +// Span the runtime apply (image pull + container create/start, or teardown). +// This is the step whose wall-clock dominates a rollout, so measuring it +// directly answers "is the delay the pull or the boot?" from the trace instead +// of guesswork. Only the deployment identity and status are recorded; the large +// value args are skipped. +#[tracing::instrument( + name = "scheduler.apply_runtime", + skip_all, + fields( + otel.kind = "internal", + deployment.id = %deployment.id, + deployment.namespace = %deployment.namespace, + deployment.name = %deployment.name, + deployment.status = %deployment.status, + ) +)] async fn apply_runtime( pool: &SqlitePool, deployment: &Deployment, @@ -1275,6 +1292,30 @@ pub(crate) async fn schedule( debug!("Processing {} deployments", list_deployments.len()); let mut deleted: Vec = Vec::new(); + // Trace a cycle only when it has work to do. An idle tick (no + // reconcilable deployment) opens no span, so a scheduler polling every + // few seconds doesn't drown the collector in empty traces — the + // recommended pattern for periodic background loops. Each productive + // cycle is its own INTERNAL root span carrying the workload size. The + // span is attached to the work future via `Instrument` rather than an + // `entered()` guard, which cannot be held across `.await` on a + // multi-thread runtime. + let cycle_span = if list_deployments.is_empty() { + tracing::Span::none() + } else { + info_span!( + "scheduler.cycle", + otel.kind = "internal", + deployments.count = list_deployments.len(), + ) + }; + + // Instrument the reconciliation work with the cycle span. A plain + // (non-`move`) async block borrows the surrounding state, and awaiting + // it in place attaches the span without holding an `entered()` guard + // across `.await` (which would make the loop future non-`Send`). An + // idle tick uses `Span::none()`, so no span is recorded at all. + async { for deployment in list_deployments.into_iter() { let runtime = match runtimes.get(&deployment.runtime) { Some(rt) => rt.clone(), @@ -1329,9 +1370,19 @@ pub(crate) async fn schedule( // Honour the retry backoff. (Deletes are handled above and never // reach this point, so they're never blocked by backoff.) if backoff.is_blocked(&deployment.id) { - debug!( - "Deployment {} in retry backoff, skipping cycle", - deployment.id + // Emit this at info as a structured event, not a bare debug line: + // a deployment being skipped by backoff is the single most useful + // signal when diagnosing "why is my rollout slow to converge?", + // and it's otherwise invisible in production. As an event inside + // the `scheduler.cycle` span it surfaces directly in the trace, + // tagged with the deployment and its retry count. + info!( + deployment.id = %deployment.id, + deployment.namespace = %deployment.namespace, + deployment.name = %deployment.name, + deployment.status = %deployment.status, + deployment.restart_count = deployment.restart_count, + "deployment skipped this cycle: in retry backoff" ); continue; } @@ -1567,6 +1618,9 @@ pub(crate) async fn schedule( } cleanup_deleted(&pool, deleted).await; + } + .instrument(cycle_span) + .await; if last_cleanup.elapsed() >= cleanup_interval { last_cleanup = Instant::now(); diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 0000000..a0b70f6 --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,207 @@ +//! OpenTelemetry distributed-tracing pipeline (opt-in). +//! +//! When `[server.telemetry.traces]` is enabled, [`build_tracer`] builds an +//! OTLP/gRPC span exporter, wires it into an [`SdkTracerProvider`] with a +//! configurable sampler, and returns a `tracing_subscriber` layer that turns +//! every `tracing` span into an OpenTelemetry span, plus a guard. +//! +//! The guard ([`OtelGuard`]) owns the provider and flushes on drop: dropping it +//! without flushing would lose the last, un-exported batch. Keep it alive for +//! the whole process (bind it in `main`), and it shuts the pipeline down cleanly +//! on exit. +//! +//! Spans are batch-exported. The batch processor (0.28+) spawns its own +//! background worker and no longer takes a runtime argument; Ring runs on a +//! multi-thread tokio runtime, which sidesteps the current-thread shutdown +//! deadlock documented upstream. +//! +//! Failure is non-fatal by design: if the exporter can't be built (bad +//! endpoint, etc.) the caller logs and continues logs-only rather than crash. + +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_otlp::WithExportConfig as _; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::{Sampler, SdkTracerProvider}; +use tracing::Instrument as _; + +use crate::config::server::TracesConfig; + +/// Parse the `sampler` config string into an OpenTelemetry [`Sampler`]. +/// +/// - `parent_based_always_on` (default): honour an upstream sampling decision, +/// otherwise sample. The right server default — if a caller is tracing, we +/// continue the trace; unparented roots are sampled. +/// - `always_on` / `always_off`: force the decision. +/// - `parent_based_always_off`: honour the parent, drop unparented roots. +/// - `ratio:<0..1>`: parent-based ratio sampler, e.g. `ratio:0.1` = 10% of +/// roots. +/// +/// An unrecognised value falls back to `parent_based_always_on` and logs once. +fn parse_sampler(spec: &str) -> Sampler { + match spec.trim().to_ascii_lowercase().as_str() { + "always_on" => Sampler::AlwaysOn, + "always_off" => Sampler::AlwaysOff, + "parent_based_always_on" => Sampler::ParentBased(Box::new(Sampler::AlwaysOn)), + "parent_based_always_off" => Sampler::ParentBased(Box::new(Sampler::AlwaysOff)), + other => { + if let Some(rest) = other.strip_prefix("ratio:") + && let Ok(ratio) = rest.parse::() + { + let ratio = ratio.clamp(0.0, 1.0); + return Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(ratio))); + } + warn!("telemetry: unknown sampler '{spec}', using parent_based_always_on"); + Sampler::ParentBased(Box::new(Sampler::AlwaysOn)) + } + } +} + +/// Keeps the tracer provider alive for the process lifetime and flushes pending +/// spans on shutdown. Drop it (or call [`OtelGuard::shutdown`]) to force-flush +/// the last batch. Held by `main` so the pipeline lives as long as the server. +pub(crate) struct OtelGuard { + provider: SdkTracerProvider, +} + +impl OtelGuard { + /// Flush and stop the exporter. Idempotent enough for a shutdown path. + pub(crate) fn shutdown(&self) { + if let Err(e) = self.provider.force_flush() { + warn!("telemetry: failed to flush spans on shutdown: {e}"); + } + if let Err(e) = self.provider.shutdown() { + warn!("telemetry: failed to shut tracer provider down: {e}"); + } + } +} + +impl Drop for OtelGuard { + fn drop(&mut self) { + self.shutdown(); + } +} + +/// Build the OTLP tracing pipeline and return `(layer, guard)`. +/// +/// The layer must be added to the `tracing_subscriber` registry; the guard must +/// be kept alive for the process. Returns `Err` if the OTLP exporter can't be +/// built so the caller can fall back to logs-only rather than crash. Endpoint +/// and service name are resolved with `OTEL_*` env overriding the TOML values. +pub(crate) fn build_tracer( + cfg: &TracesConfig, +) -> anyhow::Result<( + tracing_opentelemetry::OpenTelemetryLayer, + OtelGuard, +)> +where + S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>, +{ + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(cfg.resolved_endpoint()) + .build()?; + + let resource = Resource::builder() + .with_service_name(cfg.resolved_service_name()) + .build(); + + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_sampler(parse_sampler(&cfg.sampler)) + .with_resource(resource) + .build(); + + let tracer = provider.tracer("ring-server"); + let layer = tracing_opentelemetry::layer().with_tracer(tracer); + + Ok((layer, OtelGuard { provider })) +} + +/// axum middleware: open one `tracing` span per HTTP request, carrying the +/// stable OpenTelemetry HTTP-server semantic-convention attributes. Attached to +/// the root router, so when the OTel layer is active each request becomes a +/// span; when it isn't, this is just a cheap `tracing` span with no exporter. +/// +/// The span name is `{method} {http.route}` (low cardinality — the matched +/// route template, never the raw path). Attributes follow the semconv stable +/// set: `http.request.method`, `url.path`, `http.route`, +/// `http.response.status_code`. +pub(crate) async fn http_trace_middleware( + matched: Option, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let method = request.method().clone(); + let path = request.uri().path().to_string(); + // `http.route` is the low-cardinality template (e.g. `/users/{id}`); fall + // back to the raw path only when no route matched. + let route = matched + .as_ref() + .map(|m| m.as_str().to_string()) + .unwrap_or_else(|| path.clone()); + + let span = info_span!( + "http.server.request", + otel.name = %format!("{method} {route}"), + otel.kind = "server", + http.request.method = %method, + url.path = %path, + http.route = %route, + http.response.status_code = tracing::field::Empty, + ); + + // Attach the span to the future rather than holding an `enter()` guard + // across `.await` (which would leak the span onto whatever task the + // executor parks on at the yield point). + async move { + let response = next.run(request).await; + tracing::Span::current().record( + "http.response.status_code", + response.status().as_u16() as i64, + ); + response + } + .instrument(span) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sampler_parsing_known_values() { + assert!(matches!(parse_sampler("always_on"), Sampler::AlwaysOn)); + assert!(matches!(parse_sampler("always_off"), Sampler::AlwaysOff)); + assert!(matches!( + parse_sampler("parent_based_always_on"), + Sampler::ParentBased(_) + )); + assert!(matches!( + parse_sampler("parent_based_always_off"), + Sampler::ParentBased(_) + )); + assert!(matches!( + parse_sampler("ratio:0.1"), + Sampler::ParentBased(_) + )); + } + + #[test] + fn sampler_is_case_and_whitespace_insensitive() { + assert!(matches!(parse_sampler(" ALWAYS_ON "), Sampler::AlwaysOn)); + } + + #[test] + fn unknown_sampler_falls_back_to_parent_based() { + assert!(matches!(parse_sampler("nonsense"), Sampler::ParentBased(_))); + } + + #[test] + fn ratio_out_of_range_is_clamped_not_panicking() { + // Must not panic; clamping happens internally. + let _ = parse_sampler("ratio:5.0"); + let _ = parse_sampler("ratio:-1"); + let _ = parse_sampler("ratio:notanumber"); + } +} diff --git a/tests/e2e/server/t15_telemetry_traces_boot.sh b/tests/e2e/server/t15_telemetry_traces_boot.sh new file mode 100755 index 0000000..dab4079 --- /dev/null +++ b/tests/e2e/server/t15_telemetry_traces_boot.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# T15-server: enabling OTLP trace export wires up cleanly and degrades +# gracefully when the collector is unreachable. +# +# Autonomous test (own short-lived Ring, no collector). The e2e suite ships no +# OTLP collector, so real span ingestion is covered by the unit tests in +# `src/telemetry.rs` (sampler parsing) and `src/config/server.rs` (config +# resolution). Here we assert the operational contract: +# +# Invariants: +# 1. With `[server.telemetry.traces] enabled = true` pointing at a closed +# port, Ring still boots and becomes healthy — a telemetry export failure +# must never take the server down. +# 2. The startup log confirms trace export was enabled (so a misconfiguration +# that silently disables it is visible). +# 3. The API still serves requests normally with tracing on (the per-request +# span middleware doesn't break the response path). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +source "$SCRIPT_DIR/../lib.sh" + +log "== T15-server: telemetry traces boot + graceful degradation ==" + +# Point the exporter at a closed port on purpose: export must fail silently in +# the background, not stop the server from coming up. +export RING_EXTRA_CONFIG='[server.telemetry.traces] +enabled = true +endpoint = "http://127.0.0.1:14317" +service_name = "ring-e2e" +sampler = "always_on"' + +# Invariant 1: start_ring only returns 0 once /healthz answers, so a successful +# return already proves the server booted with an unreachable collector. +start_ring + +# Invariant 3: a normal request still succeeds with the tracing middleware in +# the stack. +if ! curl -sf "${RING_URL}/healthz" > /dev/null 2>&1; then + fail "healthz did not answer with tracing enabled" +fi +log "server healthy and serving requests with tracing enabled" + +# Invariant 2: the enable log line is present. +if ! grep -q "OTLP trace export enabled" "$RING_TEST_DIR/ring.log"; then + echo "[e2e] ring.log:" >&2 + cat "$RING_TEST_DIR/ring.log" >&2 + fail "expected 'OTLP trace export enabled' in the startup log" +fi +log "startup log confirms OTLP trace export was enabled" + +log "PASS T15-server: telemetry traces boot"