diff --git a/addons/observability/otel-agent/values.yaml b/addons/observability/otel-agent/values.yaml index 3442b05..fe0d6f1 100644 --- a/addons/observability/otel-agent/values.yaml +++ b/addons/observability/otel-agent/values.yaml @@ -170,6 +170,53 @@ config: target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod + # eks-agent-platform operator. It cannot ride the annotation-gated job + # above: charts/operator serves /metrics over HTTPS behind + # controller-runtime's authn/authz filter whenever metrics.secure is + # true — the chart default — and it suppresses its own + # prometheus.io/scrape annotations while secure, because an + # annotation scrape is plaintext and unauthenticated and that endpoint + # rejects it. So the operator was discovered by nothing, and five + # alert rules plus the whole agent-operator dashboard read a metric + # that never arrived. + # + # The authenticated route needs no new grant. The endpoint authorizes + # with a TokenReview plus a SubjectAccessReview on the /metrics + # nonResourceURL, and this collector's ClusterRole above already holds + # exactly that — so the agent presents its own ServiceAccount token, + # the same way the cadvisor job does, rather than mounting the chart's + # metrics-reader Secret from another namespace. + # + # insecure_skip_verify because controller-runtime generates a + # self-signed cert in memory; there is no CA to pin, and the bearer + # token is what actually gates access. Same reasoning the chart's own + # ServiceMonitor records for the prometheus-operator path. + - job_name: eks-agent-platform-operator + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + insecure_skip_verify: true + kubernetes_sd_configs: + - role: pod + namespaces: + names: [eks-agent-platform] + selectors: + - role: pod + field: spec.nodeName=${env:K8S_NODE_NAME} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: operator + action: keep + # The metrics port, not the probe port on 8081. Discovery yields a + # target per declared container port, so without this the agent + # would also scrape :8081 and log a failure every interval. + - source_labels: [__meta_kubernetes_pod_container_port_name] + regex: metrics + action: keep + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod processors: batch: {} memory_limiter: diff --git a/dashboards/base/alerting/agent-operator.yaml b/dashboards/base/alerting/agent-operator.yaml index 6545931..3ac96dd 100644 --- a/dashboards/base/alerting/agent-operator.yaml +++ b/dashboards/base/alerting/agent-operator.yaml @@ -1,8 +1,14 @@ # eks-agent-platform operator — Grafana-managed SLO / health alert rules. The # latency SLO is "99% of reconciles complete in <1s over 30d" (budget 0.01); each # burn rule is a dual-window check encoded as a `> bool` product. Self-contained -# over controller-runtime metrics, which reach AMP once the operator pod carries -# the prometheus.io/scrape annotation (eks-agent-platform operator-prod-scrape). +# over controller-runtime metrics, which reach AMP through the otel-agent's +# `eks-agent-platform-operator` scrape job — an authenticated HTTPS scrape using +# the agent's own ServiceAccount token. NOT the annotation path: the operator +# serves /metrics behind controller-runtime's authn/authz filter while +# metrics.secure is true (the chart default) and suppresses its own +# prometheus.io/scrape annotations for exactly that reason, since an annotation +# scrape is plaintext and unauthenticated. These rules read no data at all until +# that job exists. # # This is the PROD path (Grafana-managed, evaluated by Amazon Managed Grafana # against AMP). The operator chart's own PrometheusRule diff --git a/dashboards/base/kustomization.yaml b/dashboards/base/kustomization.yaml index 7650f3c..7bf2ea5 100644 --- a/dashboards/base/kustomization.yaml +++ b/dashboards/base/kustomization.yaml @@ -48,7 +48,9 @@ resources: - platform/agent-ops.yaml - platform/agent-founder.yaml # Operator reconcile RED + latency SLO/error-budget (controller-runtime metrics - # reach AMP via the operator pod's prometheus.io/scrape annotation). + # reach AMP via the otel-agent's authenticated eks-agent-platform-operator + # scrape job, not the annotation path — the operator's metrics endpoint is + # HTTPS + token-gated and carries no scrape annotations). - platform/agent-operator.yaml # Ops control-plane app (portal): API SLO/RED + tofu-run, River-job, watcher, # and pgxpool surfaces — self-contained PromQL over the portal_* metrics in AMP.