From b094a23d67c4a1f42c40e35761edec0bcf0a27c8 Mon Sep 17 00:00:00 2001 From: stxkxs <139715017+stxkxs@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:10:28 -0700 Subject: [PATCH] fix: the operator's metrics reach nothing, and two comments say otherwise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five alert rules and the whole agent-operator dashboard read controller-runtime metrics from the eks-agent-platform operator. Those metrics have never reached Amazon Managed Prometheus, because nothing discovers the operator. charts/operator serves /metrics over HTTPS behind controller-runtime's authentication and authorization filter whenever metrics.secure is true, which is the chart default and is overridden nowhere. The chart then suppresses its own prometheus.io/scrape annotations while secure — deliberately, because an annotation scrape is plaintext and unauthenticated and that endpoint rejects it. Rendering the chart against this repo's values emits zero scrape annotations. The otel-agent's only pod job is annotation-gated, so it never sees the operator. The chart also renders a ServiceMonitor for the authenticated path, but this fleet installs prometheus-operator-crds and no controller that consumes one, so it is read by nothing. There is no AMP managed scraper either. The result is five rules and a dashboard wired to a metric that never arrives. Two comments in this repo asserted the opposite as fact — that the metrics "reach AMP via the operator pod's prometheus.io/scrape annotation". Both are corrected here. A comment describing a wiring that does not exist is the same defect as the wiring being absent, and harder to find, because it answers the question a reader would otherwise go and check. ─────────────────────────── the fix ─────────────────────────── A dedicated scrape job on the otel-agent that takes the authenticated route: HTTPS, the agent's own ServiceAccount token as the bearer credential, TLS verification skipped because controller-runtime generates a self-signed cert in memory and there is no CA to pin — the token is what gates access, which is the same reasoning the chart's own ServiceMonitor records. No new grant is needed, and that is worth stating precisely rather than assuming: the endpoint authorizes with a TokenReview plus a SubjectAccessReview on the /metrics nonResourceURL, and this collector's ClusterRole already carries `nonResourceURLs: ["/metrics"], verbs: ["get"]` for the kubelet scrape. So the agent presents its own token rather than mounting the chart's metrics-reader Secret, which lives in another namespace and could not be mounted anyway. metrics.secure stays true. Setting it false at this layer would have been one line and would have reopened exactly the info-disclosure gap the chart closed on purpose. Discovery is namespace-scoped and node-scoped, matching every other job here, so each operator pod is scraped exactly once by the agent co-located with it. The container-port keep is not cosmetic: pod discovery yields one target per declared container port, and without it the agent would also scrape the probe port on 8081 and log a failure every interval. Verified by rendering collector chart 0.166.0 against the committed values and extracting the emitted config: the job is present with the intended scheme, token file and relabel set. Those scrape configs were then checked with `promtool check config`, which passes — and the check was itself checked by injecting an invalid relabel action, which promtool rejects with `unknown relabel action`, so the pass means it parsed something. --- addons/observability/otel-agent/values.yaml | 47 ++++++++++++++++++++ dashboards/base/alerting/agent-operator.yaml | 10 ++++- dashboards/base/kustomization.yaml | 4 +- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/addons/observability/otel-agent/values.yaml b/addons/observability/otel-agent/values.yaml index 3442b05..fe0d6f1 100644 --- a/addons/observability/otel-agent/values.yaml +++ b/addons/observability/otel-agent/values.yaml @@ -170,6 +170,53 @@ config: target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod + # eks-agent-platform operator. It cannot ride the annotation-gated job + # above: charts/operator serves /metrics over HTTPS behind + # controller-runtime's authn/authz filter whenever metrics.secure is + # true — the chart default — and it suppresses its own + # prometheus.io/scrape annotations while secure, because an + # annotation scrape is plaintext and unauthenticated and that endpoint + # rejects it. So the operator was discovered by nothing, and five + # alert rules plus the whole agent-operator dashboard read a metric + # that never arrived. + # + # The authenticated route needs no new grant. The endpoint authorizes + # with a TokenReview plus a SubjectAccessReview on the /metrics + # nonResourceURL, and this collector's ClusterRole above already holds + # exactly that — so the agent presents its own ServiceAccount token, + # the same way the cadvisor job does, rather than mounting the chart's + # metrics-reader Secret from another namespace. + # + # insecure_skip_verify because controller-runtime generates a + # self-signed cert in memory; there is no CA to pin, and the bearer + # token is what actually gates access. Same reasoning the chart's own + # ServiceMonitor records for the prometheus-operator path. + - job_name: eks-agent-platform-operator + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + insecure_skip_verify: true + kubernetes_sd_configs: + - role: pod + namespaces: + names: [eks-agent-platform] + selectors: + - role: pod + field: spec.nodeName=${env:K8S_NODE_NAME} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: operator + action: keep + # The metrics port, not the probe port on 8081. Discovery yields a + # target per declared container port, so without this the agent + # would also scrape :8081 and log a failure every interval. + - source_labels: [__meta_kubernetes_pod_container_port_name] + regex: metrics + action: keep + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod processors: batch: {} memory_limiter: diff --git a/dashboards/base/alerting/agent-operator.yaml b/dashboards/base/alerting/agent-operator.yaml index 6545931..3ac96dd 100644 --- a/dashboards/base/alerting/agent-operator.yaml +++ b/dashboards/base/alerting/agent-operator.yaml @@ -1,8 +1,14 @@ # eks-agent-platform operator — Grafana-managed SLO / health alert rules. The # latency SLO is "99% of reconciles complete in <1s over 30d" (budget 0.01); each # burn rule is a dual-window check encoded as a `> bool` product. Self-contained -# over controller-runtime metrics, which reach AMP once the operator pod carries -# the prometheus.io/scrape annotation (eks-agent-platform operator-prod-scrape). +# over controller-runtime metrics, which reach AMP through the otel-agent's +# `eks-agent-platform-operator` scrape job — an authenticated HTTPS scrape using +# the agent's own ServiceAccount token. NOT the annotation path: the operator +# serves /metrics behind controller-runtime's authn/authz filter while +# metrics.secure is true (the chart default) and suppresses its own +# prometheus.io/scrape annotations for exactly that reason, since an annotation +# scrape is plaintext and unauthenticated. These rules read no data at all until +# that job exists. # # This is the PROD path (Grafana-managed, evaluated by Amazon Managed Grafana # against AMP). The operator chart's own PrometheusRule diff --git a/dashboards/base/kustomization.yaml b/dashboards/base/kustomization.yaml index 7650f3c..7bf2ea5 100644 --- a/dashboards/base/kustomization.yaml +++ b/dashboards/base/kustomization.yaml @@ -48,7 +48,9 @@ resources: - platform/agent-ops.yaml - platform/agent-founder.yaml # Operator reconcile RED + latency SLO/error-budget (controller-runtime metrics - # reach AMP via the operator pod's prometheus.io/scrape annotation). + # reach AMP via the otel-agent's authenticated eks-agent-platform-operator + # scrape job, not the annotation path — the operator's metrics endpoint is + # HTTPS + token-gated and carries no scrape annotations). - platform/agent-operator.yaml # Ops control-plane app (portal): API SLO/RED + tofu-run, River-job, watcher, # and pgxpool surfaces — self-contained PromQL over the portal_* metrics in AMP.