fix: the operator's metrics reach nothing, and two comments say otherwise

addons/observability/otel-agent/values.yaml

@@ -170,6 +170,53 @@ config:
                 target_label: namespace
               - source_labels: [__meta_kubernetes_pod_name]
                 target_label: pod
+          # eks-agent-platform operator. It cannot ride the annotation-gated job
+          # above: charts/operator serves /metrics over HTTPS behind
+          # controller-runtime's authn/authz filter whenever metrics.secure is
+          # true — the chart default — and it suppresses its own
+          # prometheus.io/scrape annotations while secure, because an
+          # annotation scrape is plaintext and unauthenticated and that endpoint
+          # rejects it. So the operator was discovered by nothing, and five
+          # alert rules plus the whole agent-operator dashboard read a metric
+          # that never arrived.
+          #
+          # The authenticated route needs no new grant. The endpoint authorizes
+          # with a TokenReview plus a SubjectAccessReview on the /metrics
+          # nonResourceURL, and this collector's ClusterRole above already holds
+          # exactly that — so the agent presents its own ServiceAccount token,
+          # the same way the cadvisor job does, rather than mounting the chart's
+          # metrics-reader Secret from another namespace.
+          #
+          # insecure_skip_verify because controller-runtime generates a
+          # self-signed cert in memory; there is no CA to pin, and the bearer
+          # token is what actually gates access. Same reasoning the chart's own
+          # ServiceMonitor records for the prometheus-operator path.
+          - job_name: eks-agent-platform-operator
+            scheme: https
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            tls_config:
+              insecure_skip_verify: true
+            kubernetes_sd_configs:
+              - role: pod
+                namespaces:
+                  names: [eks-agent-platform]
+                selectors:
+                  - role: pod
+                    field: spec.nodeName=${env:K8S_NODE_NAME}
+            relabel_configs:
+              - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+                regex: operator
+                action: keep
+              # The metrics port, not the probe port on 8081. Discovery yields a
+              # target per declared container port, so without this the agent
+              # would also scrape :8081 and log a failure every interval.
+              - source_labels: [__meta_kubernetes_pod_container_port_name]
+                regex: metrics
+                action: keep
+              - source_labels: [__meta_kubernetes_namespace]
+                target_label: namespace
+              - source_labels: [__meta_kubernetes_pod_name]
+                target_label: pod
   processors:
     batch: {}
     memory_limiter:

dashboards/base/alerting/agent-operator.yaml

@@ -1,8 +1,14 @@
 # eks-agent-platform operator — Grafana-managed SLO / health alert rules. The
 # latency SLO is "99% of reconciles complete in <1s over 30d" (budget 0.01); each
 # burn rule is a dual-window check encoded as a `> bool` product. Self-contained
-# over controller-runtime metrics, which reach AMP once the operator pod carries
-# the prometheus.io/scrape annotation (eks-agent-platform operator-prod-scrape).
+# over controller-runtime metrics, which reach AMP through the otel-agent's
+# `eks-agent-platform-operator` scrape job — an authenticated HTTPS scrape using
+# the agent's own ServiceAccount token. NOT the annotation path: the operator
+# serves /metrics behind controller-runtime's authn/authz filter while
+# metrics.secure is true (the chart default) and suppresses its own
+# prometheus.io/scrape annotations for exactly that reason, since an annotation
+# scrape is plaintext and unauthenticated. These rules read no data at all until
+# that job exists.
 #
 # This is the PROD path (Grafana-managed, evaluated by Amazon Managed Grafana
 # against AMP). The operator chart's own PrometheusRule

dashboards/base/kustomization.yaml

@@ -48,7 +48,9 @@ resources:
   - platform/agent-ops.yaml
   - platform/agent-founder.yaml
   # Operator reconcile RED + latency SLO/error-budget (controller-runtime metrics
-  # reach AMP via the operator pod's prometheus.io/scrape annotation).
+  # reach AMP via the otel-agent's authenticated eks-agent-platform-operator
+  # scrape job, not the annotation path — the operator's metrics endpoint is
+  # HTTPS + token-gated and carries no scrape annotations).
   - platform/agent-operator.yaml
   # Ops control-plane app (portal): API SLO/RED + tofu-run, River-job, watcher,
   # and pgxpool surfaces — self-contained PromQL over the portal_* metrics in AMP.