From ee9244e76eab499816d29148f561d9b2bfdd0367 Mon Sep 17 00:00:00 2001 From: stxkxs <139715017+stxkxs@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:42:25 -0700 Subject: [PATCH] Give tempo's S3 backend the endpoint its client requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tempo ApplicationSet injected a bucket and a region and no endpoint. Tempo reaches S3 through the minio-go client, which validates the endpoint before it looks at anything else and refuses an empty one: failed to create minio client: Endpoint: does not follow ip address or domain name standards. Tempo exits non-zero at startup, so the StatefulSet crashloops and the Application sits Progressing forever. On a fresh install that is the single thing holding up catalog convergence: forty-six Applications reached Healthy and this one did not, and the installer waited its full thirty minutes on it before giving up. loki is not the same story, which is why the two blocks look asymmetric and have to stay that way: its AWS client derives an endpoint from the region, so bucket plus region is a complete configuration there. Copying loki's shape to tempo is what produced a config that renders correctly and cannot start. WHY THE GATE MISSED IT The render test asserted the bucket and the region are injected under the annotation, and that nothing is injected without it. Both were true the whole time. It asked whether the manifest was well-formed, never whether the configuration it produced was complete enough for the process to run — the same distinction between a valid artifact and a working one that this catalog keeps meeting. It now asserts the endpoint, against the rendered value rather than against a literal, so a region change carries. It fails on the previous template with the rendered block printed, which is the only evidence it catches anything. --- applicationsets/addons-tempo.yaml | 14 ++++++++++ .../rendertest/loki_tempo_render_test.go | 26 +++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/applicationsets/addons-tempo.yaml b/applicationsets/addons-tempo.yaml index 329c7d6..7de78cc 100644 --- a/applicationsets/addons-tempo.yaml +++ b/applicationsets/addons-tempo.yaml @@ -51,6 +51,19 @@ spec: # association — no static keys. Without the annotation the block renders empty and the # base values' local backend stands. `index` yields "" for the missing key under # missingkey=error. + # + # `endpoint` is REQUIRED and is not the same story as loki's. Tempo reaches S3 through + # the minio-go client, which validates the endpoint before it ever looks at the region + # and refuses an empty one: + # + # failed to create minio client: Endpoint: does not follow ip address or + # domain name standards. + # + # Tempo then exits non-zero at startup, so the StatefulSet crashloops and the + # Application sits Progressing forever — it never reaches Healthy, and on a fresh + # install that is the one thing holding up convergence. loki does not need it because + # its AWS client derives the endpoint from the region, which is why the two blocks + # look asymmetric and have to stay that way. values: | {{- if index .metadata.annotations "observability/tempo-bucket" }} tempo: @@ -60,6 +73,7 @@ spec: s3: bucket: {{ index .metadata.annotations "observability/tempo-bucket" }} region: {{ index .metadata.labels "region" }} + endpoint: s3.{{ index .metadata.labels "region" }}.amazonaws.com {{- end }} valueFiles: - $values/addons/observability/tempo/values.yaml diff --git a/applicationsets/rendertest/loki_tempo_render_test.go b/applicationsets/rendertest/loki_tempo_render_test.go index 5278a33..f63380c 100644 --- a/applicationsets/rendertest/loki_tempo_render_test.go +++ b/applicationsets/rendertest/loki_tempo_render_test.go @@ -82,8 +82,9 @@ func TestTempoS3Storage(t *testing.T) { Trace struct { Backend string `yaml:"backend"` S3 struct { - Bucket string `yaml:"bucket"` - Region string `yaml:"region"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` } `yaml:"s3"` } `yaml:"trace"` } `yaml:"storage"` @@ -101,6 +102,27 @@ func TestTempoS3Storage(t *testing.T) { if want := labels["region"]; v.Tempo.Storage.Trace.S3.Region != want { t.Fatalf("tempo.storage.trace.s3.region = %q, want %q\n%s", v.Tempo.Storage.Trace.S3.Region, want, out) } + // The assertion this test was missing, and it is not symmetric with loki's. + // + // Tempo reaches S3 through the minio-go client, which validates the endpoint before + // it looks at anything else and refuses an empty one: + // + // failed to create minio client: Endpoint: does not follow ip address or + // domain name standards. + // + // Tempo then exits at startup, the StatefulSet crashloops, and the Application sits + // Progressing forever — on a fresh install that is the single thing that holds up + // catalog convergence, for thirty minutes, until the installer gives up. A bucket + // and a region rendered correctly the whole time, which is why asserting those was + // not enough: the manifest was well-formed and the config was incomplete. + // + // loki needs no endpoint because its AWS client derives one from the region. The two + // blocks are asymmetric on purpose. + if want := "s3." + labels["region"] + ".amazonaws.com"; v.Tempo.Storage.Trace.S3.Endpoint != want { + t.Fatalf("tempo.storage.trace.s3.endpoint = %q, want %q — tempo's minio client "+ + "rejects an empty endpoint and the pod crashloops\n%s", + v.Tempo.Storage.Trace.S3.Endpoint, want, out) + } }) t.Run("cluster without the annotation keeps the local backend", func(t *testing.T) {