From f404802057a54cee33a387db610474f29cd8b715 Mon Sep 17 00:00:00 2001 From: Jay Kroger Date: Sat, 22 Aug 2026 11:42:47 -0400 Subject: [PATCH 1/5] fix kubernetes host metrics pipeline --- .../agent/transforms/tag_stream_kubernetes_host_metrics.yml | 2 +- config/aggregator/sinks/elasticsearch_metrics.yml | 4 +++- config/aggregator/transforms/route_agent_streams.yml | 2 +- config/aggregator/transforms/route_parsed_streams.yml | 2 +- .../tag_elastic_data_stream_kubernetes_host_metrics.yml | 6 +++--- tests/agent/test_tag_stream_kubernetes_host_metrics.yml | 2 +- ...test_tag_elastic_data_stream_kubernetes_host_metrics.yml | 6 +++--- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/config/agent/transforms/tag_stream_kubernetes_host_metrics.yml b/config/agent/transforms/tag_stream_kubernetes_host_metrics.yml index 54449e7..33fd340 100644 --- a/config/agent/transforms/tag_stream_kubernetes_host_metrics.yml +++ b/config/agent/transforms/tag_stream_kubernetes_host_metrics.yml @@ -2,4 +2,4 @@ type: remap inputs: - kubernetes_host_metrics source: | - .@vector.stream = "metrics.kubernetes.host" + .tags."@vector.stream" = "metrics.kubernetes.host" diff --git a/config/aggregator/sinks/elasticsearch_metrics.yml b/config/aggregator/sinks/elasticsearch_metrics.yml index 1a61320..21aeb8f 100644 --- a/config/aggregator/sinks/elasticsearch_metrics.yml +++ b/config/aggregator/sinks/elasticsearch_metrics.yml @@ -12,6 +12,8 @@ auth: password: "$${ELASTIC_PASSWORD}" mode: data_stream data_stream: - auto_routing: true + type: "{{ .tags."data_stream.type" }}" + dataset: "{{ .tags."data_stream.dataset" }}" + namespace: "{{ .tags."data_stream.namespace" }}" bulk: action: create diff --git a/config/aggregator/transforms/route_agent_streams.yml b/config/aggregator/transforms/route_agent_streams.yml index cae2554..d997fe5 100644 --- a/config/aggregator/transforms/route_agent_streams.yml +++ b/config/aggregator/transforms/route_agent_streams.yml @@ -13,4 +13,4 @@ route: source: .@vector.stream == "logs.kubernetes.flow" kubernetes_host_metrics: type: vrl - source: .@vector.stream == "metrics.kubernetes.host" + source: .tags."@vector.stream" == "metrics.kubernetes.host" diff --git a/config/aggregator/transforms/route_parsed_streams.yml b/config/aggregator/transforms/route_parsed_streams.yml index 41d2f51..4dcbae4 100644 --- a/config/aggregator/transforms/route_parsed_streams.yml +++ b/config/aggregator/transforms/route_parsed_streams.yml @@ -14,7 +14,7 @@ route: source: .@vector.stream == "logs.kubernetes.flow" kubernetes_host_metrics: type: vrl - source: .@vector.stream == "metrics.kubernetes.host" + source: .tags."@vector.stream" == "metrics.kubernetes.host" unifi_logs: type: vrl source: .@vector.stream == "logs.unifi" diff --git a/config/aggregator/transforms/tag_elastic_data_stream_kubernetes_host_metrics.yml b/config/aggregator/transforms/tag_elastic_data_stream_kubernetes_host_metrics.yml index b0dcf93..678bb1c 100644 --- a/config/aggregator/transforms/tag_elastic_data_stream_kubernetes_host_metrics.yml +++ b/config/aggregator/transforms/tag_elastic_data_stream_kubernetes_host_metrics.yml @@ -2,6 +2,6 @@ type: remap inputs: - route_parsed_streams.kubernetes_host_metrics source: | - .data_stream.type = "metrics" - .data_stream.dataset = "kubernetes.host" - .data_stream.namespace = "default" + .tags."data_stream.type" = "metrics" + .tags."data_stream.dataset" = "kubernetes.host" + .tags."data_stream.namespace" = "default" diff --git a/tests/agent/test_tag_stream_kubernetes_host_metrics.yml b/tests/agent/test_tag_stream_kubernetes_host_metrics.yml index de55da8..88434b0 100644 --- a/tests/agent/test_tag_stream_kubernetes_host_metrics.yml +++ b/tests/agent/test_tag_stream_kubernetes_host_metrics.yml @@ -10,4 +10,4 @@ tests: conditions: - type: vrl source: | - assert_eq!(.@vector.stream, "metrics.kubernetes.host") + assert_eq!(.tags."@vector.stream", "metrics.kubernetes.host") diff --git a/tests/aggregator/test_tag_elastic_data_stream_kubernetes_host_metrics.yml b/tests/aggregator/test_tag_elastic_data_stream_kubernetes_host_metrics.yml index d85449a..b9cdc03 100644 --- a/tests/aggregator/test_tag_elastic_data_stream_kubernetes_host_metrics.yml +++ b/tests/aggregator/test_tag_elastic_data_stream_kubernetes_host_metrics.yml @@ -9,6 +9,6 @@ tests: conditions: - type: vrl source: | - assert_eq!(.data_stream.type, "metrics") - assert_eq!(.data_stream.dataset, "kubernetes.host") - assert_eq!(.data_stream.namespace, "default") + assert_eq!(.tags."data_stream.type", "metrics") + assert_eq!(.tags."data_stream.dataset", "kubernetes.host") + assert_eq!(.tags."data_stream.namespace", "default") From 935aea6affc699803f51e0c3fe26280f6f52c8c1 Mon Sep 17 00:00:00 2001 From: Jay Kroger Date: Sat, 22 Aug 2026 11:52:31 -0400 Subject: [PATCH 2/5] fix elasticsearch metrics sink dataset fields --- config/aggregator/sinks/elasticsearch_metrics.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/aggregator/sinks/elasticsearch_metrics.yml b/config/aggregator/sinks/elasticsearch_metrics.yml index 21aeb8f..582a77e 100644 --- a/config/aggregator/sinks/elasticsearch_metrics.yml +++ b/config/aggregator/sinks/elasticsearch_metrics.yml @@ -12,8 +12,8 @@ auth: password: "$${ELASTIC_PASSWORD}" mode: data_stream data_stream: - type: "{{ .tags."data_stream.type" }}" - dataset: "{{ .tags."data_stream.dataset" }}" - namespace: "{{ .tags."data_stream.namespace" }}" + type: '{{ .tags."data_stream.type" }}' + dataset: '{{ .tags."data_stream.dataset" }}' + namespace: '{{ .tags."data_stream.namespace" }}' bulk: action: create From 2904e7f48470f072a37b5c5c77932de0fdebfd81 Mon Sep 17 00:00:00 2001 From: Jay Kroger Date: Sat, 22 Aug 2026 12:00:46 -0400 Subject: [PATCH 3/5] ??? --- config/aggregator/sinks/elasticsearch_metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/aggregator/sinks/elasticsearch_metrics.yml b/config/aggregator/sinks/elasticsearch_metrics.yml index 582a77e..693483a 100644 --- a/config/aggregator/sinks/elasticsearch_metrics.yml +++ b/config/aggregator/sinks/elasticsearch_metrics.yml @@ -12,7 +12,7 @@ auth: password: "$${ELASTIC_PASSWORD}" mode: data_stream data_stream: - type: '{{ .tags."data_stream.type" }}' + type: 'metrics' dataset: '{{ .tags."data_stream.dataset" }}' namespace: '{{ .tags."data_stream.namespace" }}' bulk: From 6e67ce3b5b88ff96f8ad31515dbb8f192a4d45a8 Mon Sep 17 00:00:00 2001 From: Jay Kroger Date: Sat, 22 Aug 2026 12:09:14 -0400 Subject: [PATCH 4/5] alright man --- config/aggregator/sinks/elasticsearch_metrics.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/aggregator/sinks/elasticsearch_metrics.yml b/config/aggregator/sinks/elasticsearch_metrics.yml index 693483a..7ca6e71 100644 --- a/config/aggregator/sinks/elasticsearch_metrics.yml +++ b/config/aggregator/sinks/elasticsearch_metrics.yml @@ -11,8 +11,9 @@ auth: user: "$${ELASTIC_USERNAME}" password: "$${ELASTIC_PASSWORD}" mode: data_stream +dangerously_allow_unconfined_templated_resolution: true data_stream: - type: 'metrics' + type: '{{ .tags."data_stream.type" }}' dataset: '{{ .tags."data_stream.dataset" }}' namespace: '{{ .tags."data_stream.namespace" }}' bulk: From b98abee0558c832ab85c6df04e831605f8a162b9 Mon Sep 17 00:00:00 2001 From: Jay Kroger Date: Sat, 22 Aug 2026 12:10:13 -0400 Subject: [PATCH 5/5] fix field name in elastic sink --- config/aggregator/sinks/elasticsearch_metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/aggregator/sinks/elasticsearch_metrics.yml b/config/aggregator/sinks/elasticsearch_metrics.yml index 7ca6e71..2f53a46 100644 --- a/config/aggregator/sinks/elasticsearch_metrics.yml +++ b/config/aggregator/sinks/elasticsearch_metrics.yml @@ -11,7 +11,7 @@ auth: user: "$${ELASTIC_USERNAME}" password: "$${ELASTIC_PASSWORD}" mode: data_stream -dangerously_allow_unconfined_templated_resolution: true +dangerously_allow_unconfined_template_resolution: true data_stream: type: '{{ .tags."data_stream.type" }}' dataset: '{{ .tags."data_stream.dataset" }}'