From 37d0a336760e571b94115cad48bd3e91dda9682d Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Wed, 23 Sep 2026 23:26:25 +0200 Subject: [PATCH 1/8] Add label shard chart configuration refs #663 Signed-off-by: Zbynek Roubalik --- keda/README.md | 2 + keda/templates/_kedify-helpers.tpl | 38 ++++++ .../kedify-tenant-registration-configmap.yaml | 6 + keda/templates/manager/deployment.yaml | 6 +- keda/values.schema.json | 18 +++ keda/values.yaml | 5 + kedify-agent/README.md | 1 + kedify-agent/templates/agent-deployment.yaml | 7 + kedify-agent/templates/agent-rbac.yaml | 12 ++ .../templates/agent-sharding-configmap.yaml | 73 +++++++++++ kedify-agent/test/sharding-rendering.sh | 120 ++++++++++++++++++ kedify-agent/values.yaml | 11 ++ 12 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 kedify-agent/templates/agent-sharding-configmap.yaml create mode 100755 kedify-agent/test/sharding-rendering.sh diff --git a/keda/README.md b/keda/README.md index a66e9058e..23c9baee5 100644 --- a/keda/README.md +++ b/keda/README.md @@ -107,6 +107,8 @@ their default values. | `kedify.multitenant.authority` | string | `""` | SNI authority override for gRPC connection to the tenant's keda-operator | | `kedify.multitenant.configSecretName` | string | `"kedify-multitenancy-config"` | Name of the Secret to store the connection information for tenants to configure default keda-operator-metrics-apiserver to connect to the correct keda-operator in the tenant namespace | | `kedify.multitenant.mode` | string | `""` | Multitenant mode: "" (disabled), "default" or "tenant" "": multitenancy disabled, standard KEDA deployment "default": deploys keda-operator + keda-operator-metrics-apiserver (and optionally keda-admission-webhooks); the metrics apiserver is configured to route HPA metric requests to the tenant keda-operators "tenant": deploys only keda-operator. The shared singletons (metrics apiserver, admission webhooks, CRDs) and the operator's webhook/apiservice patching are turned off automatically in this mode, so you do not need to set metricsServer.enabled / webhooks.enabled / crds.install. Give each tenant a unique operator.name so its cluster-scoped RBAC and leader election do not collide with the default install or other tenants. serviceAccount.operator.name and certificates.secretName only need overriding when multiple tenants share a namespace. | +| `kedify.sharding.pool` | string | `""` | Shard pool name. Set with `kedify.sharding.id` only in tenant mode, with an explicit `watchNamespace` and KPA enabled as the default class. | +| `kedify.sharding.id` | string | `""` | Shard ID within the pool. The chart derives `kedify.io/shard-pool=,kedify.io/shard=` for the registration and operator watch selector. Match the paired KPA namespace and selector. | | `networkPolicy.cilium` | object | `{"operator":{"extraEgressRules":[]}}` | Allow use of extra egress rules for cilium network policies | | `networkPolicy.enabled` | bool | `false` | Enable network policies | | `networkPolicy.flavor` | string | `"cilium"` | Flavor of the network policies (cilium, kubernetes) | diff --git a/keda/templates/_kedify-helpers.tpl b/keda/templates/_kedify-helpers.tpl index fe7b08b6e..02569753c 100644 --- a/keda/templates/_kedify-helpers.tpl +++ b/keda/templates/_kedify-helpers.tpl @@ -18,6 +18,44 @@ unique, non-colliding resource names. Unchanged in default / non-multitenant mod {{- end -}} {{- end -}} +{{/* A shard's selector is fixed by its pool and id, never supplied independently. */}} +{{- define "keda.shardWatchLabelSelector" -}} +{{- $sharding := default (dict) .Values.kedify.sharding -}} +{{- $pool := default "" $sharding.pool -}} +{{- $id := default "" $sharding.id -}} +{{- if ne (empty $pool) (empty $id) -}} + {{- fail "kedify.sharding.pool and kedify.sharding.id must be set together" -}} +{{- end -}} +{{- if $pool -}} + {{- if or (gt (len $pool) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $pool)) (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} + {{- fail "kedify.sharding.pool and id must be valid Kubernetes label values" -}} + {{- end -}} + {{- if ne .Values.kedify.multitenant.mode "tenant" -}} + {{- fail "kedify.sharding requires kedify.multitenant.mode=tenant" -}} + {{- end -}} + {{- if not (trim .Values.watchNamespace) -}} + {{- fail "kedify.sharding requires explicit, nonempty watchNamespace" -}} + {{- end -}} + {{- if not (and .Values.kedify.kpa.enabled (eq (include "keda.kedifyKpaDefaultClass" .) "kpa") .Values.kedify.kpa.deploymentName) -}} + {{- fail "kedify.sharding requires enabled KPA, defaultClass=kpa and exact KPA deploymentName" -}} + {{- end -}} + {{- range $key := list "watch-label-selector" "enable-kpa" -}} + {{- if hasKey (default (dict) $.Values.extraArgs.keda) $key -}} + {{- fail (printf "extraArgs.keda.%s cannot override shard configuration" $key) -}} + {{- end -}} + {{- end -}} + {{- range .Values.env -}} + {{- if has .name (list "WATCH_LABEL_SELECTOR" "WATCH_NAMESPACE") -}} + {{- fail "env cannot override the shard WATCH_LABEL_SELECTOR or WATCH_NAMESPACE" -}} + {{- end -}} + {{- if and (eq .name "KEDIFY_SCALINGGROUPS_ENABLED") (eq (toString .value) "true") -}} + {{- fail "ScalingGroups must be disabled for a shard" -}} + {{- end -}} + {{- end -}} +{{- printf "kedify.io/shard-pool=%s,kedify.io/shard=%s" $pool $id -}} +{{- end -}} +{{- end -}} + {{/* Effective KEDA operator ServiceAccount name: the configured serviceAccount.operator.name (falling back to serviceAccount.name), suffixed with the Helm release name in multitenant diff --git a/keda/templates/kedify-tenant-registration-configmap.yaml b/keda/templates/kedify-tenant-registration-configmap.yaml index cb90ba3ea..ff2a57446 100644 --- a/keda/templates/kedify-tenant-registration-configmap.yaml +++ b/keda/templates/kedify-tenant-registration-configmap.yaml @@ -1,4 +1,5 @@ {{- if and .Values.kedify.multitenant .Values.kedify.multitenant.mode }} +{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}} apiVersion: v1 kind: ConfigMap metadata: @@ -22,6 +23,11 @@ data: tlsSecretRef: {{ (include "keda.certificates.secretName" .) | quote }} isDefaultTenant: {{ eq .Values.kedify.multitenant.mode "default" | quote }} operatorDeploymentName: {{ (include "keda.operator.name" .) | quote }} + {{- if $shardSelector }} + shardPool: {{ .Values.kedify.sharding.pool | quote }} + shardId: {{ .Values.kedify.sharding.id | quote }} + watchLabelSelector: {{ $shardSelector | quote }} + {{- end }} {{- if .Values.kedify.kpa.enabled }} kpaDeploymentName: {{ required "kedify.kpa.deploymentName is required when KPA and multitenant mode are enabled" .Values.kedify.kpa.deploymentName | quote }} {{- end }} diff --git a/keda/templates/manager/deployment.yaml b/keda/templates/manager/deployment.yaml index be4861e74..6f27c7322 100644 --- a/keda/templates/manager/deployment.yaml +++ b/keda/templates/manager/deployment.yaml @@ -5,6 +5,7 @@ {{- $tokens := include "keda.serviceAccountTokens" . | fromJson -}} {{- $tokenAudiences := include "keda.serviceAccountTokenAudiences" . | fromJsonArray -}} {{- $kedifyKpaDefaultClass := include "keda.kedifyKpaDefaultClass" . -}} +{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}} {{- $globalFeatures := default (dict) .Values.global.features -}} {{- $globalMulticluster := default (dict) (index $globalFeatures "multicluster") -}} apiVersion: apps/v1 @@ -118,6 +119,9 @@ spec: - "--k8s-cluster-name={{ tpl .Values.clusterName . }}" - "--k8s-cluster-domain={{ .Values.clusterDomain }}" - "--enable-prometheus-metrics={{ .Values.prometheus.operator.enabled }}" + {{- if $shardSelector }} + - "--watch-label-selector={{ $shardSelector }}" + {{- end }} {{- if .Values.kedify.kpa.enabled }} - --enable-kpa - --autoscaling-default-class={{ $kedifyKpaDefaultClass }} @@ -194,7 +198,7 @@ spec: - name: WATCH_NAMESPACE value: {{ .Values.watchNamespace | quote }} - name: WATCH_LABEL_SELECTOR - value: {{ .Values.watchLabelSelector | quote }} + value: {{ default .Values.watchLabelSelector $shardSelector | quote }} - name: WATCH_LABEL_SELECTOR_FOR_TRIGGERAUTH value: {{ .Values.watchLabelSelectorForTriggerauth | quote }} - name: POD_NAME diff --git a/keda/values.schema.json b/keda/values.schema.json index abd845017..98853f194 100644 --- a/keda/values.schema.json +++ b/keda/values.schema.json @@ -317,6 +317,9 @@ "Kedify": { "type": "object", "properties": { + "sharding": { + "$ref": "#/definitions/KedifySharding" + }, "kpa": { "$ref": "#/definitions/KedifyKPA" }, @@ -329,6 +332,21 @@ "multitenant" ] }, + "KedifySharding": { + "type": "object", + "properties": { + "pool": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + }, + "id": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + } + } + }, "KedifyKPA": { "type": "object", "properties": { diff --git a/keda/values.yaml b/keda/values.yaml index b996117f7..60e2ffd51 100644 --- a/keda/values.yaml +++ b/keda/values.yaml @@ -82,6 +82,11 @@ watchLabelSelector: "" watchLabelSelectorForTriggerauth: "" kedify: + # -- Optional KPA-only label shard identity. Both pool and id must be set. + # The Agent pool ConfigMap supplies namespace membership and tenantRef. + sharding: + pool: "" + id: "" kpa: # -- Enable KEDA generation and validation of KedifyPodAutoscalers plus required RBAC. A compatible KPA CRD and controller must be installed separately. Transition ScaledObjects back to HPA before disabling. enabled: false diff --git a/kedify-agent/README.md b/kedify-agent/README.md index 0d3554f15..04c055219 100644 --- a/kedify-agent/README.md +++ b/kedify-agent/README.md @@ -41,6 +41,7 @@ Kubernetes: `>=v1.23.0-0` | agent.features.distributedScaledJobsEnabled | bool | `false` | Deprecated: use multicluster.enabled. Enables only the DistributedScaledJob controller during the compatibility period. | | agent.features.scaleAdaptersEnabled | bool | `false` | Enable the ScaleAdapter controller that bridges HPA/KEDA to resources with an incomplete /scale subresource (e.g. Agones Fleet), or without one at all (spec.desiredReplicasPath). The agent additionally needs RBAC via agent.extraRbacRules: get + update on the target kinds' /scale subresource, or get, list, watch + update on the whole resource for targets adapted through replica field paths. | | agent.features.kedifyPodAutoscalerEnabled | bool | `false` | Enable resource metrics collection and Prometheus endpoint discovery for Kedify Pod Autoscaler (KPA). KPA can be installed separately; leave disabled when its CRD/controller is not present | +| agent.sharding.pools | list | `[]` | Label-shard pools rendered in the Agent namespace as `kedify-agent-sharding` with a `pools.yaml` entry. Each pool has a name, explicit disjoint namespaces, optional `Rendezvous` or `LoadAware` strategy and `keyLabel`, and shards with unique `id` and `tenantRef` (`namespace/release`). Requires multi-tenant KEDA and KPA features. Create pools before adding SO/SJ resources; the Agent owns the enrollment annotation. | | agent.multicluster.localCluster.enabled | bool | `false` | Register the KEDA cluster itself as a multi-cluster member. This grants the agent permissions to scale local Deployments and manage local Jobs. | | agent.multicluster.localCluster.name | string | `"multicluster-local"` | Member-cluster alias. This is also the name of the generated kubeconfig Secret. | diff --git a/kedify-agent/templates/agent-deployment.yaml b/kedify-agent/templates/agent-deployment.yaml index 15f43ef56..44cd01e0d 100644 --- a/kedify-agent/templates/agent-deployment.yaml +++ b/kedify-agent/templates/agent-deployment.yaml @@ -58,10 +58,17 @@ spec: - --zap-log-level={{ default .Values.agent.logLevel .Values.agent.logging.level }} - --zap-encoder={{ .Values.agent.logging.format }} - --zap-time-encoding={{ .Values.agent.logging.timeEncoding }} + {{- if .Values.agent.sharding.pools }} + - --sharding-configmap-name=kedify-agent-sharding + - --sharding-configmap-namespace={{ .Release.Namespace }} + {{- end }} {{- if .Values.agent.logging.stackTracesEnabled }} - --zap-stacktrace-level=error {{- end }} {{- range $key, $value := .Values.agent.extraArgs }} + {{- if and $.Values.agent.sharding.pools (has $key (list "sharding-configmap-name" "sharding-configmap-namespace")) }} + {{- fail "agent.extraArgs cannot override sharding ConfigMap references" }} + {{- end }} - "--{{ $key }}={{ $value }}" {{- end }} command: diff --git a/kedify-agent/templates/agent-rbac.yaml b/kedify-agent/templates/agent-rbac.yaml index b62073ed2..a2c936c1b 100644 --- a/kedify-agent/templates/agent-rbac.yaml +++ b/kedify-agent/templates/agent-rbac.yaml @@ -68,6 +68,18 @@ metadata: name: kedify-manager-role namespace: {{ .Release.Namespace }} rules: +{{- if .Values.agent.sharding.pools }} +# Enrollment proof is an Agent-owned annotation on its Helm-managed pool ConfigMap. +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - kedify-agent-sharding + verbs: + - get + - patch +{{- end }} {{- if .Values.agent.rbac.selfUpdates }} # Kedify agent needs to be able to update its own container image diff --git a/kedify-agent/templates/agent-sharding-configmap.yaml b/kedify-agent/templates/agent-sharding-configmap.yaml new file mode 100644 index 000000000..8d17fa2d1 --- /dev/null +++ b/kedify-agent/templates/agent-sharding-configmap.yaml @@ -0,0 +1,73 @@ +{{- $pools := .Values.agent.sharding.pools | default (list) -}} +{{- if $pools -}} +{{- if not (and (eq (include "kedify-agent.featureEnabled" (list . "multitenantKEDAEnabled")) "true") (eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true")) -}} + {{- fail "agent.sharding.pools requires multitenantKEDAEnabled and kedifyPodAutoscalerEnabled" -}} +{{- end -}} +{{- $poolNames := dict -}} +{{- $allNamespaces := dict -}} +{{- $allTenantRefs := dict -}} +{{- $rendered := list -}} +{{- range $pool := $pools -}} + {{- if not (kindIs "map" $pool) -}} + {{- fail "agent.sharding.pools entries must be objects" -}} + {{- end -}} + {{- $name := required "agent.sharding.pools[].name is required" $pool.name -}} + {{- if or (gt (len $name) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) -}} + {{- fail (printf "invalid shard pool name %q" $name) -}} + {{- end -}} + {{- if hasKey $poolNames $name -}} + {{- fail (printf "duplicate shard pool name %q" $name) -}} + {{- end -}} + {{- $_ := set $poolNames $name true -}} + {{- $strategy := default "Rendezvous" $pool.strategy -}} + {{- if not (has $strategy (list "Rendezvous" "LoadAware")) -}} + {{- fail (printf "pool %q strategy must be Rendezvous or LoadAware" $name) -}} + {{- end -}} + {{- $namespaces := required (printf "pool %q needs explicit namespaces" $name) $pool.namespaces -}} + {{- if not (kindIs "slice" $namespaces) -}} + {{- fail (printf "pool %q namespaces must be a list" $name) -}} + {{- end -}} + {{- range $namespace := $namespaces -}} + {{- if or (not (kindIs "string" $namespace)) (gt (len $namespace) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $namespace)) -}} + {{- fail (printf "pool %q has an invalid namespace" $name) -}} + {{- end -}} + {{- if hasKey $allNamespaces $namespace -}} + {{- fail (printf "namespace %q belongs to more than one shard pool" $namespace) -}} + {{- end -}} + {{- $_ := set $allNamespaces $namespace true -}} + {{- end -}} + {{- $shards := required (printf "pool %q needs shards" $name) $pool.shards -}} + {{- if not (kindIs "slice" $shards) -}} + {{- fail (printf "pool %q shards must be a list" $name) -}} + {{- end -}} + {{- $ids := dict -}} + {{- $renderedShards := list -}} + {{- range $shard := $shards -}} + {{- $id := required (printf "pool %q shard id is required" $name) $shard.id -}} + {{- $ref := required (printf "pool %q shard tenantRef is required" $name) $shard.tenantRef -}} + {{- if or (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} + {{- fail (printf "pool %q has invalid shard id %q" $name $id) -}} + {{- end -}} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?/[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $ref) -}} + {{- fail (printf "pool %q has invalid tenantRef %q" $name $ref) -}} + {{- end -}} + {{- if or (hasKey $ids $id) (hasKey $allTenantRefs $ref) -}} + {{- fail (printf "pool %q has duplicate shard id or tenantRef" $name) -}} + {{- end -}} + {{- $_ := set $ids $id true -}} + {{- $_ := set $allTenantRefs $ref true -}} + {{- $renderedShards = append $renderedShards (dict "id" $id "tenantRef" $ref) -}} + {{- end -}} + {{- $rendered = append $rendered (dict "name" $name "namespaces" $namespaces "strategy" $strategy "keyLabel" (default "" $pool.keyLabel) "shards" $renderedShards) -}} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kedify-agent-sharding + namespace: {{ .Release.Namespace }} + labels: + {{- include "kedify-agent.labels" . | nindent 4 }} +data: + pools.yaml: | + {{- toYaml (dict "pools" $rendered) | nindent 4 }} +{{- end -}} diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh new file mode 100755 index 000000000..6bc2dabb4 --- /dev/null +++ b/kedify-agent/test/sharding-rendering.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +test_dir="$(mktemp -d)" +trap 'rm -rf "${test_dir}"' EXIT + +# The Agent umbrella has remote dependencies. Render its own templates in a +# temporary copy so this focused test is independent of chart repositories. +cp -R "${repo_dir}/kedify-agent" "${test_dir}/agent" +sed -i.bak '/^dependencies:/,$d' "${test_dir}/agent/Chart.yaml" +rm "${test_dir}/agent/Chart.yaml.bak" + +helm template test "${test_dir}/agent" --namespace keda \ + --set agent.createApiKeySecret=false >"${test_dir}/default-agent.yaml" +if grep -qF 'name: kedify-agent-sharding' "${test_dir}/default-agent.yaml" || \ + grep -qF -- '--sharding-configmap-name=' "${test_dir}/default-agent.yaml"; then + echo 'Default Agent installation must remain unsharded' >&2 + exit 1 +fi + +cat >"${test_dir}/agent-values.yaml" <<'EOF' +agent: + createApiKeySecret: false + sharding: + pools: + - name: applications + namespaces: [app-a, app-b] + shards: + - id: s0 + tenantRef: operators/keda-s0 + - id: s1 + tenantRef: operators/keda-s1 +global: + features: + multitenantKEDAEnabled: true + kedifyPodAutoscalerEnabled: true +EOF + +helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" >"${test_dir}/agent-render.yaml" +for expected in 'name: kedify-agent-sharding' 'pools.yaml: |' 'name: applications' \ + 'strategy: Rendezvous' 'tenantRef: operators/keda-s0' \ + '--sharding-configmap-name=kedify-agent-sharding' \ + '--sharding-configmap-namespace=keda'; do + grep -qF -- "${expected}" "${test_dir}/agent-render.yaml" +done +if grep -qF 'kedify.io/sharding-enrollment' "${test_dir}/agent-render.yaml"; then + echo 'The Agent, not Helm, owns the enrollment annotation' >&2 + exit 1 +fi + +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set global.features.kedifyPodAutoscalerEnabled=false >/dev/null 2>&1; then + echo 'Agent pools require KPA feature enablement' >&2 + exit 1 +fi +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set 'agent.sharding.pools[0].namespaces[1]=app-a' >/dev/null 2>&1; then + echo 'Pool namespaces must be unique' >&2 + exit 1 +fi +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set agent.extraArgs.sharding-configmap-name=other >/dev/null 2>&1; then + echo 'Agent extraArgs must not override the rendered pool ConfigMap' >&2 + exit 1 +fi + +cat >"${test_dir}/keda-values.yaml" <<'EOF' +watchNamespace: app-a,app-b +kedify: + multitenant: + mode: tenant + kpa: + enabled: true + defaultClass: kpa + deploymentName: kpa-s0 + sharding: + pool: applications + id: s0 +EOF + +helm template shard "${repo_dir}/keda" --namespace operators \ + -f "${test_dir}/keda-values.yaml" \ + --show-only templates/kedify-tenant-registration-configmap.yaml \ + --show-only templates/manager/deployment.yaml >"${test_dir}/keda-render.yaml" +for expected in 'shardPool: "applications"' 'shardId: "s0"' \ + 'watchLabelSelector: "kedify.io/shard-pool=applications,kedify.io/shard=s0"' \ + '--watch-label-selector=kedify.io/shard-pool=applications,kedify.io/shard=s0' \ + 'name: WATCH_LABEL_SELECTOR'; do + grep -qF -- "${expected}" "${test_dir}/keda-render.yaml" +done + +for override in \ + '--set kedify.multitenant.mode=default' \ + '--set watchNamespace=' \ + '--set kedify.kpa.defaultClass=hpa' \ + '--set kedify.kpa.enabled=false' \ + '--set kedify.kpa.deploymentName=' \ + '--set extraArgs.keda.watch-label-selector=other' \ + '--set extraArgs.keda.enable-kpa=false' \ + '--set env[0].name=WATCH_LABEL_SELECTOR' \ + '--set env[0].name=KEDIFY_SCALINGGROUPS_ENABLED --set env[0].value=true'; do + read -r -a args <<<"${override}" + if helm template shard "${repo_dir}/keda" --namespace operators \ + -f "${test_dir}/keda-values.yaml" "${args[@]}" >/dev/null 2>&1; then + echo "Unsafe shard setting was accepted: ${override}" >&2 + exit 1 + fi +done + +helm template default "${repo_dir}/keda" --namespace operators \ + --show-only templates/manager/deployment.yaml >"${test_dir}/default-keda.yaml" +if grep -qF -- '--watch-label-selector=' "${test_dir}/default-keda.yaml"; then + echo 'Default KEDA must not receive a shard selector' >&2 + exit 1 +fi diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index 8ed0aa72f..ae92a1aed 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -28,6 +28,17 @@ global: kedifyPodAutoscalerEnabled: false agent: replicas: 1 + # -- Label-shard pools assigned by this Agent. Requires multitenant KEDA and KPA features. + # Pools are additions-only after enrollment; create them before adding SO/SJ resources. + sharding: + pools: [] + # - name: applications + # namespaces: [app-a, app-b] + # strategy: Rendezvous # or LoadAware + # keyLabel: "" + # shards: + # - id: s0 + # tenantRef: operators/keda-s0 # you should already have this orgId: "" # you should already have this From b8abfa174928fddfb8085c2e97941f9103fe9c40 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Wed, 23 Sep 2026 23:30:59 +0200 Subject: [PATCH 2/8] Grant shard controller namespace and deployment reads refs #663 Signed-off-by: Zbynek Roubalik --- kedify-agent/templates/agent-rbac.yaml | 9 +++++++-- kedify-agent/test/sharding-rendering.sh | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/kedify-agent/templates/agent-rbac.yaml b/kedify-agent/templates/agent-rbac.yaml index a2c936c1b..6c8c1525c 100644 --- a/kedify-agent/templates/agent-rbac.yaml +++ b/kedify-agent/templates/agent-rbac.yaml @@ -10,6 +10,7 @@ {{- $argoRolloutsEnabled := eq (include "kedify-agent.featureEnabled" (list . "argoRolloutsEnabled")) "true" -}} {{- $kedifyPodAutoscalerEnabled := eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true" -}} {{- $scaleAdaptersEnabled := eq (include "kedify-agent.featureEnabled" (list . "scaleAdaptersEnabled")) "true" -}} +{{- $shardingEnabled := not (empty .Values.agent.sharding.pools) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -423,7 +424,8 @@ rules: - get {{- end }} {{- end }} -{{- if .Values.agent.rbac.readMetrics }} +{{- if or .Values.agent.rbac.readMetrics (and $shardingEnabled (not $recommendationsForLabeledNamespaces)) }} +# Shard enrollment watches namespace identity independently of telemetry. - apiGroups: - "" resources: @@ -432,6 +434,8 @@ rules: - list - watch - get +{{- end }} +{{- if .Values.agent.rbac.readMetrics }} - apiGroups: - metrics.k8s.io resources: @@ -448,7 +452,8 @@ rules: - list - watch {{- end }} -{{- if .Values.agent.rbac.readDeploymentsClusterwide }} +{{- if or .Values.agent.rbac.readDeploymentsClusterwide $shardingEnabled }} +# Shard readiness validates the observed operator Deployments. - apiGroups: - apps resources: diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 6bc2dabb4..0b0570b61 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -50,6 +50,23 @@ if grep -qF 'kedify.io/sharding-enrollment' "${test_dir}/agent-render.yaml"; the exit 1 fi +helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set agent.rbac.readMetrics=false \ + --set agent.rbac.readDeploymentsClusterwide=false \ + --set agent.features.recommendationsForLabeledNamespaces=false \ + --set global.features.recommendationsForLabeledNamespaces=false \ + --show-only templates/agent-rbac.yaml >"${test_dir}/shard-rbac.yaml" +for resource in namespaces deployments; do + rule="$(grep -A6 -- "^ - ${resource}$" "${test_dir}/shard-rbac.yaml")" + for verb in get list watch; do + if ! grep -q -- "- ${verb}" <<<"${rule}"; then + echo "Shard RBAC needs ${verb} on ${resource} without telemetry reads" >&2 + exit 1 + fi + done +done + if helm template test "${test_dir}/agent" --namespace keda \ -f "${test_dir}/agent-values.yaml" \ --set global.features.kedifyPodAutoscalerEnabled=false >/dev/null 2>&1; then From 319ae78c44b224ad8b9964631ca2c6cd23db410b Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Wed, 23 Sep 2026 23:44:36 +0200 Subject: [PATCH 3/8] Preserve shard status in Helm-installed CRD refs #663 Signed-off-by: Zbynek Roubalik --- kedify-agent/files/crds/kedify-configuration.yaml | 8 ++++++++ kedify-agent/test/sharding-rendering.sh | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/kedify-agent/files/crds/kedify-configuration.yaml b/kedify-agent/files/crds/kedify-configuration.yaml index 218247ff1..c82b1a98e 100644 --- a/kedify-agent/files/crds/kedify-configuration.yaml +++ b/kedify-agent/files/crds/kedify-configuration.yaml @@ -517,10 +517,18 @@ spec: description: OperatorReady indicates whether the keda-operator deployment is ready type: boolean + shardId: + type: string + shardPool: + description: Shard identity and selector are observed from the + tenant registration. + type: string tlsCertReady: description: TLSCertReady indicates whether the TLS cert secret is found and populated type: boolean + watchLabelSelector: + type: string watchNamespace: description: WatchNamespace is the namespace KEDA watches for ScaledObjects/ScaledJobs diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 0b0570b61..036243c09 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -39,6 +39,12 @@ EOF helm template test "${test_dir}/agent" --namespace keda \ -f "${test_dir}/agent-values.yaml" >"${test_dir}/agent-render.yaml" +for field in shardPool shardId watchLabelSelector; do + if ! grep -q -- "^ ${field}:$" "${repo_dir}/kedify-agent/files/crds/kedify-configuration.yaml"; then + echo "The Helm-installed KedifyConfiguration CRD must preserve status.discoveredTenants[].${field}" >&2 + exit 1 + fi +done for expected in 'name: kedify-agent-sharding' 'pools.yaml: |' 'name: applications' \ 'strategy: Rendezvous' 'tenantRef: operators/keda-s0' \ '--sharding-configmap-name=kedify-agent-sharding' \ From 9deb8182094f5b673f3cae051f4124fa4bfaf3ab Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Wed, 23 Sep 2026 23:50:06 +0200 Subject: [PATCH 4/8] Run shard chart rendering checks in CI refs #663 Signed-off-by: Zbynek Roubalik --- .github/workflows/ci-core.yml | 3 +++ .github/workflows/ci-kedify-agent.yml | 3 +++ kedify-agent/test/sharding-rendering.sh | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9716df771..66033bdf2 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -37,6 +37,9 @@ jobs: - name: Verify KPA RBAC and arguments run: ./keda/test/kpa-rendering.sh + - name: Verify label shard rendering + run: ./kedify-agent/test/sharding-rendering.sh + deploy-helm-3-x: name: Deploy to Kubernetes ${{ matrix.kubernetesVersion }} in '${{matrix.namespace}}' namespace (${{ (matrix.enableAzureWorkloadIdentity == true && 'With Azure Workload Identity') || 'Without Azure Workload Identity' }} | ${{ (matrix.enableCertManager == true && 'With cert-manager') || 'Without cert-manager' }}) needs: lint-helm-3-x diff --git a/.github/workflows/ci-kedify-agent.yml b/.github/workflows/ci-kedify-agent.yml index fdbeaf815..919a69eca 100644 --- a/.github/workflows/ci-kedify-agent.yml +++ b/.github/workflows/ci-kedify-agent.yml @@ -45,6 +45,9 @@ jobs: - name: Test KPA rendering run: kedify-agent/test/kpa-rendering.sh + - name: Test label shard rendering + run: kedify-agent/test/sharding-rendering.sh + - name: Test DaemonSet rendering run: kedify-agent/test/daemonset-rendering.sh diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 036243c09..743d516c8 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -8,6 +8,7 @@ trap 'rm -rf "${test_dir}"' EXIT # The Agent umbrella has remote dependencies. Render its own templates in a # temporary copy so this focused test is independent of chart repositories. cp -R "${repo_dir}/kedify-agent" "${test_dir}/agent" +rm -rf "${test_dir}/agent/charts" sed -i.bak '/^dependencies:/,$d' "${test_dir}/agent/Chart.yaml" rm "${test_dir}/agent/Chart.yaml.bak" @@ -51,7 +52,7 @@ for expected in 'name: kedify-agent-sharding' 'pools.yaml: |' 'name: application '--sharding-configmap-namespace=keda'; do grep -qF -- "${expected}" "${test_dir}/agent-render.yaml" done -if grep -qF 'kedify.io/sharding-enrollment' "${test_dir}/agent-render.yaml"; then +if grep -qF 'kedify.io/shard-enrollment' "${test_dir}/agent-render.yaml"; then echo 'The Agent, not Helm, owns the enrollment annotation' >&2 exit 1 fi From 0084859f21155eb9e6cc60bc2e199e6476d02a24 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Thu, 24 Sep 2026 00:45:17 +0200 Subject: [PATCH 5/8] Harden shard chart validation and clarify release pairing refs #663 Signed-off-by: Zbynek Roubalik --- keda/templates/_kedify-helpers.tpl | 4 ++-- kedify-agent/README.md | 2 +- kedify-agent/test/sharding-rendering.sh | 24 +++++++++++++++++------- kedify-agent/values.yaml | 2 ++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/keda/templates/_kedify-helpers.tpl b/keda/templates/_kedify-helpers.tpl index 02569753c..feea8d1b2 100644 --- a/keda/templates/_kedify-helpers.tpl +++ b/keda/templates/_kedify-helpers.tpl @@ -48,8 +48,8 @@ unique, non-colliding resource names. Unchanged in default / non-multitenant mod {{- if has .name (list "WATCH_LABEL_SELECTOR" "WATCH_NAMESPACE") -}} {{- fail "env cannot override the shard WATCH_LABEL_SELECTOR or WATCH_NAMESPACE" -}} {{- end -}} - {{- if and (eq .name "KEDIFY_SCALINGGROUPS_ENABLED") (eq (toString .value) "true") -}} - {{- fail "ScalingGroups must be disabled for a shard" -}} + {{- if eq .name "KEDIFY_SCALINGGROUPS_ENABLED" -}} + {{- fail "env cannot override KEDIFY_SCALINGGROUPS_ENABLED for a shard" -}} {{- end -}} {{- end -}} {{- printf "kedify.io/shard-pool=%s,kedify.io/shard=%s" $pool $id -}} diff --git a/kedify-agent/README.md b/kedify-agent/README.md index 04c055219..73b8f686b 100644 --- a/kedify-agent/README.md +++ b/kedify-agent/README.md @@ -41,7 +41,7 @@ Kubernetes: `>=v1.23.0-0` | agent.features.distributedScaledJobsEnabled | bool | `false` | Deprecated: use multicluster.enabled. Enables only the DistributedScaledJob controller during the compatibility period. | | agent.features.scaleAdaptersEnabled | bool | `false` | Enable the ScaleAdapter controller that bridges HPA/KEDA to resources with an incomplete /scale subresource (e.g. Agones Fleet), or without one at all (spec.desiredReplicasPath). The agent additionally needs RBAC via agent.extraRbacRules: get + update on the target kinds' /scale subresource, or get, list, watch + update on the whole resource for targets adapted through replica field paths. | | agent.features.kedifyPodAutoscalerEnabled | bool | `false` | Enable resource metrics collection and Prometheus endpoint discovery for Kedify Pod Autoscaler (KPA). KPA can be installed separately; leave disabled when its CRD/controller is not present | -| agent.sharding.pools | list | `[]` | Label-shard pools rendered in the Agent namespace as `kedify-agent-sharding` with a `pools.yaml` entry. Each pool has a name, explicit disjoint namespaces, optional `Rendezvous` or `LoadAware` strategy and `keyLabel`, and shards with unique `id` and `tenantRef` (`namespace/release`). Requires multi-tenant KEDA and KPA features. Create pools before adding SO/SJ resources; the Agent owns the enrollment annotation. | +| agent.sharding.pools | list | `[]` | Label-shard pools rendered in the Agent namespace as `kedify-agent-sharding` with a `pools.yaml` entry. Each pool has a name, explicit disjoint namespaces, optional `Rendezvous` or `LoadAware` strategy and `keyLabel`, and shards with unique `id` and `tenantRef` (`namespace/release`). Requires multi-tenant KEDA and KPA features. Each tenantRef names a separately installed, selector-capable tenant KEDA/KPA pair; the bundled KEDA dependency remains the ordinary, unsharded default installation. Create pools before adding SO/SJ resources; the Agent owns the enrollment annotation. | | agent.multicluster.localCluster.enabled | bool | `false` | Register the KEDA cluster itself as a multi-cluster member. This grants the agent permissions to scale local Deployments and manage local Jobs. | | agent.multicluster.localCluster.name | string | `"multicluster-local"` | Member-cluster alias. This is also the name of the generated kubeconfig Secret. | diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 743d516c8..1984b7dd9 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -61,17 +61,26 @@ helm template test "${test_dir}/agent" --namespace keda \ -f "${test_dir}/agent-values.yaml" \ --set agent.rbac.readMetrics=false \ --set agent.rbac.readDeploymentsClusterwide=false \ + --set agent.rbac.selfUpdates=false \ --set agent.features.recommendationsForLabeledNamespaces=false \ --set global.features.recommendationsForLabeledNamespaces=false \ --show-only templates/agent-rbac.yaml >"${test_dir}/shard-rbac.yaml" for resource in namespaces deployments; do - rule="$(grep -A6 -- "^ - ${resource}$" "${test_dir}/shard-rbac.yaml")" - for verb in get list watch; do - if ! grep -q -- "- ${verb}" <<<"${rule}"; then - echo "Shard RBAC needs ${verb} on ${resource} without telemetry reads" >&2 - exit 1 - fi - done + if ! awk -v target="${resource}" ' + function check_rule() { + if (resource && get && list && watch && !scoped) found = 1 + } + /^- apiGroups:/ { check_rule(); resource = get = list = watch = scoped = 0 } + $0 == " - " target { resource = 1 } + $0 == " - get" { get = 1 } + $0 == " - list" { list = 1 } + $0 == " - watch" { watch = 1 } + /^ resourceNames:/ { scoped = 1 } + END { check_rule(); exit !found } + ' "${test_dir}/shard-rbac.yaml"; then + echo "Shard RBAC needs unscoped get/list/watch on ${resource} without telemetry reads" >&2 + exit 1 + fi done if helm template test "${test_dir}/agent" --namespace keda \ @@ -127,6 +136,7 @@ for override in \ '--set extraArgs.keda.watch-label-selector=other' \ '--set extraArgs.keda.enable-kpa=false' \ '--set env[0].name=WATCH_LABEL_SELECTOR' \ + '--set env[0].name=KEDIFY_SCALINGGROUPS_ENABLED --set env[0].valueFrom.secretKeyRef.name=flags --set env[0].valueFrom.secretKeyRef.key=enabled' \ '--set env[0].name=KEDIFY_SCALINGGROUPS_ENABLED --set env[0].value=true'; do read -r -a args <<<"${override}" if helm template shard "${repo_dir}/keda" --namespace operators \ diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index ae92a1aed..b0ae040d7 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -29,6 +29,8 @@ global: agent: replicas: 1 # -- Label-shard pools assigned by this Agent. Requires multitenant KEDA and KPA features. + # Each tenantRef names a separately installed, selector-capable tenant KEDA/KPA pair; + # the bundled KEDA dependency remains the ordinary, unsharded default installation. # Pools are additions-only after enrollment; create them before adding SO/SJ resources. sharding: pools: [] From 555af19858e0a399d868713cc21d9779dc913a4f Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Thu, 24 Sep 2026 22:14:36 +0200 Subject: [PATCH 6/8] Use native KEDA selector environment for shards refs #663 Signed-off-by: Zbynek Roubalik --- keda/templates/manager/deployment.yaml | 3 --- kedify-agent/test/sharding-rendering.sh | 10 +++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/keda/templates/manager/deployment.yaml b/keda/templates/manager/deployment.yaml index 6f27c7322..07291e117 100644 --- a/keda/templates/manager/deployment.yaml +++ b/keda/templates/manager/deployment.yaml @@ -119,9 +119,6 @@ spec: - "--k8s-cluster-name={{ tpl .Values.clusterName . }}" - "--k8s-cluster-domain={{ .Values.clusterDomain }}" - "--enable-prometheus-metrics={{ .Values.prometheus.operator.enabled }}" - {{- if $shardSelector }} - - "--watch-label-selector={{ $shardSelector }}" - {{- end }} {{- if .Values.kedify.kpa.enabled }} - --enable-kpa - --autoscaling-default-class={{ $kedifyKpaDefaultClass }} diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 1984b7dd9..9bd2578cc 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -122,10 +122,14 @@ helm template shard "${repo_dir}/keda" --namespace operators \ --show-only templates/manager/deployment.yaml >"${test_dir}/keda-render.yaml" for expected in 'shardPool: "applications"' 'shardId: "s0"' \ 'watchLabelSelector: "kedify.io/shard-pool=applications,kedify.io/shard=s0"' \ - '--watch-label-selector=kedify.io/shard-pool=applications,kedify.io/shard=s0' \ - 'name: WATCH_LABEL_SELECTOR'; do + 'name: WATCH_LABEL_SELECTOR' \ + 'value: "kedify.io/shard-pool=applications,kedify.io/shard=s0"'; do grep -qF -- "${expected}" "${test_dir}/keda-render.yaml" done +if grep -qF -- '--watch-label-selector=' "${test_dir}/keda-render.yaml"; then + echo 'Shard KEDA must use WATCH_LABEL_SELECTOR without an unsupported selector flag' >&2 + exit 1 +fi for override in \ '--set kedify.multitenant.mode=default' \ @@ -148,7 +152,7 @@ done helm template default "${repo_dir}/keda" --namespace operators \ --show-only templates/manager/deployment.yaml >"${test_dir}/default-keda.yaml" -if grep -qF -- '--watch-label-selector=' "${test_dir}/default-keda.yaml"; then +if grep -qF -- 'name: WATCH_LABEL_SELECTOR' "${test_dir}/default-keda.yaml"; then echo 'Default KEDA must not receive a shard selector' >&2 exit 1 fi From e3c53dfb539d919750d3f894bd13d8a4aa9e129a Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Thu, 24 Sep 2026 22:22:50 +0200 Subject: [PATCH 7/8] ci: retrigger shard chart checks refs #663 Signed-off-by: Zbynek Roubalik From addb38078bb2aaed1e656c8c9a6e16e8fc3b7a2d Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Thu, 24 Sep 2026 22:26:24 +0200 Subject: [PATCH 8/8] test: preserve native KEDA selector behavior after upstream sync refs #663 Signed-off-by: Zbynek Roubalik --- kedify-agent/test/sharding-rendering.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh index 9bd2578cc..2821515d3 100755 --- a/kedify-agent/test/sharding-rendering.sh +++ b/kedify-agent/test/sharding-rendering.sh @@ -126,6 +126,11 @@ for expected in 'shardPool: "applications"' 'shardId: "s0"' \ 'value: "kedify.io/shard-pool=applications,kedify.io/shard=s0"'; do grep -qF -- "${expected}" "${test_dir}/keda-render.yaml" done +if [[ "$(grep -cE '^ *- name: WATCH_LABEL_SELECTOR$' "${test_dir}/keda-render.yaml")" != 1 ]] || \ + [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/keda-render.yaml")" != ' value: "kedify.io/shard-pool=applications,kedify.io/shard=s0"' ]]; then + echo 'Shard KEDA must have one exact native selector environment variable' >&2 + exit 1 +fi if grep -qF -- '--watch-label-selector=' "${test_dir}/keda-render.yaml"; then echo 'Shard KEDA must use WATCH_LABEL_SELECTOR without an unsupported selector flag' >&2 exit 1 @@ -152,7 +157,14 @@ done helm template default "${repo_dir}/keda" --namespace operators \ --show-only templates/manager/deployment.yaml >"${test_dir}/default-keda.yaml" -if grep -qF -- 'name: WATCH_LABEL_SELECTOR' "${test_dir}/default-keda.yaml"; then - echo 'Default KEDA must not receive a shard selector' >&2 +if [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/default-keda.yaml")" != ' value: ""' ]]; then + echo 'Default KEDA selector must remain empty' >&2 + exit 1 +fi +helm template custom "${repo_dir}/keda" --namespace operators \ + --set 'watchLabelSelector=team=payments' \ + --show-only templates/manager/deployment.yaml >"${test_dir}/custom-keda.yaml" +if [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/custom-keda.yaml")" != ' value: "team=payments"' ]]; then + echo 'Nonsharded KEDA must retain its user-configured native selector' >&2 exit 1 fi