diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9716df771..66033bdf2 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -37,6 +37,9 @@ jobs: - name: Verify KPA RBAC and arguments run: ./keda/test/kpa-rendering.sh + - name: Verify label shard rendering + run: ./kedify-agent/test/sharding-rendering.sh + deploy-helm-3-x: name: Deploy to Kubernetes ${{ matrix.kubernetesVersion }} in '${{matrix.namespace}}' namespace (${{ (matrix.enableAzureWorkloadIdentity == true && 'With Azure Workload Identity') || 'Without Azure Workload Identity' }} | ${{ (matrix.enableCertManager == true && 'With cert-manager') || 'Without cert-manager' }}) needs: lint-helm-3-x diff --git a/.github/workflows/ci-kedify-agent.yml b/.github/workflows/ci-kedify-agent.yml index fdbeaf815..919a69eca 100644 --- a/.github/workflows/ci-kedify-agent.yml +++ b/.github/workflows/ci-kedify-agent.yml @@ -45,6 +45,9 @@ jobs: - name: Test KPA rendering run: kedify-agent/test/kpa-rendering.sh + - name: Test label shard rendering + run: kedify-agent/test/sharding-rendering.sh + - name: Test DaemonSet rendering run: kedify-agent/test/daemonset-rendering.sh diff --git a/keda/README.md b/keda/README.md index a66e9058e..23c9baee5 100644 --- a/keda/README.md +++ b/keda/README.md @@ -107,6 +107,8 @@ their default values. | `kedify.multitenant.authority` | string | `""` | SNI authority override for gRPC connection to the tenant's keda-operator | | `kedify.multitenant.configSecretName` | string | `"kedify-multitenancy-config"` | Name of the Secret to store the connection information for tenants to configure default keda-operator-metrics-apiserver to connect to the correct keda-operator in the tenant namespace | | `kedify.multitenant.mode` | string | `""` | Multitenant mode: "" (disabled), "default" or "tenant" "": multitenancy disabled, standard KEDA deployment "default": deploys keda-operator + keda-operator-metrics-apiserver (and optionally keda-admission-webhooks); the metrics apiserver is configured to route HPA metric requests to the tenant keda-operators "tenant": deploys only keda-operator. The shared singletons (metrics apiserver, admission webhooks, CRDs) and the operator's webhook/apiservice patching are turned off automatically in this mode, so you do not need to set metricsServer.enabled / webhooks.enabled / crds.install. Give each tenant a unique operator.name so its cluster-scoped RBAC and leader election do not collide with the default install or other tenants. serviceAccount.operator.name and certificates.secretName only need overriding when multiple tenants share a namespace. | +| `kedify.sharding.pool` | string | `""` | Shard pool name. Set with `kedify.sharding.id` only in tenant mode, with an explicit `watchNamespace` and KPA enabled as the default class. | +| `kedify.sharding.id` | string | `""` | Shard ID within the pool. The chart derives `kedify.io/shard-pool=,kedify.io/shard=` for the registration and operator watch selector. Match the paired KPA namespace and selector. | | `networkPolicy.cilium` | object | `{"operator":{"extraEgressRules":[]}}` | Allow use of extra egress rules for cilium network policies | | `networkPolicy.enabled` | bool | `false` | Enable network policies | | `networkPolicy.flavor` | string | `"cilium"` | Flavor of the network policies (cilium, kubernetes) | diff --git a/keda/templates/_kedify-helpers.tpl b/keda/templates/_kedify-helpers.tpl index fe7b08b6e..feea8d1b2 100644 --- a/keda/templates/_kedify-helpers.tpl +++ b/keda/templates/_kedify-helpers.tpl @@ -18,6 +18,44 @@ unique, non-colliding resource names. Unchanged in default / non-multitenant mod {{- end -}} {{- end -}} +{{/* A shard's selector is fixed by its pool and id, never supplied independently. */}} +{{- define "keda.shardWatchLabelSelector" -}} +{{- $sharding := default (dict) .Values.kedify.sharding -}} +{{- $pool := default "" $sharding.pool -}} +{{- $id := default "" $sharding.id -}} +{{- if ne (empty $pool) (empty $id) -}} + {{- fail "kedify.sharding.pool and kedify.sharding.id must be set together" -}} +{{- end -}} +{{- if $pool -}} + {{- if or (gt (len $pool) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $pool)) (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} + {{- fail "kedify.sharding.pool and id must be valid Kubernetes label values" -}} + {{- end -}} + {{- if ne .Values.kedify.multitenant.mode "tenant" -}} + {{- fail "kedify.sharding requires kedify.multitenant.mode=tenant" -}} + {{- end -}} + {{- if not (trim .Values.watchNamespace) -}} + {{- fail "kedify.sharding requires explicit, nonempty watchNamespace" -}} + {{- end -}} + {{- if not (and .Values.kedify.kpa.enabled (eq (include "keda.kedifyKpaDefaultClass" .) "kpa") .Values.kedify.kpa.deploymentName) -}} + {{- fail "kedify.sharding requires enabled KPA, defaultClass=kpa and exact KPA deploymentName" -}} + {{- end -}} + {{- range $key := list "watch-label-selector" "enable-kpa" -}} + {{- if hasKey (default (dict) $.Values.extraArgs.keda) $key -}} + {{- fail (printf "extraArgs.keda.%s cannot override shard configuration" $key) -}} + {{- end -}} + {{- end -}} + {{- range .Values.env -}} + {{- if has .name (list "WATCH_LABEL_SELECTOR" "WATCH_NAMESPACE") -}} + {{- fail "env cannot override the shard WATCH_LABEL_SELECTOR or WATCH_NAMESPACE" -}} + {{- end -}} + {{- if eq .name "KEDIFY_SCALINGGROUPS_ENABLED" -}} + {{- fail "env cannot override KEDIFY_SCALINGGROUPS_ENABLED for a shard" -}} + {{- end -}} + {{- end -}} +{{- printf "kedify.io/shard-pool=%s,kedify.io/shard=%s" $pool $id -}} +{{- end -}} +{{- end -}} + {{/* Effective KEDA operator ServiceAccount name: the configured serviceAccount.operator.name (falling back to serviceAccount.name), suffixed with the Helm release name in multitenant diff --git a/keda/templates/kedify-tenant-registration-configmap.yaml b/keda/templates/kedify-tenant-registration-configmap.yaml index cb90ba3ea..ff2a57446 100644 --- a/keda/templates/kedify-tenant-registration-configmap.yaml +++ b/keda/templates/kedify-tenant-registration-configmap.yaml @@ -1,4 +1,5 @@ {{- if and .Values.kedify.multitenant .Values.kedify.multitenant.mode }} +{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}} apiVersion: v1 kind: ConfigMap metadata: @@ -22,6 +23,11 @@ data: tlsSecretRef: {{ (include "keda.certificates.secretName" .) | quote }} isDefaultTenant: {{ eq .Values.kedify.multitenant.mode "default" | quote }} operatorDeploymentName: {{ (include "keda.operator.name" .) | quote }} + {{- if $shardSelector }} + shardPool: {{ .Values.kedify.sharding.pool | quote }} + shardId: {{ .Values.kedify.sharding.id | quote }} + watchLabelSelector: {{ $shardSelector | quote }} + {{- end }} {{- if .Values.kedify.kpa.enabled }} kpaDeploymentName: {{ required "kedify.kpa.deploymentName is required when KPA and multitenant mode are enabled" .Values.kedify.kpa.deploymentName | quote }} {{- end }} diff --git a/keda/templates/manager/deployment.yaml b/keda/templates/manager/deployment.yaml index be4861e74..07291e117 100644 --- a/keda/templates/manager/deployment.yaml +++ b/keda/templates/manager/deployment.yaml @@ -5,6 +5,7 @@ {{- $tokens := include "keda.serviceAccountTokens" . | fromJson -}} {{- $tokenAudiences := include "keda.serviceAccountTokenAudiences" . | fromJsonArray -}} {{- $kedifyKpaDefaultClass := include "keda.kedifyKpaDefaultClass" . -}} +{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}} {{- $globalFeatures := default (dict) .Values.global.features -}} {{- $globalMulticluster := default (dict) (index $globalFeatures "multicluster") -}} apiVersion: apps/v1 @@ -194,7 +195,7 @@ spec: - name: WATCH_NAMESPACE value: {{ .Values.watchNamespace | quote }} - name: WATCH_LABEL_SELECTOR - value: {{ .Values.watchLabelSelector | quote }} + value: {{ default .Values.watchLabelSelector $shardSelector | quote }} - name: WATCH_LABEL_SELECTOR_FOR_TRIGGERAUTH value: {{ .Values.watchLabelSelectorForTriggerauth | quote }} - name: POD_NAME diff --git a/keda/values.schema.json b/keda/values.schema.json index abd845017..98853f194 100644 --- a/keda/values.schema.json +++ b/keda/values.schema.json @@ -317,6 +317,9 @@ "Kedify": { "type": "object", "properties": { + "sharding": { + "$ref": "#/definitions/KedifySharding" + }, "kpa": { "$ref": "#/definitions/KedifyKPA" }, @@ -329,6 +332,21 @@ "multitenant" ] }, + "KedifySharding": { + "type": "object", + "properties": { + "pool": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + }, + "id": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + } + } + }, "KedifyKPA": { "type": "object", "properties": { diff --git a/keda/values.yaml b/keda/values.yaml index b996117f7..60e2ffd51 100644 --- a/keda/values.yaml +++ b/keda/values.yaml @@ -82,6 +82,11 @@ watchLabelSelector: "" watchLabelSelectorForTriggerauth: "" kedify: + # -- Optional KPA-only label shard identity. Both pool and id must be set. + # The Agent pool ConfigMap supplies namespace membership and tenantRef. + sharding: + pool: "" + id: "" kpa: # -- Enable KEDA generation and validation of KedifyPodAutoscalers plus required RBAC. A compatible KPA CRD and controller must be installed separately. Transition ScaledObjects back to HPA before disabling. enabled: false diff --git a/kedify-agent/README.md b/kedify-agent/README.md index 0d3554f15..73b8f686b 100644 --- a/kedify-agent/README.md +++ b/kedify-agent/README.md @@ -41,6 +41,7 @@ Kubernetes: `>=v1.23.0-0` | agent.features.distributedScaledJobsEnabled | bool | `false` | Deprecated: use multicluster.enabled. Enables only the DistributedScaledJob controller during the compatibility period. | | agent.features.scaleAdaptersEnabled | bool | `false` | Enable the ScaleAdapter controller that bridges HPA/KEDA to resources with an incomplete /scale subresource (e.g. Agones Fleet), or without one at all (spec.desiredReplicasPath). The agent additionally needs RBAC via agent.extraRbacRules: get + update on the target kinds' /scale subresource, or get, list, watch + update on the whole resource for targets adapted through replica field paths. | | agent.features.kedifyPodAutoscalerEnabled | bool | `false` | Enable resource metrics collection and Prometheus endpoint discovery for Kedify Pod Autoscaler (KPA). KPA can be installed separately; leave disabled when its CRD/controller is not present | +| agent.sharding.pools | list | `[]` | Label-shard pools rendered in the Agent namespace as `kedify-agent-sharding` with a `pools.yaml` entry. Each pool has a name, explicit disjoint namespaces, optional `Rendezvous` or `LoadAware` strategy and `keyLabel`, and shards with unique `id` and `tenantRef` (`namespace/release`). Requires multi-tenant KEDA and KPA features. Each tenantRef names a separately installed, selector-capable tenant KEDA/KPA pair; the bundled KEDA dependency remains the ordinary, unsharded default installation. Create pools before adding SO/SJ resources; the Agent owns the enrollment annotation. | | agent.multicluster.localCluster.enabled | bool | `false` | Register the KEDA cluster itself as a multi-cluster member. This grants the agent permissions to scale local Deployments and manage local Jobs. | | agent.multicluster.localCluster.name | string | `"multicluster-local"` | Member-cluster alias. This is also the name of the generated kubeconfig Secret. | diff --git a/kedify-agent/files/crds/kedify-configuration.yaml b/kedify-agent/files/crds/kedify-configuration.yaml index 218247ff1..c82b1a98e 100644 --- a/kedify-agent/files/crds/kedify-configuration.yaml +++ b/kedify-agent/files/crds/kedify-configuration.yaml @@ -517,10 +517,18 @@ spec: description: OperatorReady indicates whether the keda-operator deployment is ready type: boolean + shardId: + type: string + shardPool: + description: Shard identity and selector are observed from the + tenant registration. + type: string tlsCertReady: description: TLSCertReady indicates whether the TLS cert secret is found and populated type: boolean + watchLabelSelector: + type: string watchNamespace: description: WatchNamespace is the namespace KEDA watches for ScaledObjects/ScaledJobs diff --git a/kedify-agent/templates/agent-deployment.yaml b/kedify-agent/templates/agent-deployment.yaml index 15f43ef56..44cd01e0d 100644 --- a/kedify-agent/templates/agent-deployment.yaml +++ b/kedify-agent/templates/agent-deployment.yaml @@ -58,10 +58,17 @@ spec: - --zap-log-level={{ default .Values.agent.logLevel .Values.agent.logging.level }} - --zap-encoder={{ .Values.agent.logging.format }} - --zap-time-encoding={{ .Values.agent.logging.timeEncoding }} + {{- if .Values.agent.sharding.pools }} + - --sharding-configmap-name=kedify-agent-sharding + - --sharding-configmap-namespace={{ .Release.Namespace }} + {{- end }} {{- if .Values.agent.logging.stackTracesEnabled }} - --zap-stacktrace-level=error {{- end }} {{- range $key, $value := .Values.agent.extraArgs }} + {{- if and $.Values.agent.sharding.pools (has $key (list "sharding-configmap-name" "sharding-configmap-namespace")) }} + {{- fail "agent.extraArgs cannot override sharding ConfigMap references" }} + {{- end }} - "--{{ $key }}={{ $value }}" {{- end }} command: diff --git a/kedify-agent/templates/agent-rbac.yaml b/kedify-agent/templates/agent-rbac.yaml index b62073ed2..6c8c1525c 100644 --- a/kedify-agent/templates/agent-rbac.yaml +++ b/kedify-agent/templates/agent-rbac.yaml @@ -10,6 +10,7 @@ {{- $argoRolloutsEnabled := eq (include "kedify-agent.featureEnabled" (list . "argoRolloutsEnabled")) "true" -}} {{- $kedifyPodAutoscalerEnabled := eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true" -}} {{- $scaleAdaptersEnabled := eq (include "kedify-agent.featureEnabled" (list . "scaleAdaptersEnabled")) "true" -}} +{{- $shardingEnabled := not (empty .Values.agent.sharding.pools) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -68,6 +69,18 @@ metadata: name: kedify-manager-role namespace: {{ .Release.Namespace }} rules: +{{- if .Values.agent.sharding.pools }} +# Enrollment proof is an Agent-owned annotation on its Helm-managed pool ConfigMap. +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - kedify-agent-sharding + verbs: + - get + - patch +{{- end }} {{- if .Values.agent.rbac.selfUpdates }} # Kedify agent needs to be able to update its own container image @@ -411,7 +424,8 @@ rules: - get {{- end }} {{- end }} -{{- if .Values.agent.rbac.readMetrics }} +{{- if or .Values.agent.rbac.readMetrics (and $shardingEnabled (not $recommendationsForLabeledNamespaces)) }} +# Shard enrollment watches namespace identity independently of telemetry. - apiGroups: - "" resources: @@ -420,6 +434,8 @@ rules: - list - watch - get +{{- end }} +{{- if .Values.agent.rbac.readMetrics }} - apiGroups: - metrics.k8s.io resources: @@ -436,7 +452,8 @@ rules: - list - watch {{- end }} -{{- if .Values.agent.rbac.readDeploymentsClusterwide }} +{{- if or .Values.agent.rbac.readDeploymentsClusterwide $shardingEnabled }} +# Shard readiness validates the observed operator Deployments. - apiGroups: - apps resources: diff --git a/kedify-agent/templates/agent-sharding-configmap.yaml b/kedify-agent/templates/agent-sharding-configmap.yaml new file mode 100644 index 000000000..8d17fa2d1 --- /dev/null +++ b/kedify-agent/templates/agent-sharding-configmap.yaml @@ -0,0 +1,73 @@ +{{- $pools := .Values.agent.sharding.pools | default (list) -}} +{{- if $pools -}} +{{- if not (and (eq (include "kedify-agent.featureEnabled" (list . "multitenantKEDAEnabled")) "true") (eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true")) -}} + {{- fail "agent.sharding.pools requires multitenantKEDAEnabled and kedifyPodAutoscalerEnabled" -}} +{{- end -}} +{{- $poolNames := dict -}} +{{- $allNamespaces := dict -}} +{{- $allTenantRefs := dict -}} +{{- $rendered := list -}} +{{- range $pool := $pools -}} + {{- if not (kindIs "map" $pool) -}} + {{- fail "agent.sharding.pools entries must be objects" -}} + {{- end -}} + {{- $name := required "agent.sharding.pools[].name is required" $pool.name -}} + {{- if or (gt (len $name) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) -}} + {{- fail (printf "invalid shard pool name %q" $name) -}} + {{- end -}} + {{- if hasKey $poolNames $name -}} + {{- fail (printf "duplicate shard pool name %q" $name) -}} + {{- end -}} + {{- $_ := set $poolNames $name true -}} + {{- $strategy := default "Rendezvous" $pool.strategy -}} + {{- if not (has $strategy (list "Rendezvous" "LoadAware")) -}} + {{- fail (printf "pool %q strategy must be Rendezvous or LoadAware" $name) -}} + {{- end -}} + {{- $namespaces := required (printf "pool %q needs explicit namespaces" $name) $pool.namespaces -}} + {{- if not (kindIs "slice" $namespaces) -}} + {{- fail (printf "pool %q namespaces must be a list" $name) -}} + {{- end -}} + {{- range $namespace := $namespaces -}} + {{- if or (not (kindIs "string" $namespace)) (gt (len $namespace) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $namespace)) -}} + {{- fail (printf "pool %q has an invalid namespace" $name) -}} + {{- end -}} + {{- if hasKey $allNamespaces $namespace -}} + {{- fail (printf "namespace %q belongs to more than one shard pool" $namespace) -}} + {{- end -}} + {{- $_ := set $allNamespaces $namespace true -}} + {{- end -}} + {{- $shards := required (printf "pool %q needs shards" $name) $pool.shards -}} + {{- if not (kindIs "slice" $shards) -}} + {{- fail (printf "pool %q shards must be a list" $name) -}} + {{- end -}} + {{- $ids := dict -}} + {{- $renderedShards := list -}} + {{- range $shard := $shards -}} + {{- $id := required (printf "pool %q shard id is required" $name) $shard.id -}} + {{- $ref := required (printf "pool %q shard tenantRef is required" $name) $shard.tenantRef -}} + {{- if or (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} + {{- fail (printf "pool %q has invalid shard id %q" $name $id) -}} + {{- end -}} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?/[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $ref) -}} + {{- fail (printf "pool %q has invalid tenantRef %q" $name $ref) -}} + {{- end -}} + {{- if or (hasKey $ids $id) (hasKey $allTenantRefs $ref) -}} + {{- fail (printf "pool %q has duplicate shard id or tenantRef" $name) -}} + {{- end -}} + {{- $_ := set $ids $id true -}} + {{- $_ := set $allTenantRefs $ref true -}} + {{- $renderedShards = append $renderedShards (dict "id" $id "tenantRef" $ref) -}} + {{- end -}} + {{- $rendered = append $rendered (dict "name" $name "namespaces" $namespaces "strategy" $strategy "keyLabel" (default "" $pool.keyLabel) "shards" $renderedShards) -}} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kedify-agent-sharding + namespace: {{ .Release.Namespace }} + labels: + {{- include "kedify-agent.labels" . | nindent 4 }} +data: + pools.yaml: | + {{- toYaml (dict "pools" $rendered) | nindent 4 }} +{{- end -}} diff --git a/kedify-agent/test/sharding-rendering.sh b/kedify-agent/test/sharding-rendering.sh new file mode 100755 index 000000000..2821515d3 --- /dev/null +++ b/kedify-agent/test/sharding-rendering.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +test_dir="$(mktemp -d)" +trap 'rm -rf "${test_dir}"' EXIT + +# The Agent umbrella has remote dependencies. Render its own templates in a +# temporary copy so this focused test is independent of chart repositories. +cp -R "${repo_dir}/kedify-agent" "${test_dir}/agent" +rm -rf "${test_dir}/agent/charts" +sed -i.bak '/^dependencies:/,$d' "${test_dir}/agent/Chart.yaml" +rm "${test_dir}/agent/Chart.yaml.bak" + +helm template test "${test_dir}/agent" --namespace keda \ + --set agent.createApiKeySecret=false >"${test_dir}/default-agent.yaml" +if grep -qF 'name: kedify-agent-sharding' "${test_dir}/default-agent.yaml" || \ + grep -qF -- '--sharding-configmap-name=' "${test_dir}/default-agent.yaml"; then + echo 'Default Agent installation must remain unsharded' >&2 + exit 1 +fi + +cat >"${test_dir}/agent-values.yaml" <<'EOF' +agent: + createApiKeySecret: false + sharding: + pools: + - name: applications + namespaces: [app-a, app-b] + shards: + - id: s0 + tenantRef: operators/keda-s0 + - id: s1 + tenantRef: operators/keda-s1 +global: + features: + multitenantKEDAEnabled: true + kedifyPodAutoscalerEnabled: true +EOF + +helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" >"${test_dir}/agent-render.yaml" +for field in shardPool shardId watchLabelSelector; do + if ! grep -q -- "^ ${field}:$" "${repo_dir}/kedify-agent/files/crds/kedify-configuration.yaml"; then + echo "The Helm-installed KedifyConfiguration CRD must preserve status.discoveredTenants[].${field}" >&2 + exit 1 + fi +done +for expected in 'name: kedify-agent-sharding' 'pools.yaml: |' 'name: applications' \ + 'strategy: Rendezvous' 'tenantRef: operators/keda-s0' \ + '--sharding-configmap-name=kedify-agent-sharding' \ + '--sharding-configmap-namespace=keda'; do + grep -qF -- "${expected}" "${test_dir}/agent-render.yaml" +done +if grep -qF 'kedify.io/shard-enrollment' "${test_dir}/agent-render.yaml"; then + echo 'The Agent, not Helm, owns the enrollment annotation' >&2 + exit 1 +fi + +helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set agent.rbac.readMetrics=false \ + --set agent.rbac.readDeploymentsClusterwide=false \ + --set agent.rbac.selfUpdates=false \ + --set agent.features.recommendationsForLabeledNamespaces=false \ + --set global.features.recommendationsForLabeledNamespaces=false \ + --show-only templates/agent-rbac.yaml >"${test_dir}/shard-rbac.yaml" +for resource in namespaces deployments; do + if ! awk -v target="${resource}" ' + function check_rule() { + if (resource && get && list && watch && !scoped) found = 1 + } + /^- apiGroups:/ { check_rule(); resource = get = list = watch = scoped = 0 } + $0 == " - " target { resource = 1 } + $0 == " - get" { get = 1 } + $0 == " - list" { list = 1 } + $0 == " - watch" { watch = 1 } + /^ resourceNames:/ { scoped = 1 } + END { check_rule(); exit !found } + ' "${test_dir}/shard-rbac.yaml"; then + echo "Shard RBAC needs unscoped get/list/watch on ${resource} without telemetry reads" >&2 + exit 1 + fi +done + +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set global.features.kedifyPodAutoscalerEnabled=false >/dev/null 2>&1; then + echo 'Agent pools require KPA feature enablement' >&2 + exit 1 +fi +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set 'agent.sharding.pools[0].namespaces[1]=app-a' >/dev/null 2>&1; then + echo 'Pool namespaces must be unique' >&2 + exit 1 +fi +if helm template test "${test_dir}/agent" --namespace keda \ + -f "${test_dir}/agent-values.yaml" \ + --set agent.extraArgs.sharding-configmap-name=other >/dev/null 2>&1; then + echo 'Agent extraArgs must not override the rendered pool ConfigMap' >&2 + exit 1 +fi + +cat >"${test_dir}/keda-values.yaml" <<'EOF' +watchNamespace: app-a,app-b +kedify: + multitenant: + mode: tenant + kpa: + enabled: true + defaultClass: kpa + deploymentName: kpa-s0 + sharding: + pool: applications + id: s0 +EOF + +helm template shard "${repo_dir}/keda" --namespace operators \ + -f "${test_dir}/keda-values.yaml" \ + --show-only templates/kedify-tenant-registration-configmap.yaml \ + --show-only templates/manager/deployment.yaml >"${test_dir}/keda-render.yaml" +for expected in 'shardPool: "applications"' 'shardId: "s0"' \ + 'watchLabelSelector: "kedify.io/shard-pool=applications,kedify.io/shard=s0"' \ + 'name: WATCH_LABEL_SELECTOR' \ + 'value: "kedify.io/shard-pool=applications,kedify.io/shard=s0"'; do + grep -qF -- "${expected}" "${test_dir}/keda-render.yaml" +done +if [[ "$(grep -cE '^ *- name: WATCH_LABEL_SELECTOR$' "${test_dir}/keda-render.yaml")" != 1 ]] || \ + [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/keda-render.yaml")" != ' value: "kedify.io/shard-pool=applications,kedify.io/shard=s0"' ]]; then + echo 'Shard KEDA must have one exact native selector environment variable' >&2 + exit 1 +fi +if grep -qF -- '--watch-label-selector=' "${test_dir}/keda-render.yaml"; then + echo 'Shard KEDA must use WATCH_LABEL_SELECTOR without an unsupported selector flag' >&2 + exit 1 +fi + +for override in \ + '--set kedify.multitenant.mode=default' \ + '--set watchNamespace=' \ + '--set kedify.kpa.defaultClass=hpa' \ + '--set kedify.kpa.enabled=false' \ + '--set kedify.kpa.deploymentName=' \ + '--set extraArgs.keda.watch-label-selector=other' \ + '--set extraArgs.keda.enable-kpa=false' \ + '--set env[0].name=WATCH_LABEL_SELECTOR' \ + '--set env[0].name=KEDIFY_SCALINGGROUPS_ENABLED --set env[0].valueFrom.secretKeyRef.name=flags --set env[0].valueFrom.secretKeyRef.key=enabled' \ + '--set env[0].name=KEDIFY_SCALINGGROUPS_ENABLED --set env[0].value=true'; do + read -r -a args <<<"${override}" + if helm template shard "${repo_dir}/keda" --namespace operators \ + -f "${test_dir}/keda-values.yaml" "${args[@]}" >/dev/null 2>&1; then + echo "Unsafe shard setting was accepted: ${override}" >&2 + exit 1 + fi +done + +helm template default "${repo_dir}/keda" --namespace operators \ + --show-only templates/manager/deployment.yaml >"${test_dir}/default-keda.yaml" +if [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/default-keda.yaml")" != ' value: ""' ]]; then + echo 'Default KEDA selector must remain empty' >&2 + exit 1 +fi +helm template custom "${repo_dir}/keda" --namespace operators \ + --set 'watchLabelSelector=team=payments' \ + --show-only templates/manager/deployment.yaml >"${test_dir}/custom-keda.yaml" +if [[ "$(awk '/- name: WATCH_LABEL_SELECTOR$/{getline; print}' "${test_dir}/custom-keda.yaml")" != ' value: "team=payments"' ]]; then + echo 'Nonsharded KEDA must retain its user-configured native selector' >&2 + exit 1 +fi diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index 8ed0aa72f..b0ae040d7 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -28,6 +28,19 @@ global: kedifyPodAutoscalerEnabled: false agent: replicas: 1 + # -- Label-shard pools assigned by this Agent. Requires multitenant KEDA and KPA features. + # Each tenantRef names a separately installed, selector-capable tenant KEDA/KPA pair; + # the bundled KEDA dependency remains the ordinary, unsharded default installation. + # Pools are additions-only after enrollment; create them before adding SO/SJ resources. + sharding: + pools: [] + # - name: applications + # namespaces: [app-a, app-b] + # strategy: Rendezvous # or LoadAware + # keyLabel: "" + # shards: + # - id: s0 + # tenantRef: operators/keda-s0 # you should already have this orgId: "" # you should already have this