Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci-core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ jobs:
- name: Verify KPA RBAC and arguments
run: ./keda/test/kpa-rendering.sh

- name: Verify label shard rendering
run: ./kedify-agent/test/sharding-rendering.sh

deploy-helm-3-x:
name: Deploy to Kubernetes ${{ matrix.kubernetesVersion }} in '${{matrix.namespace}}' namespace (${{ (matrix.enableAzureWorkloadIdentity == true && 'With Azure Workload Identity') || 'Without Azure Workload Identity' }} | ${{ (matrix.enableCertManager == true && 'With cert-manager') || 'Without cert-manager' }})
needs: lint-helm-3-x
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci-kedify-agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ jobs:
- name: Test KPA rendering
run: kedify-agent/test/kpa-rendering.sh

- name: Test label shard rendering
run: kedify-agent/test/sharding-rendering.sh

- name: Test DaemonSet rendering
run: kedify-agent/test/daemonset-rendering.sh

Expand Down
2 changes: 2 additions & 0 deletions keda/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ their default values.
| `kedify.multitenant.authority` | string | `""` | SNI authority override for gRPC connection to the tenant's keda-operator |
| `kedify.multitenant.configSecretName` | string | `"kedify-multitenancy-config"` | Name of the Secret to store the connection information for tenants to configure default keda-operator-metrics-apiserver to connect to the correct keda-operator in the tenant namespace |
| `kedify.multitenant.mode` | string | `""` | Multitenant mode: "" (disabled), "default" or "tenant" "": multitenancy disabled, standard KEDA deployment "default": deploys keda-operator + keda-operator-metrics-apiserver (and optionally keda-admission-webhooks); the metrics apiserver is configured to route HPA metric requests to the tenant keda-operators "tenant": deploys only keda-operator. The shared singletons (metrics apiserver, admission webhooks, CRDs) and the operator's webhook/apiservice patching are turned off automatically in this mode, so you do not need to set metricsServer.enabled / webhooks.enabled / crds.install. Give each tenant a unique operator.name so its cluster-scoped RBAC and leader election do not collide with the default install or other tenants. serviceAccount.operator.name and certificates.secretName only need overriding when multiple tenants share a namespace. |
| `kedify.sharding.pool` | string | `""` | Shard pool name. Set with `kedify.sharding.id` only in tenant mode, with an explicit `watchNamespace` and KPA enabled as the default class. |
| `kedify.sharding.id` | string | `""` | Shard ID within the pool. The chart derives `kedify.io/shard-pool=<pool>,kedify.io/shard=<id>` for the registration and operator watch selector. Match the paired KPA namespace and selector. |
| `networkPolicy.cilium` | object | `{"operator":{"extraEgressRules":[]}}` | Allow use of extra egress rules for cilium network policies |
| `networkPolicy.enabled` | bool | `false` | Enable network policies |
| `networkPolicy.flavor` | string | `"cilium"` | Flavor of the network policies (cilium, kubernetes) |
Expand Down
38 changes: 38 additions & 0 deletions keda/templates/_kedify-helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,44 @@ unique, non-colliding resource names. Unchanged in default / non-multitenant mod
{{- end -}}
{{- end -}}

{{/* A shard's selector is fixed by its pool and id, never supplied independently. */}}
{{- define "keda.shardWatchLabelSelector" -}}
{{- $sharding := default (dict) .Values.kedify.sharding -}}
{{- $pool := default "" $sharding.pool -}}
{{- $id := default "" $sharding.id -}}
{{- if ne (empty $pool) (empty $id) -}}
{{- fail "kedify.sharding.pool and kedify.sharding.id must be set together" -}}
{{- end -}}
{{- if $pool -}}
{{- if or (gt (len $pool) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $pool)) (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}}
{{- fail "kedify.sharding.pool and id must be valid Kubernetes label values" -}}
{{- end -}}
{{- if ne .Values.kedify.multitenant.mode "tenant" -}}
{{- fail "kedify.sharding requires kedify.multitenant.mode=tenant" -}}
{{- end -}}
{{- if not (trim .Values.watchNamespace) -}}
{{- fail "kedify.sharding requires explicit, nonempty watchNamespace" -}}
{{- end -}}
{{- if not (and .Values.kedify.kpa.enabled (eq (include "keda.kedifyKpaDefaultClass" .) "kpa") .Values.kedify.kpa.deploymentName) -}}
{{- fail "kedify.sharding requires enabled KPA, defaultClass=kpa and exact KPA deploymentName" -}}
{{- end -}}
{{- range $key := list "watch-label-selector" "enable-kpa" -}}
{{- if hasKey (default (dict) $.Values.extraArgs.keda) $key -}}
{{- fail (printf "extraArgs.keda.%s cannot override shard configuration" $key) -}}
{{- end -}}
{{- end -}}
{{- range .Values.env -}}
{{- if has .name (list "WATCH_LABEL_SELECTOR" "WATCH_NAMESPACE") -}}
{{- fail "env cannot override the shard WATCH_LABEL_SELECTOR or WATCH_NAMESPACE" -}}
{{- end -}}
{{- if eq .name "KEDIFY_SCALINGGROUPS_ENABLED" -}}
{{- fail "env cannot override KEDIFY_SCALINGGROUPS_ENABLED for a shard" -}}
{{- end -}}
{{- end -}}
{{- printf "kedify.io/shard-pool=%s,kedify.io/shard=%s" $pool $id -}}
{{- end -}}
{{- end -}}

{{/*
Effective KEDA operator ServiceAccount name: the configured serviceAccount.operator.name
(falling back to serviceAccount.name), suffixed with the Helm release name in multitenant
Expand Down
6 changes: 6 additions & 0 deletions keda/templates/kedify-tenant-registration-configmap.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{{- if and .Values.kedify.multitenant .Values.kedify.multitenant.mode }}
{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}}
apiVersion: v1
kind: ConfigMap
metadata:
Expand All @@ -22,6 +23,11 @@ data:
tlsSecretRef: {{ (include "keda.certificates.secretName" .) | quote }}
isDefaultTenant: {{ eq .Values.kedify.multitenant.mode "default" | quote }}
operatorDeploymentName: {{ (include "keda.operator.name" .) | quote }}
{{- if $shardSelector }}
shardPool: {{ .Values.kedify.sharding.pool | quote }}
shardId: {{ .Values.kedify.sharding.id | quote }}
watchLabelSelector: {{ $shardSelector | quote }}
{{- end }}
{{- if .Values.kedify.kpa.enabled }}
kpaDeploymentName: {{ required "kedify.kpa.deploymentName is required when KPA and multitenant mode are enabled" .Values.kedify.kpa.deploymentName | quote }}
{{- end }}
Expand Down
3 changes: 2 additions & 1 deletion keda/templates/manager/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
{{- $tokens := include "keda.serviceAccountTokens" . | fromJson -}}
{{- $tokenAudiences := include "keda.serviceAccountTokenAudiences" . | fromJsonArray -}}
{{- $kedifyKpaDefaultClass := include "keda.kedifyKpaDefaultClass" . -}}
{{- $shardSelector := include "keda.shardWatchLabelSelector" . -}}
{{- $globalFeatures := default (dict) .Values.global.features -}}
{{- $globalMulticluster := default (dict) (index $globalFeatures "multicluster") -}}
apiVersion: apps/v1
Expand Down Expand Up @@ -194,7 +195,7 @@ spec:
- name: WATCH_NAMESPACE
value: {{ .Values.watchNamespace | quote }}
- name: WATCH_LABEL_SELECTOR
value: {{ .Values.watchLabelSelector | quote }}
value: {{ default .Values.watchLabelSelector $shardSelector | quote }}
- name: WATCH_LABEL_SELECTOR_FOR_TRIGGERAUTH
value: {{ .Values.watchLabelSelectorForTriggerauth | quote }}
- name: POD_NAME
Expand Down
18 changes: 18 additions & 0 deletions keda/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,9 @@
"Kedify": {
"type": "object",
"properties": {
"sharding": {
"$ref": "#/definitions/KedifySharding"
},
"kpa": {
"$ref": "#/definitions/KedifyKPA"
},
Expand All @@ -329,6 +332,21 @@
"multitenant"
]
},
"KedifySharding": {
"type": "object",
"properties": {
"pool": {
"type": "string",
"maxLength": 63,
"pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
},
"id": {
"type": "string",
"maxLength": 63,
"pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
}
}
},
"KedifyKPA": {
"type": "object",
"properties": {
Expand Down
5 changes: 5 additions & 0 deletions keda/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ watchLabelSelector: ""
watchLabelSelectorForTriggerauth: ""

kedify:
# -- Optional KPA-only label shard identity. Both pool and id must be set.
# The Agent pool ConfigMap supplies namespace membership and tenantRef.
sharding:
pool: ""
id: ""
kpa:
# -- Enable KEDA generation and validation of KedifyPodAutoscalers plus required RBAC. A compatible KPA CRD and controller must be installed separately. Transition ScaledObjects back to HPA before disabling.
enabled: false
Expand Down
1 change: 1 addition & 0 deletions kedify-agent/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ Kubernetes: `>=v1.23.0-0`
| agent.features.distributedScaledJobsEnabled | bool | `false` | Deprecated: use multicluster.enabled. Enables only the DistributedScaledJob controller during the compatibility period. |
| agent.features.scaleAdaptersEnabled | bool | `false` | Enable the ScaleAdapter controller that bridges HPA/KEDA to resources with an incomplete /scale subresource (e.g. Agones Fleet), or without one at all (spec.desiredReplicasPath). The agent additionally needs RBAC via agent.extraRbacRules: get + update on the target kinds' /scale subresource, or get, list, watch + update on the whole resource for targets adapted through replica field paths. |
| agent.features.kedifyPodAutoscalerEnabled | bool | `false` | Enable resource metrics collection and Prometheus endpoint discovery for Kedify Pod Autoscaler (KPA). KPA can be installed separately; leave disabled when its CRD/controller is not present |
| agent.sharding.pools | list | `[]` | Label-shard pools rendered in the Agent namespace as `kedify-agent-sharding` with a `pools.yaml` entry. Each pool has a name, explicit disjoint namespaces, optional `Rendezvous` or `LoadAware` strategy and `keyLabel`, and shards with unique `id` and `tenantRef` (`namespace/release`). Requires multi-tenant KEDA and KPA features. Each tenantRef names a separately installed, selector-capable tenant KEDA/KPA pair; the bundled KEDA dependency remains the ordinary, unsharded default installation. Create pools before adding SO/SJ resources; the Agent owns the enrollment annotation. |
| agent.multicluster.localCluster.enabled | bool | `false` | Register the KEDA cluster itself as a multi-cluster member. This grants the agent permissions to scale local Deployments and manage local Jobs. |
| agent.multicluster.localCluster.name | string | `"multicluster-local"` | Member-cluster alias. This is also the name of the generated kubeconfig Secret. |

Expand Down
8 changes: 8 additions & 0 deletions kedify-agent/files/crds/kedify-configuration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -517,10 +517,18 @@ spec:
description: OperatorReady indicates whether the keda-operator
deployment is ready
type: boolean
shardId:
type: string
shardPool:
description: Shard identity and selector are observed from the
tenant registration.
type: string
tlsCertReady:
description: TLSCertReady indicates whether the TLS cert secret
is found and populated
type: boolean
watchLabelSelector:
type: string
watchNamespace:
description: WatchNamespace is the namespace KEDA watches for
ScaledObjects/ScaledJobs
Expand Down
7 changes: 7 additions & 0 deletions kedify-agent/templates/agent-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,17 @@ spec:
- --zap-log-level={{ default .Values.agent.logLevel .Values.agent.logging.level }}
- --zap-encoder={{ .Values.agent.logging.format }}
- --zap-time-encoding={{ .Values.agent.logging.timeEncoding }}
{{- if .Values.agent.sharding.pools }}
- --sharding-configmap-name=kedify-agent-sharding
- --sharding-configmap-namespace={{ .Release.Namespace }}
{{- end }}
{{- if .Values.agent.logging.stackTracesEnabled }}
- --zap-stacktrace-level=error
{{- end }}
{{- range $key, $value := .Values.agent.extraArgs }}
{{- if and $.Values.agent.sharding.pools (has $key (list "sharding-configmap-name" "sharding-configmap-namespace")) }}
{{- fail "agent.extraArgs cannot override sharding ConfigMap references" }}
{{- end }}
- "--{{ $key }}={{ $value }}"
{{- end }}
command:
Expand Down
21 changes: 19 additions & 2 deletions kedify-agent/templates/agent-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
{{- $argoRolloutsEnabled := eq (include "kedify-agent.featureEnabled" (list . "argoRolloutsEnabled")) "true" -}}
{{- $kedifyPodAutoscalerEnabled := eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true" -}}
{{- $scaleAdaptersEnabled := eq (include "kedify-agent.featureEnabled" (list . "scaleAdaptersEnabled")) "true" -}}
{{- $shardingEnabled := not (empty .Values.agent.sharding.pools) -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
Expand Down Expand Up @@ -68,6 +69,18 @@ metadata:
name: kedify-manager-role
namespace: {{ .Release.Namespace }}
rules:
{{- if .Values.agent.sharding.pools }}
# Enrollment proof is an Agent-owned annotation on its Helm-managed pool ConfigMap.
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
- kedify-agent-sharding
verbs:
- get
- patch
{{- end }}

{{- if .Values.agent.rbac.selfUpdates }}
# Kedify agent needs to be able to update its own container image
Expand Down Expand Up @@ -411,7 +424,8 @@ rules:
- get
{{- end }}
{{- end }}
{{- if .Values.agent.rbac.readMetrics }}
{{- if or .Values.agent.rbac.readMetrics (and $shardingEnabled (not $recommendationsForLabeledNamespaces)) }}
# Shard enrollment watches namespace identity independently of telemetry.
- apiGroups:
- ""
resources:
Expand All @@ -420,6 +434,8 @@ rules:
- list
- watch
- get
{{- end }}
{{- if .Values.agent.rbac.readMetrics }}
- apiGroups:
- metrics.k8s.io
resources:
Expand All @@ -436,7 +452,8 @@ rules:
- list
- watch
{{- end }}
{{- if .Values.agent.rbac.readDeploymentsClusterwide }}
{{- if or .Values.agent.rbac.readDeploymentsClusterwide $shardingEnabled }}
# Shard readiness validates the observed operator Deployments.
- apiGroups:
- apps
resources:
Expand Down
73 changes: 73 additions & 0 deletions kedify-agent/templates/agent-sharding-configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{{- $pools := .Values.agent.sharding.pools | default (list) -}}
{{- if $pools -}}
{{- if not (and (eq (include "kedify-agent.featureEnabled" (list . "multitenantKEDAEnabled")) "true") (eq (include "kedify-agent.featureEnabled" (list . "kedifyPodAutoscalerEnabled")) "true")) -}}
{{- fail "agent.sharding.pools requires multitenantKEDAEnabled and kedifyPodAutoscalerEnabled" -}}
{{- end -}}
{{- $poolNames := dict -}}
{{- $allNamespaces := dict -}}
{{- $allTenantRefs := dict -}}
{{- $rendered := list -}}
{{- range $pool := $pools -}}
{{- if not (kindIs "map" $pool) -}}
{{- fail "agent.sharding.pools entries must be objects" -}}
{{- end -}}
{{- $name := required "agent.sharding.pools[].name is required" $pool.name -}}
{{- if or (gt (len $name) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) -}}
{{- fail (printf "invalid shard pool name %q" $name) -}}
{{- end -}}
{{- if hasKey $poolNames $name -}}
{{- fail (printf "duplicate shard pool name %q" $name) -}}
{{- end -}}
{{- $_ := set $poolNames $name true -}}
{{- $strategy := default "Rendezvous" $pool.strategy -}}
{{- if not (has $strategy (list "Rendezvous" "LoadAware")) -}}
{{- fail (printf "pool %q strategy must be Rendezvous or LoadAware" $name) -}}
{{- end -}}
{{- $namespaces := required (printf "pool %q needs explicit namespaces" $name) $pool.namespaces -}}
{{- if not (kindIs "slice" $namespaces) -}}
{{- fail (printf "pool %q namespaces must be a list" $name) -}}
{{- end -}}
{{- range $namespace := $namespaces -}}
{{- if or (not (kindIs "string" $namespace)) (gt (len $namespace) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $namespace)) -}}
{{- fail (printf "pool %q has an invalid namespace" $name) -}}
{{- end -}}
{{- if hasKey $allNamespaces $namespace -}}
{{- fail (printf "namespace %q belongs to more than one shard pool" $namespace) -}}
{{- end -}}
{{- $_ := set $allNamespaces $namespace true -}}
{{- end -}}
{{- $shards := required (printf "pool %q needs shards" $name) $pool.shards -}}
{{- if not (kindIs "slice" $shards) -}}
{{- fail (printf "pool %q shards must be a list" $name) -}}
{{- end -}}
{{- $ids := dict -}}
{{- $renderedShards := list -}}
{{- range $shard := $shards -}}
{{- $id := required (printf "pool %q shard id is required" $name) $shard.id -}}
{{- $ref := required (printf "pool %q shard tenantRef is required" $name) $shard.tenantRef -}}
{{- if or (gt (len $id) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}}
{{- fail (printf "pool %q has invalid shard id %q" $name $id) -}}
{{- end -}}
{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?/[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $ref) -}}
{{- fail (printf "pool %q has invalid tenantRef %q" $name $ref) -}}
{{- end -}}
{{- if or (hasKey $ids $id) (hasKey $allTenantRefs $ref) -}}
{{- fail (printf "pool %q has duplicate shard id or tenantRef" $name) -}}
{{- end -}}
{{- $_ := set $ids $id true -}}
{{- $_ := set $allTenantRefs $ref true -}}
{{- $renderedShards = append $renderedShards (dict "id" $id "tenantRef" $ref) -}}
{{- end -}}
{{- $rendered = append $rendered (dict "name" $name "namespaces" $namespaces "strategy" $strategy "keyLabel" (default "" $pool.keyLabel) "shards" $renderedShards) -}}
{{- end -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: kedify-agent-sharding
namespace: {{ .Release.Namespace }}
labels:
{{- include "kedify-agent.labels" . | nindent 4 }}
data:
pools.yaml: |
{{- toYaml (dict "pools" $rendered) | nindent 4 }}
{{- end -}}
Loading
Loading