From 165e69dbb53467db7346cca49563a65031d84ab6 Mon Sep 17 00:00:00 2001 From: devex-sa Date: Mon, 10 Aug 2026 10:58:44 +0200 Subject: [PATCH 1/2] Update public.ecr.aws/karpenter/karpenter Docker tag to v1.14.0 --- apps/karpenter/sources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/karpenter/sources.yaml b/apps/karpenter/sources.yaml index fdf952e3..7d805329 100644 --- a/apps/karpenter/sources.yaml +++ b/apps/karpenter/sources.yaml @@ -7,4 +7,4 @@ spec: interval: 5m url: oci://public.ecr.aws/karpenter/karpenter ref: - tag: "1.6.3" \ No newline at end of file + tag: "1.14.0" \ No newline at end of file From 4fb81ba03f5059f519f2073a154b21a66d677ec5 Mon Sep 17 00:00:00 2001 From: devex-sa Date: Mon, 10 Aug 2026 08:59:21 +0000 Subject: [PATCH 2/2] Update CRDs and render helm templates --- apps/karpenter/crds.yaml | 784 ++++++++++++++++-- .../manifests/ClusterRole-karpenter-core.yml | 17 +- .../manifests/Deployment-karpenter.yml | 16 +- 3 files changed, 743 insertions(+), 74 deletions(-) diff --git a/apps/karpenter/crds.yaml b/apps/karpenter/crds.yaml index 3c5dfd23..29146127 100644 --- a/apps/karpenter/crds.yaml +++ b/apps/karpenter/crds.yaml @@ -3,7 +3,271 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.20.1 + name: capacitybuffers.autoscaling.x-k8s.io +spec: + group: autoscaling.x-k8s.io + names: + categories: + - autoscaling + kind: CapacityBuffer + listKind: CapacityBufferList + plural: capacitybuffers + shortNames: + - cb + singular: capacitybuffer + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The strategy to be used. + jsonPath: .spec.provisioningStrategy + name: Strategy + type: string + - description: The name of the PodTemplate used. + jsonPath: .status.podTemplateRef.name + name: PodTemplate + type: string + - description: The actual number of buffer chunks. + jsonPath: .status.replicas + name: Replicas + type: integer + - description: List of all condition types. + jsonPath: .status.conditions[*].type + name: ConditionsType + type: string + - description: List of all condition statuses. + jsonPath: .status.conditions[*].status + name: ConditionsStatus + type: string + - description: List of all condition reasons. + jsonPath: .status.conditions[*].reason + name: ConditionsReason + type: string + - description: The age of the CapacityBuffer. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + CapacityBuffer is the configuration that an autoscaler can use to provision buffer capacity within a cluster. + This buffer is represented by placeholder pods that trigger the Cluster Autoscaler to scale up nodes in advance, + ensuring that there is always spare capacity available to handle sudden workload spikes or to speed up scaling events. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired characteristics of the buffer. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + limits specifies resource constraints that limit the number of chunks created for this buffer + based on total resource requests (e.g., CPU, memory). If there are no other + limitations for the number of chunks (i.e., `replicas` or `percentage` are not set), + this will be used to create as many chunks as fit into these limits. + type: object + percentage: + description: |- + percentage defines the desired buffer capacity as a percentage of the + `scalableRef`'s current replicas. This is only applicable if `scalableRef` is set. + The absolute number of replicas is calculated from the percentage by rounding up to a minimum of 1. + For example, if `scalableRef` has 10 replicas and `percentage` is 20, 2 buffer chunks will be created. + format: int32 + minimum: 0 + type: integer + podTemplateRef: + description: |- + podTemplateRef is a reference to a PodTemplate resource in the same namespace + that declares the shape of a single chunk of the buffer. The pods created + from this template will be used as placeholder pods for the buffer capacity. + Exactly one of `podTemplateRef`, `scalableRef` should be specified. + properties: + name: + description: name of the referent. + minLength: 1 + type: string + required: + - name + type: object + provisioningStrategy: + default: buffer.x-k8s.io/active-capacity + description: |- + provisioningStrategy defines how the buffer is utilized. + "buffer.x-k8s.io/active-capacity" is the default strategy, where the buffer actively scales up the cluster by creating placeholder pods. + type: string + replicas: + description: |- + replicas defines the desired number of buffer chunks to provision. + If neither `replicas` nor `percentage` is set, as many chunks as fit within + defined resource limits (if any) will be created. If both are set, the minimum + of the two will be used. + format: int32 + minimum: 0 + type: integer + scalableRef: + description: |- + scalableRef is a reference to an object of a kind that has a scale subresource + and specifies its label selector field. This allows the CapacityBuffer to + manage the buffer by scaling an existing scalable resource. + Exactly one of `podTemplateRef`, `scalableRef` should be specified. + properties: + apiGroup: + description: |- + apiGroup is the API group of the referent. + Empty string for the core API group. + type: string + kind: + description: kind is the kind of the referent. + minLength: 1 + type: string + name: + description: name is the name of the referent. + minLength: 1 + type: string + required: + - kind + - name + type: object + type: object + x-kubernetes-validations: + - message: If podTemplateRef is set, replicas or limits must also be set + rule: '!has(self.podTemplateRef) || has(self.replicas) || has(self.limits)' + - message: You must define either PodTemplateRef or ScalableRef, but not + both + rule: '!(has(self.podTemplateRef) && has(self.scalableRef))' + status: + description: status represents the current state of the buffer and its + readiness for autoprovisioning. + properties: + conditions: + description: |- + conditions provide a standard mechanism for reporting the buffer's state. + The "Ready" condition indicates if the buffer is successfully provisioned + and active. Other conditions may report on various aspects of the buffer's + health and provisioning process. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + podTemplateGeneration: + description: |- + podTemplateGeneration is the observed generation of the PodTemplate, used + to determine if the status is up-to-date with the desired `spec.podTemplateRef`. + format: int64 + type: integer + podTemplateRef: + description: |- + podTemplateRef is the observed reference to the PodTemplate that was used + to provision the buffer. If this field is not set, and the `conditions` + indicate an error, it provides details about the error state. + properties: + name: + description: name of the referent. + minLength: 1 + type: string + required: + - name + type: object + provisioningStrategy: + description: provisioningStrategy defines how the buffer should be + utilized. + type: string + replicas: + description: replicas is the actual number of buffer chunks currently + provisioned. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws @@ -72,6 +336,7 @@ spec: - Custom - Windows2019 - Windows2022 + - Windows2025 type: string amiSelectorTerms: description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. @@ -84,7 +349,7 @@ spec: description: |- Alias specifies which EKS optimized AMI to select. Each alias consists of a family and an AMI version, specified as "family@version". - Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + Valid families include: al2, al2023, bottlerocket, windows2019, windows2022, windows2025. The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. Note: The Windows families do **not** support version pinning, and only latest may be used. @@ -93,10 +358,10 @@ spec: x-kubernetes-validations: - message: '''alias'' is improperly formatted, must match the format ''family@version''' rule: self.matches('^[a-zA-Z0-9]+@.+$') - - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' - rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022'', ''windows2025''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022','windows2025'] - message: windows families may only specify version 'latest' - rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022'',''windows2025''] ? self.split(''@'')[1] == ''latest'' : true' id: description: ID is the ami id in EC2 pattern: ami-[0-9a-z]+ @@ -262,6 +527,12 @@ spec: description: ID is the capacity reservation id in EC2 pattern: ^cr-[0-9a-z]+$ type: string + instanceMatchCriteria: + description: InstanceMatchCriteria specifies how instances are matched to capacity reservations. + enum: + - open + - targeted + type: string ownerID: description: Owner is the owner id for the ami. pattern: ^[0-9]{12}$ @@ -281,15 +552,70 @@ spec: maxItems: 30 type: array x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id'] - rule: self.all(x, has(x.tags) || has(x.id)) - - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + - message: expected at least one, got none, ['tags', 'id', 'instanceMatchCriteria'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.instanceMatchCriteria)) + - message: '''id'' is mutually exclusive, cannot be set along with other fields in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID) || has(x.instanceMatchCriteria)))' + connectionTracking: + description: |- + ConnectionTracking configures idle connection tracking timeouts for + ENIs Karpenter provisions in the launch template. EFA-only interfaces + are excluded. See ConnectionTracking. + properties: + tcpEstablishedTimeout: + description: |- + TCPEstablishedTimeout is the timeout (in seconds) for idle TCP connections + in an established state. + Value must be between 60 and 432,000 (5 days). + If unset, EC2 applies its default which is 350 seconds for Nitro v6 + instance types (excluding P6e-GB200) and 432,000 seconds for other + instance types. + format: int32 + maximum: 432000 + minimum: 60 + type: integer + udpStreamTimeout: + description: |- + UDPStreamTimeout is the timeout (in seconds) for idle UDP "stream" flows + that have seen more than one request-response transaction. + Value must be between 60 and 180. + If unset, EC2 applies its default of 180 seconds. + format: int32 + maximum: 180 + minimum: 60 + type: integer + udpTimeout: + description: |- + UDPTimeout is the timeout (in seconds) for idle UDP flows that have seen + traffic only in a single direction or a single request-response transaction. + Value must be between 30 and 60. + If unset, EC2 applies its default of 30 seconds. + format: int32 + maximum: 60 + minimum: 30 + type: integer + type: object + x-kubernetes-validations: + - message: at least one of tcpEstablishedTimeout, udpStreamTimeout, or udpTimeout must be set + rule: has(self.tcpEstablishedTimeout) || has(self.udpStreamTimeout) || has(self.udpTimeout) context: description: |- Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string + cpuOptions: + description: CPUOptions defines the CPU options for the instance. + properties: + nestedVirtualization: + description: |- + NestedVirtualization enables or disables nested virtualization on the instance. + When enabled, Karpenter filters instance types to only those reporting + "nested-virtualization" in ProcessorInfo.SupportedFeatures from DescribeInstanceTypes. + enum: + - enabled + - disabled + type: string + type: object detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched type: boolean @@ -308,6 +634,11 @@ spec: enum: - RAID0 type: string + ipPrefixCount: + description: IPPrefixCount sets the number of IPv4 prefixes to be automatically assigned to the network interface. + format: int32 + minimum: 0 + type: integer kubelet: description: |- Kubelet defines args to be used when configuring kubelet on provisioned nodes. @@ -501,19 +832,66 @@ spec: - optional type: string type: object + networkInterfaces: + description: NetworkInterfaces specifies the network interface configurations to be attached to provisioned instances. + items: + description: |- + NetworkInterface specifies the configuration for a network interface to be attached + to provisioned instances. + properties: + deviceIndex: + description: DeviceIndex is the device index for the network interface attachment. + format: int32 + minimum: 0 + type: integer + interfaceType: + description: InterfaceType is the type of network interface. Valid values are "interface" and "efa-only". + enum: + - interface + - efa-only + type: string + networkCardIndex: + description: NetworkCardIndex is the index of the network card to attach the interface to. + format: int32 + minimum: 0 + type: integer + required: + - deviceIndex + - interfaceType + - networkCardIndex + type: object + maxItems: 150 + type: array + x-kubernetes-validations: + - message: networkInterfaces must include a primary interface with interfaceType='interface' + rule: self.size() == 0 || self.exists(x, x.deviceIndex == 0 && x.networkCardIndex == 0 && x.interfaceType == 'interface') + - message: networkInterfaces must not have duplicate networkCardIndex and deviceIndex pairs, and can have at most one efa device per network card + rule: self.all(x, self.filter(y, x.networkCardIndex == y.networkCardIndex && x.deviceIndex == y.deviceIndex).size() == 1 && (x.interfaceType != 'efa-only' || self.filter(y, x.networkCardIndex == y.networkCardIndex && y.interfaceType == 'efa-only').size() == 1)) + placementGroupSelector: + description: PlacementGroupSelector defines the name or the id of the placement to resolve with the nodeclass. + properties: + id: + description: ID is the placement group id in EC2 + pattern: ^pg-[0-9a-z]+$ + type: string + name: + description: Name is the placement group name in EC2 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: expected at least one, got none, ['name', 'id'] + rule: has(self.name) || has(self.id) + - message: '''name'' and ''id'' are mutually exclusive' + rule: '!(has(self.name) && has(self.id))' role: description: |- - Role is the AWS identity that nodes use. This field is immutable. + Role is the AWS identity that nodes use. This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - for the old instance profiles on an update. type: string x-kubernetes-validations: - message: role cannot be empty rule: self != '' - - message: immutable field changed - rule: self == oldSelf securityGroupSelectorTerms: description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. items: @@ -617,8 +995,6 @@ spec: x-kubernetes-validations: - message: must specify exactly one of ['role', 'instanceProfile'] rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) - - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. - rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias @@ -629,6 +1005,8 @@ spec: rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: if set, amiFamily must be 'Windows2025' or 'Custom' when using a Windows2025 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2025'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2025'') : true)' - message: must specify amiFamily if amiSelectorTerms does not contain an alias rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' status: @@ -714,6 +1092,9 @@ spec: instanceType: description: The instance type for the capacity reservation. type: string + interruptible: + description: Indicates whether this capacity reservation is interruptible + type: boolean ownerID: description: The ID of the AWS account that owns the capacity reservation. pattern: ^[0-9]{12}$ @@ -851,7 +1232,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.20.1 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -968,7 +1349,7 @@ spec: description: Requirements are layered with GetLabels and applied to every node. items: description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + A node selector requirement is a selector that contains values, a key, an operator that relates the key and values and minValues that represent the requirement to have at least that many values. properties: key: @@ -977,16 +1358,12 @@ spec: maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - message: label domain "karpenter.sh" is restricted rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-tenancy", "karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/capacity-reservation-interruptible", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count", "karpenter.k8s.aws/instance-capability-flex", "karpenter.k8s.aws/placement-group-id", "karpenter.k8s.aws/placement-group-partition"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -997,20 +1374,22 @@ spec: operator: description: |- Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, Lt, Gte, and Lte. enum: + - Gte + - Lte - In - NotIn - Exists - DoesNotExist - Gt - Lt + type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values + the values array must be empty. If the operator is Gt, Lt, Gte, or Lte, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: @@ -1025,11 +1404,12 @@ spec: type: object maxItems: 100 type: array + x-kubernetes-list-type: atomic x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements operator 'Gt', 'Lt', 'Gte', or 'Lte' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'' || x.operator == ''Gte'' || x.operator == ''Lte'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - message: requirements with 'minValues' must have at least that many values specified in the 'values' field rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' resources: @@ -1047,7 +1427,7 @@ spec: type: object startupTaints: description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + startupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. @@ -1072,9 +1452,7 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the taint was added. format: date-time type: string value: @@ -1086,8 +1464,9 @@ spec: - key type: object type: array + x-kubernetes-list-type: atomic taints: - description: Taints will be applied to the NodeClaim's node. + description: taints will be applied to the NodeClaim's node. items: description: |- The node this Taint is attached to has the "effect" on @@ -1109,9 +1488,7 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the taint was added. format: date-time type: string value: @@ -1123,6 +1500,7 @@ spec: - key type: object type: array + x-kubernetes-list-type: atomic terminationGracePeriod: description: |- TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. @@ -1221,6 +1599,9 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map imageID: description: ImageID is an identifier for the image that runs on the node type: string @@ -1251,7 +1632,237 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.20.1 + name: nodeoverlays.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeOverlay + listKind: NodeOverlayList + plural: nodeoverlays + shortNames: + - overlays + singular: nodeoverlay + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Capacity adds extended resources only, and does not replace any existing resources. + These extended resources are appended to the node's existing resource list. + Note: This field does not modify or override standard resources like cpu, memory, ephemeral-storage, or pods. + type: object + x-kubernetes-validations: + - message: invalid resource restricted + rule: self.all(x, !(x in ['cpu', 'memory', 'ephemeral-storage', 'pods'])) + price: + description: Price specifies amount for an instance types that match the specified labels. Users can override prices using a signed float representing the price override + pattern: ^\d+(\.\d+)?$ + type: string + priceAdjustment: + description: |- + PriceAdjustment specifies the price change for matching instance types. Accepts either: + - A fixed price modifier (e.g., -0.5, 1.2) + - A percentage modifier (e.g., +10% for increase, -15% for decrease) + pattern: ^(([+-]{1}(\d*\.?\d+))|(\+{1}\d*\.?\d+%)|(^(-\d{1,2}(\.\d+)?%)$)|(-100%))$ + type: string + requirements: + description: |- + requirements constrain when this NodeOverlay is applied during scheduling simulations. + These requirements can match: + - Well-known labels (e.g., node.kubernetes.io/instance-type, karpenter.sh/nodepool) + - Custom labels from NodePool's spec.template.labels + items: + description: |- + A node selector requirement is a selector that contains values, a key, an operator that relates the key and values + to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count", "karpenter.k8s.aws/instance-capability-flex"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, Lt, Gte, and Lte. + enum: + - Gte + - Lte + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt, Lt, Gte, or Lte, the values + array must have a single element, which will be interpreted as an integer. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: requirements with operator 'NotIn' must have a value defined + rule: 'self.all(x, x.operator == ''NotIn'' ? x.values.size() != 0 : true)' + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt', 'Lt', 'Gte' or 'Lte' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'' || x.operator == ''Gte'' || x.operator == ''Lte'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + weight: + description: |- + Weight defines the priority of this NodeOverlay when overriding node attributes. + NodeOverlays with higher numerical weights take precedence over those with lower weights. + If no weight is specified, the NodeOverlay is treated as having a weight of 0. + When multiple NodeOverlays have identical weights, they are merged in alphabetical order. + format: int32 + maximum: 10000 + minimum: 1 + type: integer + required: + - requirements + type: object + x-kubernetes-validations: + - message: cannot set both 'price' and 'priceAdjustment' + rule: '!has(self.price) || !has(self.priceAdjustment)' + status: + description: NodeOverlayStatus defines the observed state of NodeOverlay + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -1268,7 +1879,7 @@ spec: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass type: string - - jsonPath: .status.resources.nodes + - jsonPath: .status.nodes name: Nodes type: string - jsonPath: .status.conditions[?(@.type=="Ready")].status @@ -1360,7 +1971,7 @@ spec: type: string reasons: description: |- - Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. Otherwise, this will apply to each reason defined. allowed reasons are Underutilized, Empty, and Drifted. items: @@ -1370,7 +1981,9 @@ spec: - Empty - Drifted type: string + maxItems: 50 type: array + x-kubernetes-list-type: set schedule: description: |- Schedule specifies when a budget begins being active, following @@ -1384,6 +1997,7 @@ spec: type: object maxItems: 50 type: array + x-kubernetes-list-type: atomic x-kubernetes-validations: - message: '''schedule'' must be set with ''duration''' rule: self.all(x, has(x.schedule) == has(x.duration)) @@ -1392,16 +2006,20 @@ spec: ConsolidateAfter is the duration the controller will wait before attempting to terminate nodes that are underutilized. Refer to ConsolidationPolicy for how underutilization is considered. + When replicas is set, ConsolidateAfter is simply ignored pattern: ^(([0-9]+(s|m|h))+|Never)$ type: string consolidationPolicy: default: WhenEmptyOrUnderutilized description: |- ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified. + Valid values: "WhenEmpty", "WhenEmptyOrUnderutilized", "Balanced". + When replicas is set, ConsolidationPolicy is simply ignored. enum: - WhenEmpty - WhenEmptyOrUnderutilized + - Balanced type: string required: - consolidateAfter @@ -1413,8 +2031,24 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Limits define a set of bounds for provisioning capacity. + description: |- + Limits define a set of bounds for provisioning capacity. + Limits other than limits.nodes is not supported when replicas is set. type: object + replicas: + description: |- + Replicas is the desired number of nodes for the NodePool. When specified, the NodePool will + maintain this fixed number of replicas rather than scaling based on pod demand. + When replicas is set: + - The following fields are ignored: + * disruption.consolidationPolicy + * disruption.consolidateAfter + - Only limits.nodes is supported; other resource limits (e.g., CPU, memory) must not be specified. + - Weight is not supported. + Note: This field is alpha. + format: int64 + minimum: 0 + type: integer template: description: |- Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. @@ -1426,28 +2060,26 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations type: object + x-kubernetes-map-type: granular labels: additionalProperties: type: string maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ description: |- - Map of string keys and values that can be used to organize and categorize + labels is a map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels type: object + x-kubernetes-map-type: atomic maxProperties: 100 x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) - - message: label domain "k8s.io" is restricted - rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) - message: label domain "karpenter.sh" is restricted rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) - message: label "karpenter.sh/nodepool" is restricted @@ -1455,7 +2087,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-tenancy", "karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/capacity-reservation-interruptible", "karpenter.k8s.aws/capacity-reservation-interruptible", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count", "karpenter.k8s.aws/instance-capability-flex", "karpenter.k8s.aws/placement-group-id", "karpenter.k8s.aws/placement-group-partition"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: |- @@ -1508,7 +2140,7 @@ spec: description: Requirements are layered with GetLabels and applied to every node. items: description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + A node selector requirement is a selector that contains values, a key, an operator that relates the key and values and minValues that represent the requirement to have at least that many values. properties: key: @@ -1517,10 +2149,6 @@ spec: maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - message: label domain "karpenter.sh" is restricted rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - message: label "karpenter.sh/nodepool" is restricted @@ -1528,7 +2156,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-tenancy", "karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/capacity-reservation-interruptible", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count", "karpenter.k8s.aws/instance-capability-flex", "karpenter.k8s.aws/placement-group-id", "karpenter.k8s.aws/placement-group-partition"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -1539,20 +2167,22 @@ spec: operator: description: |- Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, Lt, Gte, and Lte. enum: + - Gte + - Lte - In - NotIn - Exists - DoesNotExist - Gt - Lt + type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values + the values array must be empty. If the operator is Gt, Lt, Gte, or Lte, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: @@ -1567,16 +2197,17 @@ spec: type: object maxItems: 100 type: array + x-kubernetes-list-type: atomic x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements operator 'Gt', 'Lt', 'Gte', or 'Lte' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'' || x.operator == ''Gte'' || x.operator == ''Lte'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - message: requirements with 'minValues' must have at least that many values specified in the 'values' field rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' startupTaints: description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + startupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. @@ -1601,9 +2232,7 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the taint was added. format: date-time type: string value: @@ -1615,8 +2244,9 @@ spec: - key type: object type: array + x-kubernetes-list-type: atomic taints: - description: Taints will be applied to the NodeClaim's node. + description: taints will be applied to the NodeClaim's node. items: description: |- The node this Taint is attached to has the "effect" on @@ -1638,9 +2268,7 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the taint was added. format: date-time type: string value: @@ -1652,6 +2280,7 @@ spec: - key type: object type: array + x-kubernetes-list-type: atomic terminationGracePeriod: description: |- TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. @@ -1682,6 +2311,7 @@ spec: numerical weight indicates that this nodepool will be ordered ahead of other nodepools with lower weights. A nodepool with no weight will be treated as if it is a nodepool with a weight of 0. + Weight is not supported when replicas is set. format: int32 maximum: 100 minimum: 1 @@ -1689,6 +2319,13 @@ spec: required: - template type: object + x-kubernetes-validations: + - message: Cannot transition NodePool between static (replicas set) and dynamic (replicas unset) provisioning modes + rule: has(self.replicas) == has(oldSelf.replicas) + - message: only 'limits.nodes' is supported on static NodePools + rule: '!has(self.replicas) || (!has(self.limits) || size(self.limits) == 0 || (size(self.limits) == 1 && ''nodes'' in self.limits))' + - message: '''weight'' is not supported on static NodePools' + rule: '!has(self.replicas) || !has(self.weight)' status: description: NodePoolStatus defines the observed state of NodePool properties: @@ -1748,12 +2385,20 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map nodeClassObservedGeneration: description: |- NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset format: int64 type: integer + nodes: + default: 0 + description: Nodes is the count of nodes associated with this NodePool + format: int64 + type: integer resources: additionalProperties: anyOf: @@ -1770,5 +2415,8 @@ spec: served: true storage: true subresources: + scale: + specReplicasPath: .spec.replicas + statusReplicasPath: .status.nodes status: {} diff --git a/apps/karpenter/manifests/ClusterRole-karpenter-core.yml b/apps/karpenter/manifests/ClusterRole-karpenter-core.yml index aaf8971d..5348547c 100644 --- a/apps/karpenter/manifests/ClusterRole-karpenter-core.yml +++ b/apps/karpenter/manifests/ClusterRole-karpenter-core.yml @@ -11,7 +11,13 @@ metadata: rules: # Read - apiGroups: ["karpenter.sh"] - resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status", "nodeoverlays", "nodeoverlays/status"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling.x-k8s.io"] + resources: ["capacitybuffers", "capacitybuffers/status"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["podtemplates"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] @@ -21,19 +27,22 @@ rules: verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] - verbs: ["list", "watch"] + verbs: ["get", "list", "watch"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] - verbs: ["get", list, "watch"] + verbs: ["get", "list", "watch"] # Write - apiGroups: ["karpenter.sh"] resources: ["nodeclaims", "nodeclaims/status"] verbs: ["create", "delete", "update", "patch"] - apiGroups: ["karpenter.sh"] - resources: ["nodepools", "nodepools/status"] + resources: ["nodepools", "nodepools/status", "nodeoverlays/status"] + verbs: ["update", "patch"] + - apiGroups: ["autoscaling.x-k8s.io"] + resources: ["capacitybuffers/status"] verbs: ["update", "patch"] - apiGroups: [""] resources: ["events"] diff --git a/apps/karpenter/manifests/Deployment-karpenter.yml b/apps/karpenter/manifests/Deployment-karpenter.yml index 2dfb8a85..54ccb0db 100644 --- a/apps/karpenter/manifests/Deployment-karpenter.yml +++ b/apps/karpenter/manifests/Deployment-karpenter.yml @@ -47,7 +47,7 @@ spec: capabilities: drop: - ALL - image: public.ecr.aws/karpenter/controller:1.6.3@sha256:b76551596698381c4940c15093afc733357b9ffffd5c97612d324b7a794aa0ed + image: public.ecr.aws/karpenter/controller:1.14.0@sha256:723130949e4cab989461ff07d1be4cd781d63515432494641b33c235f300ae23 imagePullPolicy: IfNotPresent env: - name: KUBERNETES_MIN_VERSION @@ -68,6 +68,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: CPU_REQUESTS + valueFrom: + resourceFieldRef: + containerName: controller + divisor: 1m + resource: requests.cpu - name: MEMORY_LIMIT valueFrom: resourceFieldRef: @@ -75,7 +81,7 @@ spec: divisor: "0" resource: limits.memory - name: FEATURE_GATES - value: "ReservedCapacity=true,SpotToSpotConsolidation=true,NodeRepair=false" + value: "ReservedCapacity=true,SpotToSpotConsolidation=true,NodeRepair=false,NodeOverlay=false,StaticCapacity=false,CapacityBuffer=false" - name: BATCH_MAX_DURATION value: "10s" - name: BATCH_IDLE_DURATION @@ -92,6 +98,12 @@ spec: value: "Karpenter-${flux_cluster_name}" - name: RESERVED_ENIS value: "0" + - name: IGNORE_DRA_REQUESTS + value: "true" + - name: AMI_REFRESH_INTERVAL + value: "1m" + - name: SUBNET_REFRESH_INTERVAL + value: "1m" ports: - name: http-metrics containerPort: 8080