diff --git a/helm_chart/HyperPodHelmChart/Chart.yaml b/helm_chart/HyperPodHelmChart/Chart.yaml index eebee375..7242153a 100644 --- a/helm_chart/HyperPodHelmChart/Chart.yaml +++ b/helm_chart/HyperPodHelmChart/Chart.yaml @@ -81,7 +81,7 @@ dependencies: repository: "file://charts/team-role-and-bindings" condition: team-role-and-bindings.enabled - name: hyperpod-inference-operator - version: "2.2.2" + version: "2.3.0" repository: "file://charts/inference-operator" condition: inferenceOperators.enabled - name: hyperpod-patching diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml index 4f3c04d8..e6160f00 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml @@ -15,11 +15,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.2.2 +version: 2.3.0 # This is the version number of the application being deployed. Keep this aligned # with operator image MAJOR.MINOR version. -appVersion: "3.2" +appVersion: "3.3" dependencies: - name: aws-mountpoint-s3-csi-driver diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml index a4903174..c25a22b1 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml @@ -4394,6 +4394,13 @@ spec: to /ping if not specified. pattern: ^/.* type: string + idleTimeoutSeconds: + description: Idle timeout in seconds for the ALB connection. If + not specified, defaults to 60 seconds. + format: int32 + maximum: 4000 + minimum: 1 + type: integer routingAlgorithm: default: least_outstanding_requests description: Routing algorithm for the ALB target group (least_oustanding_requests @@ -4442,6 +4449,39 @@ spec: type: integer type: object type: object + modelCacheConfig: + description: Configuration for model caching (image and data). + properties: + imageCache: + description: |- + Configuration for image caching. When enabled, the operator pre-pulls the + inference server container image onto target nodes via a DaemonSet. + properties: + enabled: + description: |- + Enable or disable image caching for this deployment. + When enabled, the operator creates a ModelImageCache resource that pins + the inference server image on target nodes. + type: boolean + required: + - enabled + type: object + weightsCache: + description: Configuration for pre-caching model weights on local + NVMe storage. + properties: + enabled: + default: false + description: Whether model data caching is enabled. + type: boolean + hostPath: + default: /opt/dlami/nvme + description: Host path where cached model weights are stored. + maxLength: 255 + minLength: 1 + type: string + type: object + type: object modelName: description: Name of model that will be created on Sagemaker maxLength: 63 @@ -6656,6 +6696,30 @@ spec: - state type: object type: object + imageCacheStatus: + description: Status of model server image caching + properties: + message: + description: Human-readable message. + type: string + readyNodes: + description: Number of nodes where the image is cached. + format: int32 + type: integer + state: + description: Current state of the image cache. + enum: + - Pending + - Downloading + - Ready + - Failed + - PartiallyReady + type: string + targetNodes: + description: Total number of target nodes. + format: int32 + type: integer + type: object metricsStatus: description: Status of metrics collection properties: diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml index a5e37ffd..90a9b6a7 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml @@ -907,6 +907,13 @@ spec: to /ping if not specified. pattern: ^/.* type: string + idleTimeoutSeconds: + description: Idle timeout in seconds for the ALB connection. If + not specified, defaults to 60 seconds. + format: int32 + maximum: 4000 + minimum: 1 + type: integer routingAlgorithm: default: least_outstanding_requests description: Routing algorithm for the ALB target group (least_oustanding_requests @@ -1007,6 +1014,39 @@ spec: - acceptEula - modelId type: object + modelCacheConfig: + description: Configuration for model caching (image and data). + properties: + imageCache: + description: |- + Configuration for image caching. When enabled, the operator pre-pulls the + inference server container image onto target nodes via a DaemonSet. + properties: + enabled: + description: |- + Enable or disable image caching for this deployment. + When enabled, the operator creates a ModelImageCache resource that pins + the inference server image on target nodes. + type: boolean + required: + - enabled + type: object + weightsCache: + description: Configuration for pre-caching model weights on local + NVMe storage. + properties: + enabled: + default: false + description: Whether model data caching is enabled. + type: boolean + hostPath: + default: /opt/dlami/nvme + description: Host path where cached model weights are stored. + maxLength: 255 + minLength: 1 + type: string + type: object + type: object replicas: default: 1 description: The desired number of inference server replicas. Default @@ -1349,6 +1389,30 @@ spec: - state type: object type: object + imageCacheStatus: + description: Status of model server image caching + properties: + message: + description: Human-readable message. + type: string + readyNodes: + description: Number of nodes where the image is cached. + format: int32 + type: integer + state: + description: Current state of the image cache. + enum: + - Pending + - Downloading + - Ready + - Failed + - PartiallyReady + type: string + targetNodes: + description: Total number of target nodes. + format: int32 + type: integer + type: object metricsStatus: description: Status of metrics collection properties: diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelimagecaches.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelimagecaches.yaml new file mode 100644 index 00000000..e9ccc635 --- /dev/null +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelimagecaches.yaml @@ -0,0 +1,338 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: modelimagecaches.inference.sagemaker.aws.amazon.com +spec: + group: inference.sagemaker.aws.amazon.com + names: + kind: ModelImageCache + listKind: ModelImageCacheList + plural: modelimagecaches + shortNames: + - mic + singular: modelimagecache + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.image + name: Image + type: string + - jsonPath: .status.state + name: State + type: string + - jsonPath: .status.readyNodes + name: Ready + type: integer + - jsonPath: .status.targetNodes + name: Target + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + ModelImageCache is the Schema for the modelimagecaches API. + It represents a cached container image on a set of nodes, managed via a backing DaemonSet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ModelImageCacheSpec defines the desired state of ModelImageCache. + properties: + image: + description: The full container image URI to cache on target nodes. + type: string + nodeAffinity: + description: Node affinity for more complex node scheduling constraints. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: Node selector to restrict which nodes receive the cached + image. + type: object + parentReferences: + description: |- + List of parent resources that reference this image cache. + Used to track ownership across namespaces. + items: + description: ParentReference identifies a resource that uses this + image cache. + properties: + kind: + description: Kind of the parent resource (e.g., InferenceEndpointConfig, + JumpStartModel). + type: string + name: + description: Name of the parent resource. + type: string + namespace: + description: Namespace of the parent resource. + type: string + uid: + description: UID of the parent resource. + type: string + required: + - kind + - name + - namespace + - uid + type: object + type: array + required: + - image + type: object + status: + description: ModelImageCacheStatus defines the observed state of ModelImageCache. + properties: + lastTransitionTime: + description: Time of the last state transition. + format: date-time + type: string + message: + description: Human-readable message describing the current state. + type: string + readyNodes: + description: Number of nodes where the image has been successfully + cached. + format: int32 + type: integer + resolvedDigest: + description: |- + The resolved image digest as reported by the container runtime (e.g., sha256:abc123...). + Populated once a cache pod successfully pulls and runs the image. + type: string + state: + description: Current state of the image cache. + enum: + - Pending + - Downloading + - Ready + - Failed + - PartiallyReady + - Deleting + type: string + targetNodes: + description: Total number of nodes targeted by this image cache. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelweightscacheconfigs.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelweightscacheconfigs.yaml new file mode 100644 index 00000000..7f4a6202 --- /dev/null +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_modelweightscacheconfigs.yaml @@ -0,0 +1,507 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: modelweightscacheconfigs.inference.sagemaker.aws.amazon.com +spec: + group: inference.sagemaker.aws.amazon.com + names: + kind: ModelWeightsCacheConfig + listKind: ModelWeightsCacheConfigList + plural: modelweightscacheconfigs + singular: modelweightscacheconfig + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string + - jsonPath: .status.readyNodes + name: Ready + type: integer + - jsonPath: .status.targetNodes + name: Target + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: ModelWeightsCacheConfig is the Schema for the modelweightscacheconfigs + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ModelWeightsCacheConfigSpec defines the desired state of + ModelWeightsCacheConfig. + properties: + enabled: + default: true + description: Whether data caching is enabled. When false, the controller + takes no action. + type: boolean + hostPath: + default: /opt/dlami/nvme + description: Host path where cached model weights are stored. + maxLength: 255 + minLength: 1 + type: string + jumpStartModel: + description: |- + JumpStart model configuration. When present, the controller uses JumpStart model resolution + regardless of modelSourceType. Reuses the existing ModelSpec from JumpStartModel CRD. + properties: + acceptEula: + default: false + description: For models that require a Model Access Config, specify + True or False to indicate whether model terms of use have been + accepted. + type: boolean + additionalConfigs: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + maxItems: 10 + type: array + gatedModelDownloadRole: + description: The Amazon Resource Name (ARN) of an IAM role that + will be used to download gated model + maxLength: 2048 + minLength: 20 + pattern: ^arn:aws[a-z\-]*:iam::\d{12}:role/?[a-zA-Z_0-9+=,.@\-_/]+$ + type: string + modelHubName: + default: SageMakerPublicHub + description: The name of the model hub content. Can be an ARN + or a simple name. + maxLength: 63 + pattern: ^[a-zA-Z0-9](-*[a-zA-Z0-9]){0,62}$ + type: string + modelId: + description: The unique identifier of the model within the specified + hub (hubContentArn). + maxLength: 63 + pattern: ^[a-zA-Z0-9](-*[a-zA-Z0-9]){0,62}$ + type: string + modelVersion: + description: The version of the model to deploy, in semantic versioning + format (e.g., 1.0.0). + maxLength: 14 + minLength: 5 + pattern: ^\d{1,4}.\d{1,4}.\d{1,4}$ + type: string + required: + - acceptEula + - modelId + type: object + modelSourceConfig: + description: Reference to the model source to cache. Reuses the existing + ModelSourceConfig structure. + properties: + fsxStorage: + properties: + dnsName: + description: FSX File System DNS Name + type: string + fileSystemId: + description: FSX File System ID + type: string + mountName: + description: FSX File System Mount Name + type: string + required: + - fileSystemId + type: object + huggingFaceModel: + description: HuggingFace model configuration. Required when modelSourceType + is "huggingface". + properties: + commitSHA: + description: |- + Git commit SHA for the model revision. Must be a full 40-character lowercase hex SHA. + If not provided, the operator defaults to "main" branch. + pattern: ^[0-9a-f]{40}$ + type: string + modelId: + description: HuggingFace Hub model identifier in org/model + format (e.g. "meta-llama/Llama-3.1-8B-Instruct"). + pattern: ^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$ + type: string + tokenSecretRef: + description: |- + Reference to a Kubernetes Secret containing the HuggingFace API token. + The token is injected as the HF_TOKEN environment variable into the InitContainer only. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + required: + - modelId + type: object + modelLocation: + description: Specific location where the model data exists + type: string + modelSourceType: + enum: + - fsx + - s3 + - huggingface + - kubernetesVolume + type: string + prefetchEnabled: + default: false + description: In case the model seems to fit within the instance's + memory (VRAM), this option can be used to pre-fetch the model + to RAM and then the inference server will load to the GPU/CPU + device thereafter. + type: boolean + s3Storage: + properties: + bucketName: + description: S3 bucket location + type: string + region: + description: S3 bucket region + type: string + required: + - bucketName + - region + type: object + required: + - modelSourceType + type: object + nodeAffinity: + description: Node affinity for cache population scheduling. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: Node selector constraints for cache population. Cache + pods run only on matching nodes. + type: object + required: + - modelSourceConfig + type: object + status: + description: ModelWeightsCacheConfigStatus defines the observed state + of ModelWeightsCacheConfig. + properties: + conditions: + description: Conditions for detailed status reporting + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + message: + description: Human-readable message about current state + type: string + readyNodes: + description: Number of nodes where cache is ready + format: int32 + type: integer + state: + description: Current state of the cache (Pending, Downloading, Ready, + Failed, Deleting) + type: string + targetNodes: + description: Total number of eligible nodes targeted by the DaemonSet + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/rbac/role.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/rbac/role.yaml index eb929cbd..fe7cba26 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/rbac/role.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/rbac/role.yaml @@ -13,6 +13,8 @@ rules: - jumpstartmodels - inferenceendpointconfigs - sagemakerendpointregistrations + - modelweightscacheconfigs + - modelimagecaches verbs: - create - delete @@ -27,6 +29,8 @@ rules: - jumpstartmodels/finalizers - inferenceendpointconfigs/finalizers - sagemakerendpointregistrations/finalizers + - modelweightscacheconfigs/finalizers + - modelimagecaches/finalizers verbs: - update - apiGroups: @@ -35,6 +39,8 @@ rules: - jumpstartmodels/status - inferenceendpointconfigs/status - sagemakerendpointregistrations/status + - modelweightscacheconfigs/status + - modelimagecaches/status verbs: - get - patch @@ -118,4 +124,21 @@ rules: verbs: - get - list - - watch \ No newline at end of file + - watch +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - create + - list + - watch +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + resourceNames: + - hyperpod-image-cache + verbs: + - get + - delete diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml index 3ee8a09a..97356c6f 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml @@ -25,7 +25,7 @@ image: ap-southeast-3: 158128612970.dkr.ecr.ap-southeast-3.amazonaws.com ap-south-2: 680458885894.dkr.ecr.ap-south-2.amazonaws.com eu-south-2: 025050981094.dkr.ecr.eu-south-2.amazonaws.com - tag: v3.2 + tag: v3.3 pullPolicy: Always repository: initContainer: