From db5382eef2a492c6b8e13af75627bfc25cd45f22 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 27 Aug 2025 14:05:00 -0700 Subject: [PATCH 01/10] Add release notes for 25.3.3, update versions Signed-off-by: Andrew Chen --- gpu-operator/life-cycle-policy.rst | 13 +++++---- gpu-operator/platform-support.rst | 3 ++ gpu-operator/release-notes.rst | 45 ++++++++++++++++++++++++++++++ gpu-operator/versions.json | 14 +++++----- gpu-operator/versions1.json | 10 +++---- repo.toml | 4 +-- 6 files changed, 69 insertions(+), 20 deletions(-) diff --git a/gpu-operator/life-cycle-policy.rst b/gpu-operator/life-cycle-policy.rst index c4c084d8c..fa4f59969 100644 --- a/gpu-operator/life-cycle-policy.rst +++ b/gpu-operator/life-cycle-policy.rst @@ -91,9 +91,10 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. - ${version} * - NVIDIA GPU Driver |ki|_ - - | `580.65.06 `_ (recommended) + - | `580.82.07 `_ (recommended) + | `580.65.06 `_ | `575.57.08 `_ - | `570.172.08 `_ (default) + | `570.172.08 `_ (default) | `570.158.01 `_ | `570.148.08 `_ | `535.261.03 `_ @@ -110,7 +111,7 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. - `0.17.3 `__ * - DCGM Exporter - - `4.2.3-4.1.3 `__ + - `4.4.0-4.5.0 `__ * - Node Feature Discovery - `v0.17.3 `__ @@ -120,10 +121,10 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. - `0.17.3 `__ * - NVIDIA MIG Manager for Kubernetes - - `0.12.2 `__ + - `0.12.3 `__ * - DCGM - - `4.2.3 `__ + - `4.4.1 `__ * - Validator for NVIDIA GPU Operator - ${version} @@ -145,7 +146,7 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. - v0.1.1 * - NVIDIA GDRCopy Driver - - `v2.5.0 `__ + - `v2.5.1 `__ .. _known-issue: diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index 95e449d2e..a03260f3e 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -148,6 +148,9 @@ The following NVIDIA data center GPUs are supported on x86 based platforms: | NVIDIA RTX PRO 6000 | NVIDIA Blackwell | | Blackwell Server Edition| | +-------------------------+------------------------+ + | NVIDIA RTX PRO 6000D | NVIDIA Blackwell | + | Blackwell Server Edition| | + +-------------------------+------------------------+ | NVIDIA RTX A6000 | NVIDIA Ampere /Ada | +-------------------------+------------------------+ | NVIDIA RTX A5000 | NVIDIA Ampere | diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index a0900cb9a..8f41f653d 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -33,6 +33,51 @@ See the :ref:`GPU Operator Component Matrix` for a list of software components a ---- +.. _v25.3.3: + +25.3.3 +====== + +.. _v25.3.3-new-features: + +New Features +------------ + +* Added support for additional features: + + - RTX Pro 6000 BSE: R580TRD2 by default + - MIG profiles support + - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) + - Kubevirt/OSV: VM with time-slice vGPU (Ubuntu 22.04 only) + + - RTX Pro 6000D BSE: R580TRD2 by default + - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) + +* Includes these software component versions: + + - NVIDIA DCGM v4.4.1 + - NVIDIA DCGM Exporter v4.4.0-4.5.0 + - NVIDIA GDRCopy Driver v2.5.1 + - NVIDIA Kubernetes Device Plugin v0.17.3 + - NVIDIA Kubevirt GPU Device Plugin v1.3.1 + - NVIDIA MIG Manager for Kubernetes v0.12.3 + - NVIDIA vGPU Device Manager v0.3.0 + +* Supports these NVIDIA Data Center GPU Driver versions: + + - 580.82.07 (recommended) + - 570.172.08 (default) + +Notes +----- +* The driver container image tag for OpenShift has changed after the `OCP 4.19` release. + + - Pre OCP 4.19: + - The driver image tag is formed with suffix `-rhcos4.17` (for example with OCP 4.17). + + - Starting OCP 4.19 and onwards: + - The driver image tag is formed with suffix `-rhel9.6` (for example with OCP 4.19). + .. _v25.3.2: 25.3.2 diff --git a/gpu-operator/versions.json b/gpu-operator/versions.json index 3dc444905..b033883fe 100644 --- a/gpu-operator/versions.json +++ b/gpu-operator/versions.json @@ -1,24 +1,24 @@ { - "latest": "24.9.2", + "latest": "25.3.3", "versions": [ { - "version": "24.9.2" + "version": "25.3.3" }, { - "version": "24.9.1" + "version": "25.3.2" }, { - "version": "24.9.0" + "version": "25.3.1" }, { - "version": "24.6.2" + "version": "25.3.0" }, { - "version": "24.6.1" + "version": "24.9.2" }, { - "version": "24.6.0" + "version": "24.9.1" } ] } diff --git a/gpu-operator/versions1.json b/gpu-operator/versions1.json index 86c7cd6ff..33d939b27 100644 --- a/gpu-operator/versions1.json +++ b/gpu-operator/versions1.json @@ -1,6 +1,10 @@ [ { "preferred": "true", + "url": "../25.3.3", + "version": "25.3.3" + }, + { "url": "../25.3.2", "version": "25.3.2" }, @@ -19,9 +23,5 @@ { "url": "../24.9.1", "version": "24.9.1" - }, - { - "url": "../24.9.0", - "version": "24.9.0" } -] \ No newline at end of file +] diff --git a/repo.toml b/repo.toml index a9f9f1b76..c9b2047e9 100644 --- a/repo.toml +++ b/repo.toml @@ -166,8 +166,8 @@ output_format = "linkcheck" docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" -version = "25.3.2" -source_substitutions = { version = "v25.3.2", recommended = "580.65.06" } +version = "25.3.3" +source_substitutions = { version = "v25.3.3", recommended = "580.65.06" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From 82a7d695507bb89ed1eb0e98a014888ea1cc5860 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 22 Aug 2025 12:27:09 -0700 Subject: [PATCH 02/10] Add nouveau issue to troubleshooting.rst Signed-off-by: Andrew Chen --- gpu-operator/troubleshooting.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/gpu-operator/troubleshooting.rst b/gpu-operator/troubleshooting.rst index 71ddb9c34..58da007fc 100644 --- a/gpu-operator/troubleshooting.rst +++ b/gpu-operator/troubleshooting.rst @@ -26,6 +26,29 @@ If you are facing an issue that is not covered by this page, please file an issu `NVIDIA GPU Operator GitHub repository `_. +************************************************** +The ``nouveau`` driver fails to initialize the GPU +************************************************** + +.. rubric:: Observation + :class: h4 + +- The GPU driver fails to initialize the GPU with the error ``Failed to enable MSI-X`` in the system journal logs. +- All GPU Operator pods become stuck in the ``init`` state. + +.. rubric:: Root Cause + :class: h4 + +- The ``nouveau`` Linux kernel module is loaded. + +.. rubric:: Action + :class: h4 + +The ``nouveau`` driver must be blacklisted when using NVIDIA vGPU. + +Follow the instructions in the `NVIDIA AI Enterprise: VMware Deployment Guide `_ +to disable ``nouveau`` on your OS/distro to resolve this issue. + *********************************** GPU Operator pods are stuck in Init *********************************** From f2efc36b72042a5e08ae094f102ef875757f2e91 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Tue, 2 Sep 2025 14:08:43 -0700 Subject: [PATCH 03/10] update recommended driver container version Signed-off-by: Andrew Chen --- gpu-operator/life-cycle-policy.rst | 4 ++-- gpu-operator/release-notes.rst | 3 +-- repo.toml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gpu-operator/life-cycle-policy.rst b/gpu-operator/life-cycle-policy.rst index fa4f59969..8e0d94758 100644 --- a/gpu-operator/life-cycle-policy.rst +++ b/gpu-operator/life-cycle-policy.rst @@ -91,10 +91,10 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. - ${version} * - NVIDIA GPU Driver |ki|_ - - | `580.82.07 `_ (recommended) + - | `580.82.07 `_ (default, recommended) | `580.65.06 `_ | `575.57.08 `_ - | `570.172.08 `_ (default) + | `570.172.08 `_ | `570.158.01 `_ | `570.148.08 `_ | `535.261.03 `_ diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 8f41f653d..6d0043fad 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -65,8 +65,7 @@ New Features * Supports these NVIDIA Data Center GPU Driver versions: - - 580.82.07 (recommended) - - 570.172.08 (default) + - 580.82.07 (default, recommended) Notes ----- diff --git a/repo.toml b/repo.toml index c9b2047e9..2ce711246 100644 --- a/repo.toml +++ b/repo.toml @@ -167,7 +167,7 @@ docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" version = "25.3.3" -source_substitutions = { version = "v25.3.3", recommended = "580.65.06" } +source_substitutions = { version = "v25.3.3", recommended = "580.82.07" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From c38f8f986af4a84d8f4dff749b9303abc4e63f94 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Tue, 2 Sep 2025 14:33:32 -0700 Subject: [PATCH 04/10] formating fixes Signed-off-by: Andrew Chen --- gpu-operator/release-notes.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 6d0043fad..c8415fbe7 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -43,14 +43,20 @@ See the :ref:`GPU Operator Component Matrix` for a list of software components a New Features ------------ +* Supports these NVIDIA Data Center GPU Driver versions: + + - 580.82.07 (default, recommended) + * Added support for additional features: - - RTX Pro 6000 BSE: R580TRD2 by default + - RTX Pro 6000 BSE: R580TRD2 by default + - MIG profiles support - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) - Kubevirt/OSV: VM with time-slice vGPU (Ubuntu 22.04 only) - RTX Pro 6000D BSE: R580TRD2 by default + - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) * Includes these software component versions: @@ -63,19 +69,13 @@ New Features - NVIDIA MIG Manager for Kubernetes v0.12.3 - NVIDIA vGPU Device Manager v0.3.0 -* Supports these NVIDIA Data Center GPU Driver versions: - - - 580.82.07 (default, recommended) - Notes ----- -* The driver container image tag for OpenShift has changed after the `OCP 4.19` release. +* The driver container image tag for OpenShift has changed after the ``OCP 4.19`` release. - - Pre OCP 4.19: - - The driver image tag is formed with suffix `-rhcos4.17` (for example with OCP 4.17). + - Pre OCP 4.19: The driver image tag is formed with suffix ``-rhcos4.17`` (for example with OCP 4.17). - - Starting OCP 4.19 and onwards: - - The driver image tag is formed with suffix `-rhel9.6` (for example with OCP 4.19). + - Starting OCP 4.19 and onwards: The driver image tag is formed with suffix ``-rhel9.6`` (for example with OCP 4.19). .. _v25.3.2: From 9af4eb47365845e1cc42d3e87eb1c24afb1c856c Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 5 Sep 2025 14:38:33 -0700 Subject: [PATCH 05/10] update release notes Signed-off-by: Andrew Chen --- gpu-operator/life-cycle-policy.rst | 14 +++++++------- gpu-operator/platform-support.rst | 7 ++++--- gpu-operator/release-notes.rst | 28 +++++----------------------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/gpu-operator/life-cycle-policy.rst b/gpu-operator/life-cycle-policy.rst index 8e0d94758..4a15b619c 100644 --- a/gpu-operator/life-cycle-policy.rst +++ b/gpu-operator/life-cycle-policy.rst @@ -102,38 +102,38 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. | `535.247.01 `_ * - NVIDIA Driver Manager for Kubernetes - - `v0.8.0 `__ + - `v0.8.1 `__ * - NVIDIA Container Toolkit - `1.17.8 `__ * - NVIDIA Kubernetes Device Plugin - - `0.17.3 `__ + - `0.17.4 `__ * - DCGM Exporter - - `4.4.0-4.5.0 `__ + - `4.3.1-4.4.0 `__ * - Node Feature Discovery - `v0.17.3 `__ * - | NVIDIA GPU Feature Discovery | for Kubernetes - - `0.17.3 `__ + - `0.17.4 `__ * - NVIDIA MIG Manager for Kubernetes - `0.12.3 `__ * - DCGM - - `4.4.1 `__ + - `4.3.1 `__ * - Validator for NVIDIA GPU Operator - ${version} * - NVIDIA KubeVirt GPU Device Plugin - - `v1.3.1 `__ + - `v1.4.0 `__ * - NVIDIA vGPU Device Manager - - `v0.3.0 `__ + - `v0.4.0 `__ * - NVIDIA GDS Driver |gds|_ - `2.20.5 `__ diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index a03260f3e..77ac3fcdb 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -149,7 +149,6 @@ The following NVIDIA data center GPUs are supported on x86 based platforms: | Blackwell Server Edition| | +-------------------------+------------------------+ | NVIDIA RTX PRO 6000D | NVIDIA Blackwell | - | Blackwell Server Edition| | +-------------------------+------------------------+ | NVIDIA RTX A6000 | NVIDIA Ampere /Ada | +-------------------------+------------------------+ @@ -511,8 +510,8 @@ Operating System Kubernetes KubeVirt OpenShift Virtual \ \ | GPU vGPU | GPU vGPU | Passthrough | Passthrough ================ =========== ============= ========= ============= =========== -Ubuntu 20.04 LTS 1.23---1.29 0.36+ 0.59.1+ -Ubuntu 22.04 LTS 1.23---1.29 0.36+ 0.59.1+ +Ubuntu 20.04 LTS 1.23---1.33 0.36+ 0.59.1+ +Ubuntu 22.04 LTS 1.23---1.33 0.36+ 0.59.1+ Red Hat Core OS 4.12---4.19 4.13---4.19 ================ =========== ============= ========= ============= =========== @@ -527,6 +526,8 @@ Refer to :ref:`GPU Operator with KubeVirt` or :ref:`NVIDIA GPU Operator with Ope KubeVirt and OpenShift Virtualization with NVIDIA vGPU is supported on the following devices: +- RTX Pro 6000 Blackwell Server Edition, RTX 6000D + - H200NVL - H100 diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index c8415fbe7..17c2ef84e 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -49,33 +49,15 @@ New Features * Added support for additional features: - - RTX Pro 6000 BSE: R580TRD2 by default + - RTX Pro 6000 Blackwell Server Edition - MIG profiles support - - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) - - Kubevirt/OSV: VM with time-slice vGPU (Ubuntu 22.04 only) + - KubeVirt and OpenShift Virtualization: VM with GPU passthrough (Ubuntu 22.04 only) + - KubeVirt and OpenShift Virtualization: VM with time-slice vGPU (Ubuntu 22.04 only) - - RTX Pro 6000D BSE: R580TRD2 by default + - RTX Pro 6000D Blackwell Server Edition - - Kubevirt/OSV: VM with GPU passthrough (Ubuntu 22.04 only) - -* Includes these software component versions: - - - NVIDIA DCGM v4.4.1 - - NVIDIA DCGM Exporter v4.4.0-4.5.0 - - NVIDIA GDRCopy Driver v2.5.1 - - NVIDIA Kubernetes Device Plugin v0.17.3 - - NVIDIA Kubevirt GPU Device Plugin v1.3.1 - - NVIDIA MIG Manager for Kubernetes v0.12.3 - - NVIDIA vGPU Device Manager v0.3.0 - -Notes ------ -* The driver container image tag for OpenShift has changed after the ``OCP 4.19`` release. - - - Pre OCP 4.19: The driver image tag is formed with suffix ``-rhcos4.17`` (for example with OCP 4.17). - - - Starting OCP 4.19 and onwards: The driver image tag is formed with suffix ``-rhel9.6`` (for example with OCP 4.19). + - KubeVirt and OpenShift Virtualization: VM with GPU passthrough (Ubuntu 22.04 only) .. _v25.3.2: From 193db1b74187247b30c94443706e0869459328dc Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 10 Sep 2025 12:11:54 -0700 Subject: [PATCH 06/10] add fixed issues Signed-off-by: Andrew Chen --- gpu-operator/release-notes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 17c2ef84e..168ece099 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -59,6 +59,11 @@ New Features - KubeVirt and OpenShift Virtualization: VM with GPU passthrough (Ubuntu 22.04 only) +Fixed Issues +------------- + +* Fixed an issue where user-supplied environment variables configured in ClusterPolicy were not getting set in the rendered DaemonSet. User-supplied environment variables now take precedence over environment variables set by the ClusterPolicy controller. + .. _v25.3.2: 25.3.2 From 08dff50e8ae2b9d39f4388bd32ed75ae2697b434 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 10 Sep 2025 12:15:28 -0700 Subject: [PATCH 07/10] fix RTX Pro 6000D references Signed-off-by: Andrew Chen --- gpu-operator/platform-support.rst | 2 +- gpu-operator/release-notes.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index 77ac3fcdb..d818ae26d 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -526,7 +526,7 @@ Refer to :ref:`GPU Operator with KubeVirt` or :ref:`NVIDIA GPU Operator with Ope KubeVirt and OpenShift Virtualization with NVIDIA vGPU is supported on the following devices: -- RTX Pro 6000 Blackwell Server Edition, RTX 6000D +- RTX Pro 6000 Blackwell Server Edition - H200NVL diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 168ece099..6447c4f98 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -55,7 +55,7 @@ New Features - KubeVirt and OpenShift Virtualization: VM with GPU passthrough (Ubuntu 22.04 only) - KubeVirt and OpenShift Virtualization: VM with time-slice vGPU (Ubuntu 22.04 only) - - RTX Pro 6000D Blackwell Server Edition + - RTX Pro 6000D - KubeVirt and OpenShift Virtualization: VM with GPU passthrough (Ubuntu 22.04 only) From 6ea59f04072a318dc44cc685a417ea511eb95f4f Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 10 Sep 2025 12:37:33 -0700 Subject: [PATCH 08/10] update Supported Precompiled Drivers Signed-off-by: Andrew Chen --- gpu-operator/platform-support.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index d818ae26d..c39a2973b 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -470,6 +470,9 @@ See the :doc:`precompiled-drivers` page for more information about using precomp | Ubuntu 22.04 | Generic, NVIDIA, Azure | 5.15 | R535, R550, R570 | | | AWS, Oracle | | | +----------------------------+------------------------+----------------+---------------------+ +| Ubuntu 22.04 | Generic, NVIDIA, Azure | 6.8 | R535, R570 | +| | AWS, Oracle | | | ++----------------------------+------------------------+----------------+---------------------+ | Ubuntu 24.04 | Generic, NVIDIA, Azure | 6.8 | R550, R570 | | | AWS, Oracle | | | +----------------------------+------------------------+----------------+---------------------+ From b58dc4cc62d6f60077867542fe3f07ae4d398fbe Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 10 Sep 2025 13:01:48 -0700 Subject: [PATCH 09/10] change wording on nouveau troubleshooting note Signed-off-by: Andrew Chen --- gpu-operator/troubleshooting.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpu-operator/troubleshooting.rst b/gpu-operator/troubleshooting.rst index 58da007fc..73ea364f8 100644 --- a/gpu-operator/troubleshooting.rst +++ b/gpu-operator/troubleshooting.rst @@ -44,7 +44,7 @@ The ``nouveau`` driver fails to initialize the GPU .. rubric:: Action :class: h4 -The ``nouveau`` driver must be blacklisted when using NVIDIA vGPU. +The ``nouveau`` driver must be denylisted when using NVIDIA vGPU. Follow the instructions in the `NVIDIA AI Enterprise: VMware Deployment Guide `_ to disable ``nouveau`` on your OS/distro to resolve this issue. From 562964d5c41928a2998633484cd60cf3cc449411 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Wed, 10 Sep 2025 13:28:47 -0700 Subject: [PATCH 10/10] tweaks Signed-off-by: Andrew Chen --- gpu-operator/release-notes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 6447c4f98..afd2bf6de 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -62,7 +62,8 @@ New Features Fixed Issues ------------- -* Fixed an issue where user-supplied environment variables configured in ClusterPolicy were not getting set in the rendered DaemonSet. User-supplied environment variables now take precedence over environment variables set by the ClusterPolicy controller. +* Fixed an issue where user-supplied environment variables configured in ClusterPolicy were not getting set in the rendered DaemonSet. + User-supplied environment variables now take precedence over environment variables set by the ClusterPolicy controller. .. _v25.3.2: