From c660fb4a33c29fea003952615c200524bbfb1262 Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 14:44:59 -0600
Subject: [PATCH 1/7] WIP: run Kubernetes node conformance tests in Build Azure
 SIG Image

Adds an opt-in, non-blocking step to the Build Azure SIG Image workflow
that runs the [NodeConformance] subset of e2e_node.test against the
freshly-booted test VM.

- New optional workflow input run_node_conformance (default false).
- New helper script images/capi/hack/node-conformance.sh that downloads
  the version-matched kubernetes-test tarball from dl.k8s.io and runs
  e2e_node.test in standalone mode against the system containerd.
- Linux only; Windows is skipped (node conformance is Linux-only and
  the Windows e2e subset requires a real cluster).
- Step is continue-on-error so it surfaces signal as a warning without
  blocking promote while we iterate.
- Results are uploaded as the node-conformance-results artifact.
- Test job timeout-minutes bumped to 150 only when conformance is on.
---
 .github/workflows/build-azure-sig.yaml |  57 +++++++++++-
 images/capi/hack/node-conformance.sh   | 124 +++++++++++++++++++++++++
 2 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100755 images/capi/hack/node-conformance.sh

diff --git a/.github/workflows/build-azure-sig.yaml b/.github/workflows/build-azure-sig.yaml
index 1c86351907..6fa34ac420 100644
--- a/.github/workflows/build-azure-sig.yaml
+++ b/.github/workflows/build-azure-sig.yaml
@@ -26,6 +26,7 @@
 # - skip_test - skip the test stage
 # - skip_promote - skip the promote stage
 # - packer_debug - enable Packer debug logging (sets PACKER_LOG=1)
+# - run_node_conformance - run Kubernetes node conformance tests on the test VM (Linux only)
 
 name: Build Azure SIG Image
 
@@ -96,6 +97,11 @@ on:
         required: false
         type: boolean
         default: false
+      run_node_conformance:
+        description: 'Run Kubernetes node conformance tests on the test VM (Linux only, opt-in, non-blocking)'
+        required: false
+        type: boolean
+        default: false
       skip_promote:
         description: 'Skip the promote stage (requires manual approval)'
         required: false
@@ -281,7 +287,7 @@ jobs:
     needs: build
     if: ${{ !inputs.skip_test }}
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: ${{ inputs.run_node_conformance && 150 || 15 }}
     env:
       KUBERNETES_VERSION: ${{ inputs.kubernetes_version }}
 
@@ -559,6 +565,55 @@ jobs:
           fi
           echo "All smoke tests passed"
 
+      # -- Node conformance tests (opt-in, Linux only, non-blocking) -----------
+      - name: Run node conformance tests
+        id: node_conformance
+        if: ${{ inputs.run_node_conformance && steps.vars.outputs.OS_TYPE != 'Windows' }}
+        continue-on-error: true
+        env:
+          OS_TYPE: ${{ steps.vars.outputs.OS_TYPE }}
+        run: |
+          set -euo pipefail
+
+          SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${TMPDIR}/sshkey"
+          SCP_CMD="scp ${SSH_OPTS}"
+          SSH_CMD="ssh ${SSH_OPTS} capi@${PUBLIC_IP}"
+
+          echo "::group::Copy node-conformance helper to VM"
+          ${SCP_CMD} images/capi/hack/node-conformance.sh "capi@${PUBLIC_IP}:/tmp/node-conformance.sh"
+          ${SSH_CMD} 'chmod +x /tmp/node-conformance.sh'
+          echo "::endgroup::"
+
+          echo "::group::Run node conformance"
+          # Run the suite; tolerate non-zero exit so we can still pull artifacts.
+          set +e
+          ${SSH_CMD} "KUBERNETES_VERSION='${KUBERNETES_VERSION}' /tmp/node-conformance.sh"
+          NC_EXIT=$?
+          set -e
+          echo "node-conformance exited with ${NC_EXIT}"
+          echo "exit_code=${NC_EXIT}" >> "${GITHUB_OUTPUT}"
+          echo "::endgroup::"
+
+          echo "::group::Collect node-conformance results"
+          ARTIFACTS="${GITHUB_WORKSPACE}/_artifacts/node-conformance"
+          mkdir -p "${ARTIFACTS}"
+          ${SCP_CMD} -r "capi@${PUBLIC_IP}:/tmp/node-conformance-results/." "${ARTIFACTS}/" || true
+          ls -la "${ARTIFACTS}" || true
+          echo "::endgroup::"
+
+          if [[ "${NC_EXIT}" -ne 0 ]]; then
+            echo "::warning::node conformance tests reported failures (exit ${NC_EXIT}); not blocking the workflow"
+          fi
+
+      - name: Upload node conformance results
+        if: ${{ always() && steps.node_conformance.conclusion != 'skipped' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: node-conformance-results
+          path: _artifacts/node-conformance/
+          retention-days: 7
+          if-no-files-found: warn
+
       # -- Collect diagnostics on failure --------------------------------------
       - name: Collect diagnostics
         if: failure()
diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
new file mode 100755
index 0000000000..c7eef50739
--- /dev/null
+++ b/images/capi/hack/node-conformance.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2086
+#
+# Run the Kubernetes node-conformance subset of the e2e_node test suite on the
+# local machine.  Intended to be executed on a freshly-booted VM created from a
+# CAPI image-builder image, _not_ on a node that is already part of a cluster.
+#
+# This is a best-effort runner that pulls the official versioned kubernetes-test
+# tarball from dl.k8s.io (matched to the Kubernetes version baked into the
+# image) and invokes ./e2e_node.test in standalone mode so the framework can
+# start its own kubelet against the system's container runtime.
+#
+# Results (JUnit XML + ginkgo logs) are written to /tmp/node-conformance-results.
+#
+# Environment variables:
+#   KUBERNETES_VERSION   Required. e.g. "1.31.1" (no leading "v")
+#   GINKGO_FOCUS         Optional. Default '\[NodeConformance\]'
+#   GINKGO_SKIP          Optional. Default '\[Flaky\]|\[Serial\]|\[Slow\]|\[Alpha\]'
+#   RESULTS_DIR          Optional. Default /tmp/node-conformance-results
+#   TARBALL_URL          Optional. Override the test tarball URL.
+#
+# Exit code is the exit code of e2e_node.test (non-zero on any failed spec).
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBERNETES_VERSION="${KUBERNETES_VERSION:?KUBERNETES_VERSION is required, e.g. 1.31.1}"
+GINKGO_FOCUS="${GINKGO_FOCUS:-\[NodeConformance\]}"
+GINKGO_SKIP="${GINKGO_SKIP:-\[Flaky\]|\[Serial\]|\[Slow\]|\[Alpha\]}"
+RESULTS_DIR="${RESULTS_DIR:-/tmp/node-conformance-results}"
+
+ARCH="$(uname -m)"
+case "${ARCH}" in
+  x86_64)  GO_ARCH="amd64" ;;
+  aarch64) GO_ARCH="arm64" ;;
+  *)       echo "unsupported arch: ${ARCH}" >&2; exit 1 ;;
+esac
+
+TARBALL_URL="${TARBALL_URL:-https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-test-linux-${GO_ARCH}.tar.gz}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "==> Node conformance: kubernetes v${KUBERNETES_VERSION} (${GO_ARCH})"
+echo "    focus:   ${GINKGO_FOCUS}"
+echo "    skip:    ${GINKGO_SKIP}"
+echo "    results: ${RESULTS_DIR}"
+echo "    tarball: ${TARBALL_URL}"
+
+mkdir -p "${RESULTS_DIR}"
+sudo chown "$(id -u):$(id -g)" "${RESULTS_DIR}"
+
+echo "==> Downloading test tarball"
+curl --fail --silent --show-error --location \
+  --output "${WORKDIR}/test.tar.gz" "${TARBALL_URL}"
+
+echo "==> Extracting e2e_node.test and ginkgo"
+tar -xzf "${WORKDIR}/test.tar.gz" -C "${WORKDIR}" \
+  kubernetes/test/bin/e2e_node.test \
+  kubernetes/test/bin/ginkgo
+E2E_NODE_TEST="${WORKDIR}/kubernetes/test/bin/e2e_node.test"
+GINKGO_BIN="${WORKDIR}/kubernetes/test/bin/ginkgo"
+chmod +x "${E2E_NODE_TEST}" "${GINKGO_BIN}"
+
+# The framework starts its own kubelet under test, so stop the system unit (if
+# any) to free /var/lib/kubelet and the kubelet socket.  Don't fail if absent.
+if systemctl list-unit-files kubelet.service >/dev/null 2>&1; then
+  echo "==> Stopping system kubelet"
+  sudo systemctl stop kubelet || true
+fi
+
+# Detect the system container runtime endpoint.
+RUNTIME_ENDPOINT=""
+for sock in /run/containerd/containerd.sock /var/run/containerd/containerd.sock /var/run/crio/crio.sock; do
+  if [[ -S "${sock}" ]]; then
+    RUNTIME_ENDPOINT="unix://${sock}"
+    break
+  fi
+done
+if [[ -z "${RUNTIME_ENDPOINT}" ]]; then
+  echo "ERROR: no container runtime socket found" >&2
+  exit 1
+fi
+echo "==> Using container runtime endpoint: ${RUNTIME_ENDPOINT}"
+
+# Locate the kubelet binary that was baked into the image; the test framework
+# will exec it.
+KUBELET_BIN="$(command -v kubelet || true)"
+if [[ -z "${KUBELET_BIN}" ]]; then
+  for candidate in /usr/local/bin/kubelet /usr/bin/kubelet /opt/bin/kubelet; do
+    if [[ -x "${candidate}" ]]; then KUBELET_BIN="${candidate}"; break; fi
+  done
+fi
+if [[ -z "${KUBELET_BIN}" ]]; then
+  echo "ERROR: kubelet binary not found on PATH" >&2
+  exit 1
+fi
+echo "==> Using kubelet binary: ${KUBELET_BIN}"
+
+NODE_NAME="$(hostname)"
+
+echo "==> Running e2e_node.test"
+set +e
+sudo -E "${E2E_NODE_TEST}" \
+  --node-name="${NODE_NAME}" \
+  --standalone-mode=true \
+  --kubelet-flags="--kubelet-cgroups=/kubelet.slice --cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=/system.slice/containerd.service" \
+  --container-runtime-endpoint="${RUNTIME_ENDPOINT}" \
+  --ginkgo.focus="${GINKGO_FOCUS}" \
+  --ginkgo.skip="${GINKGO_SKIP}" \
+  --ginkgo.timeout=2h \
+  --ginkgo.v \
+  --report-dir="${RESULTS_DIR}" \
+  --report-prefix="node-conformance" \
+  2>&1 | tee "${RESULTS_DIR}/e2e_node.log"
+EXIT_CODE=${PIPESTATUS[0]}
+set -e
+
+# Make results readable for the calling user (scp back).
+sudo chown -R "$(id -u):$(id -g)" "${RESULTS_DIR}" || true
+
+echo "==> e2e_node.test exited with ${EXIT_CODE}"
+exit "${EXIT_CODE}"

From a0d592c73cb023913188a87dac565212301ca7d1 Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 14:51:10 -0600
Subject: [PATCH 2/7] Verify Kubernetes test tarball with published SHA-256

Fetch the matching .sha256 file from dl.k8s.io and verify the test
tarball before extracting it. The published checksum file contains
only the hex digest, so we synthesize a sha256sum-compatible line
and use sha256sum --check --strict for verification.
---
 images/capi/hack/node-conformance.sh | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
index c7eef50739..b1ae364eb4 100755
--- a/images/capi/hack/node-conformance.sh
+++ b/images/capi/hack/node-conformance.sh
@@ -18,6 +18,9 @@
 #   GINKGO_SKIP          Optional. Default '\[Flaky\]|\[Serial\]|\[Slow\]|\[Alpha\]'
 #   RESULTS_DIR          Optional. Default /tmp/node-conformance-results
 #   TARBALL_URL          Optional. Override the test tarball URL.
+#   TARBALL_SHA256_URL   Optional. Override the checksum URL. Defaults to
+#                        "${TARBALL_URL}.sha256". The remote file is expected
+#                        to contain just the hex digest (as dl.k8s.io serves).
 #
 # Exit code is the exit code of e2e_node.test (non-zero on any failed spec).
 
@@ -38,6 +41,7 @@ case "${ARCH}" in
 esac
 
 TARBALL_URL="${TARBALL_URL:-https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-test-linux-${GO_ARCH}.tar.gz}"
+TARBALL_SHA256_URL="${TARBALL_SHA256_URL:-${TARBALL_URL}.sha256}"
 
 WORKDIR="$(mktemp -d)"
 trap 'rm -rf "${WORKDIR}"' EXIT
@@ -55,6 +59,28 @@ echo "==> Downloading test tarball"
 curl --fail --silent --show-error --location \
   --output "${WORKDIR}/test.tar.gz" "${TARBALL_URL}"
 
+echo "==> Downloading tarball SHA-256 checksum"
+curl --fail --silent --show-error --location \
+  --output "${WORKDIR}/test.tar.gz.sha256" "${TARBALL_SHA256_URL}"
+
+echo "==> Verifying tarball checksum"
+# dl.k8s.io publishes the bare hex digest (no filename); build a sha256sum
+# compatible line so we can use the standard verifier.
+EXPECTED_SHA256="$(tr -d '[:space:]' < "${WORKDIR}/test.tar.gz.sha256")"
+if [[ -z "${EXPECTED_SHA256}" ]]; then
+  echo "ERROR: empty checksum retrieved from ${TARBALL_SHA256_URL}" >&2
+  exit 1
+fi
+echo "${EXPECTED_SHA256}  test.tar.gz" > "${WORKDIR}/test.tar.gz.sha256sum"
+( cd "${WORKDIR}" && sha256sum --check --strict --status test.tar.gz.sha256sum ) || {
+  ACTUAL_SHA256="$(sha256sum "${WORKDIR}/test.tar.gz" | awk '{print $1}')"
+  echo "ERROR: SHA-256 mismatch for ${TARBALL_URL}" >&2
+  echo "  expected: ${EXPECTED_SHA256}" >&2
+  echo "  actual:   ${ACTUAL_SHA256}" >&2
+  exit 1
+}
+echo "    OK (${EXPECTED_SHA256})"
+
 echo "==> Extracting e2e_node.test and ginkgo"
 tar -xzf "${WORKDIR}/test.tar.gz" -C "${WORKDIR}" \
   kubernetes/test/bin/e2e_node.test \

From 66e6d27b909cde2a8a4e85c84237aa4b7036a653 Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 14:53:16 -0600
Subject: [PATCH 3/7] Match kubelet --runtime-cgroups to detected container
 runtime

The runtime cgroup path was hardcoded to containerd. When CRI-O is the
detected runtime, set --runtime-cgroups to /system.slice/crio.service
instead so kubelet tracks the correct slice.
---
 images/capi/hack/node-conformance.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
index b1ae364eb4..6106b29161 100755
--- a/images/capi/hack/node-conformance.sh
+++ b/images/capi/hack/node-conformance.sh
@@ -96,11 +96,16 @@ if systemctl list-unit-files kubelet.service >/dev/null 2>&1; then
   sudo systemctl stop kubelet || true
 fi
 
-# Detect the system container runtime endpoint.
+# Detect the system container runtime endpoint and its systemd cgroup path.
 RUNTIME_ENDPOINT=""
+RUNTIME_CGROUP=""
 for sock in /run/containerd/containerd.sock /var/run/containerd/containerd.sock /var/run/crio/crio.sock; do
   if [[ -S "${sock}" ]]; then
     RUNTIME_ENDPOINT="unix://${sock}"
+    case "${sock}" in
+      */crio.sock)       RUNTIME_CGROUP="/system.slice/crio.service" ;;
+      */containerd.sock) RUNTIME_CGROUP="/system.slice/containerd.service" ;;
+    esac
     break
   fi
 done
@@ -109,6 +114,7 @@ if [[ -z "${RUNTIME_ENDPOINT}" ]]; then
   exit 1
 fi
 echo "==> Using container runtime endpoint: ${RUNTIME_ENDPOINT}"
+echo "==> Using runtime cgroup: ${RUNTIME_CGROUP}"
 
 # Locate the kubelet binary that was baked into the image; the test framework
 # will exec it.
@@ -131,7 +137,7 @@ set +e
 sudo -E "${E2E_NODE_TEST}" \
   --node-name="${NODE_NAME}" \
   --standalone-mode=true \
-  --kubelet-flags="--kubelet-cgroups=/kubelet.slice --cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=/system.slice/containerd.service" \
+  --kubelet-flags="--kubelet-cgroups=/kubelet.slice --cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=${RUNTIME_CGROUP}" \
   --container-runtime-endpoint="${RUNTIME_ENDPOINT}" \
   --ginkgo.focus="${GINKGO_FOCUS}" \
   --ginkgo.skip="${GINKGO_SKIP}" \

From 06ec6f522870a1d5e706471c85cf051776733df4 Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 14:54:52 -0600
Subject: [PATCH 4/7] Distinguish SSH transport failures from node-conformance
 failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously the node-conformance step used continue-on-error: true,
which masked all failures equally — including SSH/SCP transport,
auth, and network problems — as ordinary conformance test failures.

Drop continue-on-error and handle exit codes explicitly:
- scp helper-up failures fail the step (transport error).
- ssh exit 255 (its own transport/auth errors) fails the step.
- Any other non-zero remote exit is treated as a conformance result
  and surfaced as a workflow warning without failing the job.
- scp results-back failure is a warning only (we already have a
  conformance verdict at that point).
---
 .github/workflows/build-azure-sig.yaml | 33 +++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-azure-sig.yaml b/.github/workflows/build-azure-sig.yaml
index 6fa34ac420..717aa3d8a5 100644
--- a/.github/workflows/build-azure-sig.yaml
+++ b/.github/workflows/build-azure-sig.yaml
@@ -566,10 +566,13 @@ jobs:
           echo "All smoke tests passed"
 
       # -- Node conformance tests (opt-in, Linux only, non-blocking) -----------
+      # Conformance test failures are tolerated (the suite is informational
+      # while we iterate). SSH/SCP transport failures are NOT tolerated — they
+      # indicate infra/auth/network problems that would otherwise be masked as
+      # ordinary conformance failures.
       - name: Run node conformance tests
         id: node_conformance
         if: ${{ inputs.run_node_conformance && steps.vars.outputs.OS_TYPE != 'Windows' }}
-        continue-on-error: true
         env:
           OS_TYPE: ${{ steps.vars.outputs.OS_TYPE }}
         run: |
@@ -580,24 +583,42 @@ jobs:
           SSH_CMD="ssh ${SSH_OPTS} capi@${PUBLIC_IP}"
 
           echo "::group::Copy node-conformance helper to VM"
-          ${SCP_CMD} images/capi/hack/node-conformance.sh "capi@${PUBLIC_IP}:/tmp/node-conformance.sh"
-          ${SSH_CMD} 'chmod +x /tmp/node-conformance.sh'
+          if ! ${SCP_CMD} images/capi/hack/node-conformance.sh "capi@${PUBLIC_IP}:/tmp/node-conformance.sh"; then
+            echo "::error::Failed to scp node-conformance.sh to VM (transport failure)"
+            exit 1
+          fi
+          if ! ${SSH_CMD} 'chmod +x /tmp/node-conformance.sh'; then
+            echo "::error::Failed to chmod helper script on VM (transport failure)"
+            exit 1
+          fi
           echo "::endgroup::"
 
           echo "::group::Run node conformance"
-          # Run the suite; tolerate non-zero exit so we can still pull artifacts.
+          # ssh exits 255 only on its own transport/auth errors; any other
+          # non-zero exit comes from the remote command (the conformance run
+          # itself).  Distinguish the two so transport problems don't get
+          # silently classified as test failures.
           set +e
           ${SSH_CMD} "KUBERNETES_VERSION='${KUBERNETES_VERSION}' /tmp/node-conformance.sh"
           NC_EXIT=$?
           set -e
-          echo "node-conformance exited with ${NC_EXIT}"
+
+          if [[ "${NC_EXIT}" -eq 255 ]]; then
+            echo "::error::SSH transport failure while running node conformance (exit 255); no conformance result available"
+            exit 1
+          fi
+          echo "node-conformance remote exit code: ${NC_EXIT}"
           echo "exit_code=${NC_EXIT}" >> "${GITHUB_OUTPUT}"
           echo "::endgroup::"
 
           echo "::group::Collect node-conformance results"
           ARTIFACTS="${GITHUB_WORKSPACE}/_artifacts/node-conformance"
           mkdir -p "${ARTIFACTS}"
-          ${SCP_CMD} -r "capi@${PUBLIC_IP}:/tmp/node-conformance-results/." "${ARTIFACTS}/" || true
+          # A failure here is also transport-level, but we already have a
+          # conformance verdict — warn but don't fail the job over missing logs.
+          if ! ${SCP_CMD} -r "capi@${PUBLIC_IP}:/tmp/node-conformance-results/." "${ARTIFACTS}/"; then
+            echo "::warning::Failed to scp node-conformance results back from VM"
+          fi
           ls -la "${ARTIFACTS}" || true
           echo "::endgroup::"
 

From 7827326e3a0b6e1de30d067e07cba74e0c9333ba Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 15:03:36 -0600
Subject: [PATCH 5/7] Drop hard-coded --kubelet-cgroups=/kubelet.slice flag

systemd materializes a slice only when something is first launched
into it. On a fresh image-builder VM the system kubelet has never
been started, so /kubelet.slice may not exist when the test kubelet
is launched. Letting kubelet pick its own cgroup avoids a needless
assumption and works on systems where the slice does exist as well.
---
 images/capi/hack/node-conformance.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
index 6106b29161..595b21afab 100755
--- a/images/capi/hack/node-conformance.sh
+++ b/images/capi/hack/node-conformance.sh
@@ -137,7 +137,7 @@ set +e
 sudo -E "${E2E_NODE_TEST}" \
   --node-name="${NODE_NAME}" \
   --standalone-mode=true \
-  --kubelet-flags="--kubelet-cgroups=/kubelet.slice --cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=${RUNTIME_CGROUP}" \
+  --kubelet-flags="--cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=${RUNTIME_CGROUP}" \
   --container-runtime-endpoint="${RUNTIME_ENDPOINT}" \
   --ginkgo.focus="${GINKGO_FOCUS}" \
   --ginkgo.skip="${GINKGO_SKIP}" \

From 75dd2fc353d714eed885d1418fd1cf5ae25a92bf Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Wed, 13 May 2026 16:29:48 -0600
Subject: [PATCH 6/7] Fix node conformance kubelet discovery and exit-code
 disambiguation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First real run surfaced three issues:

1. e2e_node.test died with "Could not locate kubelet binary" because
   the framework searches dirname(argv[0]) and a Kubernetes source
   checkout for the kubelet binary, neither of which contains it on a
   prebuilt image. Pass --k8s-bin-dir=$(dirname $KUBELET_BIN) so the
   framework execs the version-matched kubelet that's already on PATH.

2. klog.Fatalf exits with 255 — the same code ssh uses for its own
   transport/auth failures. Add an EXIT trap in the helper that
   translates remote exit 255 -> 254, so the workflow can distinguish
   conformance failures from SSH transport problems.

3. The workflow exited on the suspected SSH-transport path before
   trying to scp results back, which destroyed the partial logs that
   would have made triage easier. Move the results-scp ahead of the
   exit-decision so we always attempt to retrieve whatever was
   produced before reporting failure.
---
 .github/workflows/build-azure-sig.yaml | 22 +++++++++++-----------
 images/capi/hack/node-conformance.sh   | 22 +++++++++++++++++++++-
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build-azure-sig.yaml b/.github/workflows/build-azure-sig.yaml
index 717aa3d8a5..b0499f286c 100644
--- a/.github/workflows/build-azure-sig.yaml
+++ b/.github/workflows/build-azure-sig.yaml
@@ -594,34 +594,34 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::Run node conformance"
-          # ssh exits 255 only on its own transport/auth errors; any other
-          # non-zero exit comes from the remote command (the conformance run
-          # itself).  Distinguish the two so transport problems don't get
-          # silently classified as test failures.
+          # ssh exits 255 only on its own transport/auth errors; the remote
+          # script translates klog.Fatalf's exit 255 to 254 so we can tell the
+          # two apart.
           set +e
           ${SSH_CMD} "KUBERNETES_VERSION='${KUBERNETES_VERSION}' /tmp/node-conformance.sh"
           NC_EXIT=$?
           set -e
-
-          if [[ "${NC_EXIT}" -eq 255 ]]; then
-            echo "::error::SSH transport failure while running node conformance (exit 255); no conformance result available"
-            exit 1
-          fi
           echo "node-conformance remote exit code: ${NC_EXIT}"
           echo "exit_code=${NC_EXIT}" >> "${GITHUB_OUTPUT}"
           echo "::endgroup::"
 
+          # Always try to pull whatever results exist BEFORE deciding whether
+          # to fail the step — partial results are valuable for triage even on
+          # a hard failure (e.g. the framework dying early).
           echo "::group::Collect node-conformance results"
           ARTIFACTS="${GITHUB_WORKSPACE}/_artifacts/node-conformance"
           mkdir -p "${ARTIFACTS}"
-          # A failure here is also transport-level, but we already have a
-          # conformance verdict — warn but don't fail the job over missing logs.
           if ! ${SCP_CMD} -r "capi@${PUBLIC_IP}:/tmp/node-conformance-results/." "${ARTIFACTS}/"; then
             echo "::warning::Failed to scp node-conformance results back from VM"
           fi
           ls -la "${ARTIFACTS}" || true
           echo "::endgroup::"
 
+          if [[ "${NC_EXIT}" -eq 255 ]]; then
+            echo "::error::SSH transport failure while running node conformance (exit 255); no conformance result available"
+            exit 1
+          fi
+
           if [[ "${NC_EXIT}" -ne 0 ]]; then
             echo "::warning::node conformance tests reported failures (exit ${NC_EXIT}); not blocking the workflow"
           fi
diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
index 595b21afab..e5ab5787b5 100755
--- a/images/capi/hack/node-conformance.sh
+++ b/images/capi/hack/node-conformance.sh
@@ -44,7 +44,21 @@ TARBALL_URL="${TARBALL_URL:-https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-
 TARBALL_SHA256_URL="${TARBALL_SHA256_URL:-${TARBALL_URL}.sha256}"
 
 WORKDIR="$(mktemp -d)"
-trap 'rm -rf "${WORKDIR}"' EXIT
+
+# klog.Fatalf inside e2e_node.test exits with 255 — the same code ssh uses
+# for its own transport/auth errors. Translate 255 -> 254 on the way out so
+# the calling workflow can tell the two apart.
+# shellcheck disable=SC2329  # invoked via trap
+cleanup() {
+  local rc=$?
+  rm -rf "${WORKDIR}"
+  if [[ "${rc}" -eq 255 ]]; then
+    echo "==> Translating exit 255 -> 254 to disambiguate from SSH transport failure" >&2
+    exit 254
+  fi
+  exit "${rc}"
+}
+trap cleanup EXIT
 
 echo "==> Node conformance: kubernetes v${KUBERNETES_VERSION} (${GO_ARCH})"
 echo "    focus:   ${GINKGO_FOCUS}"
@@ -133,10 +147,16 @@ echo "==> Using kubelet binary: ${KUBELET_BIN}"
 NODE_NAME="$(hostname)"
 
 echo "==> Running e2e_node.test"
+# --k8s-bin-dir tells the test framework where to find the kubelet binary
+# it will exec under test (the version-matched binary already baked into
+# the image).  Without it the framework falls back to looking for a
+# kubernetes source checkout at _output/local/go/bin/kubelet.
+KUBELET_BIN_DIR="$(dirname "${KUBELET_BIN}")"
 set +e
 sudo -E "${E2E_NODE_TEST}" \
   --node-name="${NODE_NAME}" \
   --standalone-mode=true \
+  --k8s-bin-dir="${KUBELET_BIN_DIR}" \
   --kubelet-flags="--cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=${RUNTIME_CGROUP}" \
   --container-runtime-endpoint="${RUNTIME_ENDPOINT}" \
   --ginkgo.focus="${GINKGO_FOCUS}" \

From c79d26c056ce008b6f64a97403f457c7abc4cd98 Mon Sep 17 00:00:00 2001
From: Matt Boersma <Matt.Boersma@microsoft.com>
Date: Thu, 14 May 2026 16:00:43 -0600
Subject: [PATCH 7/7] Install a minimal CNI config so the test kubelet reaches
 Ready
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First successful run produced 4 pass / 19 fail / 742 skip over 2h with
kubelet.log showing 1,425 copies of:

  Container runtime network not ready ... cni plugin not initialized

The node never reaches Ready, so pods do not schedule and almost every
NodeConformance test times out waiting on its pod.

CAPI images ship CNI plugin binaries under /opt/cni/bin but no CNI
config under /etc/cni/net.d — that file is normally written by the CNI
provider on cluster join. For standalone node-conformance there's no
provider, so write a minimal bridge + loopback config ourselves before
launching e2e_node.test. Pre-existing CNI config is left alone so we
don't conflict with a real CNI install. Plugin binaries are downloaded
from containernetworking/plugins only when missing from the image.
---
 images/capi/hack/node-conformance.sh | 49 ++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh
index e5ab5787b5..980857349a 100755
--- a/images/capi/hack/node-conformance.sh
+++ b/images/capi/hack/node-conformance.sh
@@ -130,6 +130,55 @@ fi
 echo "==> Using container runtime endpoint: ${RUNTIME_ENDPOINT}"
 echo "==> Using runtime cgroup: ${RUNTIME_CGROUP}"
 
+# The test kubelet won't mark the node Ready without a CNI plugin
+# configured (NetworkReady=false -> no pods schedule -> every networking
+# test times out). On CAPI images the plugin binaries live under
+# /opt/cni/bin but no CNI config is written until a CNI provider joins
+# the node to a cluster. For standalone node-conformance, install a
+# minimal bridge + loopback configuration. Leave any existing config
+# alone so we don't fight a real CNI install.
+echo "==> Ensuring minimal CNI configuration"
+sudo mkdir -p /etc/cni/net.d /opt/cni/bin
+if compgen -G "/etc/cni/net.d/*.conf*" >/dev/null; then
+  echo "    existing CNI config present, leaving as-is"
+else
+  echo "    no CNI config found; writing minimal bridge + loopback configs"
+  sudo tee /etc/cni/net.d/10-node-conformance.conflist >/dev/null <<'JSON'
+{
+  "cniVersion": "1.0.0",
+  "name": "node-conformance",
+  "plugins": [
+    {
+      "type": "bridge",
+      "bridge": "cni0",
+      "isGateway": true,
+      "ipMasq": true,
+      "promiscMode": true,
+      "ipam": {
+        "type": "host-local",
+        "ranges": [[{"subnet": "10.88.0.0/16"}]],
+        "routes": [{"dst": "0.0.0.0/0"}]
+      }
+    },
+    { "type": "portmap", "capabilities": {"portMappings": true} }
+  ]
+}
+JSON
+  sudo tee /etc/cni/net.d/99-loopback.conf >/dev/null <<'JSON'
+{ "cniVersion": "1.0.0", "name": "lo", "type": "loopback" }
+JSON
+fi
+
+# Download CNI plugin binaries only if the image is missing them.
+if [[ ! -x /opt/cni/bin/bridge ]] || [[ ! -x /opt/cni/bin/loopback ]] || [[ ! -x /opt/cni/bin/portmap ]] || [[ ! -x /opt/cni/bin/host-local ]]; then
+  CNI_VERSION="${CNI_VERSION:-v1.5.1}"
+  echo "    CNI plugin binaries missing; downloading ${CNI_VERSION}"
+  curl --fail --silent --show-error --location \
+    -o "${WORKDIR}/cni.tgz" \
+    "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${GO_ARCH}-${CNI_VERSION}.tgz"
+  sudo tar -xzf "${WORKDIR}/cni.tgz" -C /opt/cni/bin/
+fi
+
 # Locate the kubelet binary that was baked into the image; the test framework
 # will exec it.
 KUBELET_BIN="$(command -v kubelet || true)"