diff --git a/.github/workflows/build-azure-sig.yaml b/.github/workflows/build-azure-sig.yaml index 1c86351907..b0499f286c 100644 --- a/.github/workflows/build-azure-sig.yaml +++ b/.github/workflows/build-azure-sig.yaml @@ -26,6 +26,7 @@ # - skip_test - skip the test stage # - skip_promote - skip the promote stage # - packer_debug - enable Packer debug logging (sets PACKER_LOG=1) +# - run_node_conformance - run Kubernetes node conformance tests on the test VM (Linux only) name: Build Azure SIG Image @@ -96,6 +97,11 @@ on: required: false type: boolean default: false + run_node_conformance: + description: 'Run Kubernetes node conformance tests on the test VM (Linux only, opt-in, non-blocking)' + required: false + type: boolean + default: false skip_promote: description: 'Skip the promote stage (requires manual approval)' required: false @@ -281,7 +287,7 @@ jobs: needs: build if: ${{ !inputs.skip_test }} runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: ${{ inputs.run_node_conformance && 150 || 15 }} env: KUBERNETES_VERSION: ${{ inputs.kubernetes_version }} @@ -559,6 +565,76 @@ jobs: fi echo "All smoke tests passed" + # -- Node conformance tests (opt-in, Linux only, non-blocking) ----------- + # Conformance test failures are tolerated (the suite is informational + # while we iterate). SSH/SCP transport failures are NOT tolerated — they + # indicate infra/auth/network problems that would otherwise be masked as + # ordinary conformance failures. + - name: Run node conformance tests + id: node_conformance + if: ${{ inputs.run_node_conformance && steps.vars.outputs.OS_TYPE != 'Windows' }} + env: + OS_TYPE: ${{ steps.vars.outputs.OS_TYPE }} + run: | + set -euo pipefail + + SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${TMPDIR}/sshkey" + SCP_CMD="scp ${SSH_OPTS}" + SSH_CMD="ssh ${SSH_OPTS} capi@${PUBLIC_IP}" + + echo "::group::Copy node-conformance helper to VM" + if ! ${SCP_CMD} images/capi/hack/node-conformance.sh "capi@${PUBLIC_IP}:/tmp/node-conformance.sh"; then + echo "::error::Failed to scp node-conformance.sh to VM (transport failure)" + exit 1 + fi + if ! ${SSH_CMD} 'chmod +x /tmp/node-conformance.sh'; then + echo "::error::Failed to chmod helper script on VM (transport failure)" + exit 1 + fi + echo "::endgroup::" + + echo "::group::Run node conformance" + # ssh exits 255 only on its own transport/auth errors; the remote + # script translates klog.Fatalf's exit 255 to 254 so we can tell the + # two apart. + set +e + ${SSH_CMD} "KUBERNETES_VERSION='${KUBERNETES_VERSION}' /tmp/node-conformance.sh" + NC_EXIT=$? + set -e + echo "node-conformance remote exit code: ${NC_EXIT}" + echo "exit_code=${NC_EXIT}" >> "${GITHUB_OUTPUT}" + echo "::endgroup::" + + # Always try to pull whatever results exist BEFORE deciding whether + # to fail the step — partial results are valuable for triage even on + # a hard failure (e.g. the framework dying early). + echo "::group::Collect node-conformance results" + ARTIFACTS="${GITHUB_WORKSPACE}/_artifacts/node-conformance" + mkdir -p "${ARTIFACTS}" + if ! ${SCP_CMD} -r "capi@${PUBLIC_IP}:/tmp/node-conformance-results/." "${ARTIFACTS}/"; then + echo "::warning::Failed to scp node-conformance results back from VM" + fi + ls -la "${ARTIFACTS}" || true + echo "::endgroup::" + + if [[ "${NC_EXIT}" -eq 255 ]]; then + echo "::error::SSH transport failure while running node conformance (exit 255); no conformance result available" + exit 1 + fi + + if [[ "${NC_EXIT}" -ne 0 ]]; then + echo "::warning::node conformance tests reported failures (exit ${NC_EXIT}); not blocking the workflow" + fi + + - name: Upload node conformance results + if: ${{ always() && steps.node_conformance.conclusion != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-conformance-results + path: _artifacts/node-conformance/ + retention-days: 7 + if-no-files-found: warn + # -- Collect diagnostics on failure -------------------------------------- - name: Collect diagnostics if: failure() diff --git a/images/capi/hack/node-conformance.sh b/images/capi/hack/node-conformance.sh new file mode 100755 index 0000000000..980857349a --- /dev/null +++ b/images/capi/hack/node-conformance.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 +# +# Run the Kubernetes node-conformance subset of the e2e_node test suite on the +# local machine. Intended to be executed on a freshly-booted VM created from a +# CAPI image-builder image, _not_ on a node that is already part of a cluster. +# +# This is a best-effort runner that pulls the official versioned kubernetes-test +# tarball from dl.k8s.io (matched to the Kubernetes version baked into the +# image) and invokes ./e2e_node.test in standalone mode so the framework can +# start its own kubelet against the system's container runtime. +# +# Results (JUnit XML + ginkgo logs) are written to /tmp/node-conformance-results. +# +# Environment variables: +# KUBERNETES_VERSION Required. e.g. "1.31.1" (no leading "v") +# GINKGO_FOCUS Optional. Default '\[NodeConformance\]' +# GINKGO_SKIP Optional. Default '\[Flaky\]|\[Serial\]|\[Slow\]|\[Alpha\]' +# RESULTS_DIR Optional. Default /tmp/node-conformance-results +# TARBALL_URL Optional. Override the test tarball URL. +# TARBALL_SHA256_URL Optional. Override the checksum URL. Defaults to +# "${TARBALL_URL}.sha256". The remote file is expected +# to contain just the hex digest (as dl.k8s.io serves). +# +# Exit code is the exit code of e2e_node.test (non-zero on any failed spec). + +set -o errexit +set -o nounset +set -o pipefail + +KUBERNETES_VERSION="${KUBERNETES_VERSION:?KUBERNETES_VERSION is required, e.g. 1.31.1}" +GINKGO_FOCUS="${GINKGO_FOCUS:-\[NodeConformance\]}" +GINKGO_SKIP="${GINKGO_SKIP:-\[Flaky\]|\[Serial\]|\[Slow\]|\[Alpha\]}" +RESULTS_DIR="${RESULTS_DIR:-/tmp/node-conformance-results}" + +ARCH="$(uname -m)" +case "${ARCH}" in + x86_64) GO_ARCH="amd64" ;; + aarch64) GO_ARCH="arm64" ;; + *) echo "unsupported arch: ${ARCH}" >&2; exit 1 ;; +esac + +TARBALL_URL="${TARBALL_URL:-https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-test-linux-${GO_ARCH}.tar.gz}" +TARBALL_SHA256_URL="${TARBALL_SHA256_URL:-${TARBALL_URL}.sha256}" + +WORKDIR="$(mktemp -d)" + +# klog.Fatalf inside e2e_node.test exits with 255 — the same code ssh uses +# for its own transport/auth errors. Translate 255 -> 254 on the way out so +# the calling workflow can tell the two apart. +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + local rc=$? + rm -rf "${WORKDIR}" + if [[ "${rc}" -eq 255 ]]; then + echo "==> Translating exit 255 -> 254 to disambiguate from SSH transport failure" >&2 + exit 254 + fi + exit "${rc}" +} +trap cleanup EXIT + +echo "==> Node conformance: kubernetes v${KUBERNETES_VERSION} (${GO_ARCH})" +echo " focus: ${GINKGO_FOCUS}" +echo " skip: ${GINKGO_SKIP}" +echo " results: ${RESULTS_DIR}" +echo " tarball: ${TARBALL_URL}" + +mkdir -p "${RESULTS_DIR}" +sudo chown "$(id -u):$(id -g)" "${RESULTS_DIR}" + +echo "==> Downloading test tarball" +curl --fail --silent --show-error --location \ + --output "${WORKDIR}/test.tar.gz" "${TARBALL_URL}" + +echo "==> Downloading tarball SHA-256 checksum" +curl --fail --silent --show-error --location \ + --output "${WORKDIR}/test.tar.gz.sha256" "${TARBALL_SHA256_URL}" + +echo "==> Verifying tarball checksum" +# dl.k8s.io publishes the bare hex digest (no filename); build a sha256sum +# compatible line so we can use the standard verifier. +EXPECTED_SHA256="$(tr -d '[:space:]' < "${WORKDIR}/test.tar.gz.sha256")" +if [[ -z "${EXPECTED_SHA256}" ]]; then + echo "ERROR: empty checksum retrieved from ${TARBALL_SHA256_URL}" >&2 + exit 1 +fi +echo "${EXPECTED_SHA256} test.tar.gz" > "${WORKDIR}/test.tar.gz.sha256sum" +( cd "${WORKDIR}" && sha256sum --check --strict --status test.tar.gz.sha256sum ) || { + ACTUAL_SHA256="$(sha256sum "${WORKDIR}/test.tar.gz" | awk '{print $1}')" + echo "ERROR: SHA-256 mismatch for ${TARBALL_URL}" >&2 + echo " expected: ${EXPECTED_SHA256}" >&2 + echo " actual: ${ACTUAL_SHA256}" >&2 + exit 1 +} +echo " OK (${EXPECTED_SHA256})" + +echo "==> Extracting e2e_node.test and ginkgo" +tar -xzf "${WORKDIR}/test.tar.gz" -C "${WORKDIR}" \ + kubernetes/test/bin/e2e_node.test \ + kubernetes/test/bin/ginkgo +E2E_NODE_TEST="${WORKDIR}/kubernetes/test/bin/e2e_node.test" +GINKGO_BIN="${WORKDIR}/kubernetes/test/bin/ginkgo" +chmod +x "${E2E_NODE_TEST}" "${GINKGO_BIN}" + +# The framework starts its own kubelet under test, so stop the system unit (if +# any) to free /var/lib/kubelet and the kubelet socket. Don't fail if absent. +if systemctl list-unit-files kubelet.service >/dev/null 2>&1; then + echo "==> Stopping system kubelet" + sudo systemctl stop kubelet || true +fi + +# Detect the system container runtime endpoint and its systemd cgroup path. +RUNTIME_ENDPOINT="" +RUNTIME_CGROUP="" +for sock in /run/containerd/containerd.sock /var/run/containerd/containerd.sock /var/run/crio/crio.sock; do + if [[ -S "${sock}" ]]; then + RUNTIME_ENDPOINT="unix://${sock}" + case "${sock}" in + */crio.sock) RUNTIME_CGROUP="/system.slice/crio.service" ;; + */containerd.sock) RUNTIME_CGROUP="/system.slice/containerd.service" ;; + esac + break + fi +done +if [[ -z "${RUNTIME_ENDPOINT}" ]]; then + echo "ERROR: no container runtime socket found" >&2 + exit 1 +fi +echo "==> Using container runtime endpoint: ${RUNTIME_ENDPOINT}" +echo "==> Using runtime cgroup: ${RUNTIME_CGROUP}" + +# The test kubelet won't mark the node Ready without a CNI plugin +# configured (NetworkReady=false -> no pods schedule -> every networking +# test times out). On CAPI images the plugin binaries live under +# /opt/cni/bin but no CNI config is written until a CNI provider joins +# the node to a cluster. For standalone node-conformance, install a +# minimal bridge + loopback configuration. Leave any existing config +# alone so we don't fight a real CNI install. +echo "==> Ensuring minimal CNI configuration" +sudo mkdir -p /etc/cni/net.d /opt/cni/bin +if compgen -G "/etc/cni/net.d/*.conf*" >/dev/null; then + echo " existing CNI config present, leaving as-is" +else + echo " no CNI config found; writing minimal bridge + loopback configs" + sudo tee /etc/cni/net.d/10-node-conformance.conflist >/dev/null <<'JSON' +{ + "cniVersion": "1.0.0", + "name": "node-conformance", + "plugins": [ + { + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "promiscMode": true, + "ipam": { + "type": "host-local", + "ranges": [[{"subnet": "10.88.0.0/16"}]], + "routes": [{"dst": "0.0.0.0/0"}] + } + }, + { "type": "portmap", "capabilities": {"portMappings": true} } + ] +} +JSON + sudo tee /etc/cni/net.d/99-loopback.conf >/dev/null <<'JSON' +{ "cniVersion": "1.0.0", "name": "lo", "type": "loopback" } +JSON +fi + +# Download CNI plugin binaries only if the image is missing them. +if [[ ! -x /opt/cni/bin/bridge ]] || [[ ! -x /opt/cni/bin/loopback ]] || [[ ! -x /opt/cni/bin/portmap ]] || [[ ! -x /opt/cni/bin/host-local ]]; then + CNI_VERSION="${CNI_VERSION:-v1.5.1}" + echo " CNI plugin binaries missing; downloading ${CNI_VERSION}" + curl --fail --silent --show-error --location \ + -o "${WORKDIR}/cni.tgz" \ + "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${GO_ARCH}-${CNI_VERSION}.tgz" + sudo tar -xzf "${WORKDIR}/cni.tgz" -C /opt/cni/bin/ +fi + +# Locate the kubelet binary that was baked into the image; the test framework +# will exec it. +KUBELET_BIN="$(command -v kubelet || true)" +if [[ -z "${KUBELET_BIN}" ]]; then + for candidate in /usr/local/bin/kubelet /usr/bin/kubelet /opt/bin/kubelet; do + if [[ -x "${candidate}" ]]; then KUBELET_BIN="${candidate}"; break; fi + done +fi +if [[ -z "${KUBELET_BIN}" ]]; then + echo "ERROR: kubelet binary not found on PATH" >&2 + exit 1 +fi +echo "==> Using kubelet binary: ${KUBELET_BIN}" + +NODE_NAME="$(hostname)" + +echo "==> Running e2e_node.test" +# --k8s-bin-dir tells the test framework where to find the kubelet binary +# it will exec under test (the version-matched binary already baked into +# the image). Without it the framework falls back to looking for a +# kubernetes source checkout at _output/local/go/bin/kubelet. +KUBELET_BIN_DIR="$(dirname "${KUBELET_BIN}")" +set +e +sudo -E "${E2E_NODE_TEST}" \ + --node-name="${NODE_NAME}" \ + --standalone-mode=true \ + --k8s-bin-dir="${KUBELET_BIN_DIR}" \ + --kubelet-flags="--cgroup-driver=systemd --container-runtime-endpoint=${RUNTIME_ENDPOINT} --runtime-cgroups=${RUNTIME_CGROUP}" \ + --container-runtime-endpoint="${RUNTIME_ENDPOINT}" \ + --ginkgo.focus="${GINKGO_FOCUS}" \ + --ginkgo.skip="${GINKGO_SKIP}" \ + --ginkgo.timeout=2h \ + --ginkgo.v \ + --report-dir="${RESULTS_DIR}" \ + --report-prefix="node-conformance" \ + 2>&1 | tee "${RESULTS_DIR}/e2e_node.log" +EXIT_CODE=${PIPESTATUS[0]} +set -e + +# Make results readable for the calling user (scp back). +sudo chown -R "$(id -u):$(id -g)" "${RESULTS_DIR}" || true + +echo "==> e2e_node.test exited with ${EXIT_CODE}" +exit "${EXIT_CODE}"