From 38f53dcc21bf6047f6758d267b6008c146086dd1 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Fri, 4 Sep 2026 21:06:37 +0200 Subject: [PATCH 1/2] capi: factor shared qemu guest helpers into hack/lib hack/qemu-boot-smoke.sh already knows how to boot a built image from a throwaway copy-on-write overlay and reach it over SSH. Node conformance needs the same mechanics, so move them into hack/lib/qemu-guest.sh rather than growing a second copy: image resolution, accelerator detection, overlay creation, NoCloud seed ISO generation, daemonized boot, and the SSH wait, exec and copy helpers. Two defects are fixed while moving the code. qemu_guest_resolve_image now returns instead of exiting, and the new qemu_guest_resolve_image_path resolves into a variable before making the path absolute. Nesting the two as image="$(qemu_guest_abs_path "$(qemu_guest_resolve_image "${arg}")")" discards the inner status once the outer command runs, so an unresolvable image turned into the working directory and the caller carried on. The QEMU argument arrays are expanded as ${array[@]+"${array[@]}"}, and a trailing "--" no longer captures "${@}" bare. Both are empty in the common case, and bash before 4.4 treats an empty array or an empty "${@}" as unset under nounset, which aborted the run before QEMU started. The script also traps INT and TERM so that interrupting the SSH wait still stops the guest and removes the overlay, and it keeps its existing command line and environment contract. --- images/capi/hack/lib/qemu-guest.sh | 346 ++++++++++++++++++ images/capi/hack/qemu-boot-smoke.sh | 300 +++------------ .../qemu/scripts/qemu_guest_lib_test.py | 141 +++++++ 3 files changed, 534 insertions(+), 253 deletions(-) create mode 100644 images/capi/hack/lib/qemu-guest.sh create mode 100644 images/capi/packer/qemu/scripts/qemu_guest_lib_test.py diff --git a/images/capi/hack/lib/qemu-guest.sh b/images/capi/hack/lib/qemu-guest.sh new file mode 100644 index 0000000000..69beb8a3fc --- /dev/null +++ b/images/capi/hack/lib/qemu-guest.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared helpers for booting a built QEMU image on a throwaway copy-on-write +# overlay and driving it over SSH. Sourced by hack/qemu-boot-smoke.sh and +# hack/qemu-node-conformance.sh; it only defines functions. +# +# Callers set the QEMU_GUEST_* globals documented on each function before +# calling it. + +qemu_guest_require_command() { + if ! command -v "${1}" >/dev/null 2>&1; then + echo "${1} must be in PATH" >&2 + exit 1 + fi +} + +qemu_guest_abs_path() { + local path="${1}" + local dir + local base + + dir="$(dirname "${path}")" + base="$(basename "${path}")" + echo "$(cd "${dir}" && pwd -P)/${base}" +} + +# qemu_guest_resolve_image accepts either a disk image or a Packer output +# directory holding exactly one disk image, and prints the image path. It +# returns non-zero rather than exiting, so that a caller running it in a command +# substitution can act on the failure. +qemu_guest_resolve_image() { + local input="${1}" + local matches + local count + + if [[ -d "${input}" ]]; then + matches="$(find "${input}" -maxdepth 1 -type f \( -name "*.qcow2" -o -name "*.raw" -o -name "*.img" \) -print | sort)" + count="$(printf '%s\n' "${matches}" | sed '/^$/d' | wc -l | tr -d ' ')" + if [[ "${count}" != "1" ]]; then + echo "expected exactly one *.qcow2, *.raw, or *.img file in ${input}; found ${count}" >&2 + return 1 + fi + printf '%s\n' "${matches}" + return 0 + fi + + if [[ ! -f "${input}" ]]; then + echo "image does not exist: ${input}" >&2 + return 1 + fi + + printf '%s\n' "${input}" +} + +# qemu_guest_resolve_image_path prints the absolute path of the image to boot. +# +# Callers must not nest the two steps as +# "$(qemu_guest_abs_path "$(qemu_guest_resolve_image ...)")": the status of an +# inner command substitution is discarded once the outer command runs, so a +# failed resolve would be reported as success with the working directory as the +# image path. Resolving into a variable first keeps the failure observable. +qemu_guest_resolve_image_path() { + local input="${1}" + local image + + image="$(qemu_guest_resolve_image "${input}")" || return 1 + if [[ -z "${image}" ]]; then + echo "could not resolve an image path from: ${input}" >&2 + return 1 + fi + + qemu_guest_abs_path "${image}" +} + +qemu_guest_normalize_arch() { + case "${1}" in + x86_64 | amd64) + echo x86_64 + ;; + aarch64 | arm64) + echo aarch64 + ;; + *) + echo "${1}" + ;; + esac +} + +qemu_guest_binary_arch() { + case "$(basename "${1}")" in + qemu-system-x86_64) + echo x86_64 + ;; + qemu-system-aarch64) + echo aarch64 + ;; + *) + echo "" + ;; + esac +} + +# qemu_guest_detect_accelerator picks a default accelerator for the given QEMU +# binary. hvf and kvm both require the QEMU binary's target architecture to +# match the host architecture; e.g. running qemu-system-x86_64 on an arm64 macOS +# host to boot an amd64 image cannot use hvf and must fall back to tcg. +qemu_guest_detect_accelerator() { + local qemu_binary="${1}" + local host_arch + local binary_arch + + host_arch="$(qemu_guest_normalize_arch "$(uname -m)")" + binary_arch="$(qemu_guest_binary_arch "${qemu_binary}")" + + case "$(uname -s)" in + Linux) + if [[ -z "${binary_arch}" || "${binary_arch}" != "${host_arch}" ]]; then + echo tcg + elif [[ -r /dev/kvm && -w /dev/kvm ]]; then + echo kvm + else + echo tcg + fi + ;; + Darwin) + if [[ -z "${binary_arch}" || "${binary_arch}" != "${host_arch}" ]]; then + echo tcg + else + echo hvf + fi + ;; + *) + echo tcg + ;; + esac +} + +# qemu_guest_detect_image_format prints the format of an image. Arguments: +# qemu-img binary, image path. +qemu_guest_detect_image_format() { + local qemu_img="${1}" + local image="${2}" + local format + + qemu_guest_require_command python3 + format="$("${qemu_img}" info --output=json "${image}" | python3 -c 'import json, sys; print(json.load(sys.stdin).get("format", ""))')" + if [[ -z "${format}" ]]; then + echo "could not detect image format for ${image}; set QEMU_IMAGE_FORMAT" >&2 + exit 1 + fi + echo "${format}" +} + +# qemu_guest_create_overlay creates a qcow2 copy-on-write overlay so the built +# image is never written to. Arguments: qemu-img binary, backing image, backing +# format, overlay path. +qemu_guest_create_overlay() { + local qemu_img="${1}" + local image="${2}" + local backing_format="${3}" + local overlay="${4}" + + "${qemu_img}" create -f qcow2 -F "${backing_format}" -b "${image}" "${overlay}" >/dev/null +} + +# qemu_guest_write_seed_iso builds a NoCloud seed ISO that creates the SSH user. +# Arguments: seed directory, ISO path, user name, public key, instance name. +qemu_guest_write_seed_iso() { + local seed_dir="${1}" + local seed_iso="${2}" + local user="${3}" + local public_key="${4}" + local instance="${5}" + + mkdir -p "${seed_dir}" + cat >"${seed_dir}/meta-data" <"${seed_dir}/user-data" </dev/null 2>&1; then + cloud-localds "${seed_iso}" "${seed_dir}/user-data" "${seed_dir}/meta-data" + elif command -v genisoimage >/dev/null 2>&1; then + (cd "${seed_dir}" && genisoimage -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) + elif command -v mkisofs >/dev/null 2>&1; then + (cd "${seed_dir}" && mkisofs -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) + elif command -v xorriso >/dev/null 2>&1; then + (cd "${seed_dir}" && xorriso -as mkisofs -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) + elif command -v hdiutil >/dev/null 2>&1; then + hdiutil makehybrid -o "${seed_iso}" -hfs -joliet -iso -default-volume-name cidata "${seed_dir}" >/dev/null + else + echo "cloud-localds, genisoimage, mkisofs, xorriso, or hdiutil is required to create the seed ISO" >&2 + exit 1 + fi +} + +qemu_guest_stop() { + local pid="${1:-}" + + if [[ -z "${pid}" ]]; then + return + fi + if ! kill -0 "${pid}" >/dev/null 2>&1; then + return + fi + kill "${pid}" >/dev/null 2>&1 || true + sleep 2 + if kill -0 "${pid}" >/dev/null 2>&1; then + kill -9 "${pid}" >/dev/null 2>&1 || true + fi +} + +# qemu_guest_start boots the guest daemonized and sets QEMU_GUEST_PID. +# Globals: QEMU_GUEST_BINARY, QEMU_GUEST_ACCELERATOR, QEMU_GUEST_MACHINE, +# QEMU_GUEST_MEMORY, QEMU_GUEST_CPUS, QEMU_GUEST_DISK, QEMU_GUEST_SSH_PORT, +# QEMU_GUEST_SERIAL_LOG, QEMU_GUEST_PIDFILE, and the optional arrays +# QEMU_GUEST_SEED_ARGS and QEMU_GUEST_EXTRA_ARGS. +qemu_guest_start() { + "${QEMU_GUEST_BINARY}" \ + -accel "${QEMU_GUEST_ACCELERATOR}" \ + -machine "${QEMU_GUEST_MACHINE}" \ + -m "${QEMU_GUEST_MEMORY}" \ + -smp "${QEMU_GUEST_CPUS}" \ + -drive "file=${QEMU_GUEST_DISK},if=virtio,format=qcow2" \ + ${QEMU_GUEST_SEED_ARGS[@]+"${QEMU_GUEST_SEED_ARGS[@]}"} \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${QEMU_GUEST_SSH_PORT}-:22" \ + -device "virtio-net-pci,netdev=net0" \ + -display none \ + -serial "file:${QEMU_GUEST_SERIAL_LOG}" \ + -monitor none \ + -no-reboot \ + -pidfile "${QEMU_GUEST_PIDFILE}" \ + -daemonize \ + ${QEMU_GUEST_EXTRA_ARGS[@]+"${QEMU_GUEST_EXTRA_ARGS[@]}"} + + QEMU_GUEST_PID="$(cat "${QEMU_GUEST_PIDFILE}")" +} + +# qemu_guest_ssh runs a command in the guest. +# Globals: QEMU_GUEST_SSH_KEY, QEMU_GUEST_SSH_PORT, QEMU_GUEST_SSH_USER. +qemu_guest_ssh() { + ssh \ + -F /dev/null \ + -o BatchMode=yes \ + -o ConnectTimeout=5 \ + -o IdentitiesOnly=yes \ + -o LogLevel=ERROR \ + -o ServerAliveCountMax=20 \ + -o ServerAliveInterval=30 \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i "${QEMU_GUEST_SSH_KEY}" \ + -p "${QEMU_GUEST_SSH_PORT}" \ + "${QEMU_GUEST_SSH_USER}@127.0.0.1" \ + "${@}" +} + +# qemu_guest_scp copies files to or from the guest. Remote paths are written as +# guest:/path and are rewritten to the SSH destination. +qemu_guest_scp() { + local -a args=() + local arg + + for arg in ${@+"${@}"}; do + case "${arg}" in + guest:*) + args+=("${QEMU_GUEST_SSH_USER}@127.0.0.1:${arg#guest:}") + ;; + *) + args+=("${arg}") + ;; + esac + done + + scp \ + -F /dev/null \ + -o BatchMode=yes \ + -o ConnectTimeout=5 \ + -o IdentitiesOnly=yes \ + -o LogLevel=ERROR \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i "${QEMU_GUEST_SSH_KEY}" \ + -P "${QEMU_GUEST_SSH_PORT}" \ + -r \ + "${args[@]}" +} + +# qemu_guest_wait_for_ssh polls the guest until the given probe command +# succeeds over SSH. It fails early if QEMU exits and prints the head of the +# serial log on failure. Arguments: probe command. +# Globals: QEMU_GUEST_PID, QEMU_GUEST_SSH_TIMEOUT, QEMU_GUEST_SSH_INTERVAL, +# QEMU_GUEST_SSH_PORT, QEMU_GUEST_SERIAL_LOG. +qemu_guest_wait_for_ssh() { + local probe_command="${1}" + local deadline=$((SECONDS + QEMU_GUEST_SSH_TIMEOUT)) + + echo "Waiting up to ${QEMU_GUEST_SSH_TIMEOUT}s for SSH on 127.0.0.1:${QEMU_GUEST_SSH_PORT}..." + while ((SECONDS < deadline)); do + if ! kill -0 "${QEMU_GUEST_PID}" >/dev/null 2>&1; then + echo "QEMU exited before SSH became available" >&2 + qemu_guest_dump_serial_log + return 1 + fi + + if qemu_guest_ssh "${probe_command}" >/dev/null; then + return 0 + fi + + sleep "${QEMU_GUEST_SSH_INTERVAL}" + done + + echo "Timed out waiting for SSH on 127.0.0.1:${QEMU_GUEST_SSH_PORT}" >&2 + qemu_guest_dump_serial_log + return 1 +} + +qemu_guest_dump_serial_log() { + sed -n '1,160p' "${QEMU_GUEST_SERIAL_LOG}" >&2 || true +} diff --git a/images/capi/hack/qemu-boot-smoke.sh b/images/capi/hack/qemu-boot-smoke.sh index 9cdc0ad025..8171cf6e45 100755 --- a/images/capi/hack/qemu-boot-smoke.sh +++ b/images/capi/hack/qemu-boot-smoke.sh @@ -66,13 +66,19 @@ fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" capi_dir="$(cd "${script_dir}/.." && pwd -P)" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=lib/qemu-guest.sh +source "${script_dir}/lib/qemu-guest.sh" + image_arg="${1}" shift -qemu_extra_args=() +QEMU_GUEST_EXTRA_ARGS=() if [[ ${1:-} == "--" ]]; then shift - qemu_extra_args=("${@}") + # A trailing "--" with nothing after it leaves no positional parameters, and + # bash before 4.4 treats "${@}" as unset under nounset. + QEMU_GUEST_EXTRA_ARGS=(${@+"${@}"}) elif [[ $# -gt 0 ]]; then usage exit 1 @@ -93,189 +99,14 @@ QEMU_SMOKE_COMMAND="${QEMU_SMOKE_COMMAND:-true}" QEMU_SEED="${QEMU_SEED:-cloud-init}" QEMU_IMAGE_OS="${QEMU_IMAGE_OS:-}" -require_command() { - if ! command -v "${1}" >/dev/null 2>&1; then - echo "${1} must be in PATH" >&2 - exit 1 - fi -} - -abs_path() { - local path="${1}" - local dir - local base - - dir="$(dirname "${path}")" - base="$(basename "${path}")" - echo "$(cd "${dir}" && pwd -P)/${base}" -} - -is_flatcar_requested() { - [[ "${QEMU_IMAGE_OS}" == "flatcar" ]] -} - -resolve_image() { - local input="${1}" - local matches - local count - - if [[ -d "${input}" ]]; then - matches="$(find "${input}" -maxdepth 1 -type f \( -name "*.qcow2" -o -name "*.raw" -o -name "*.img" \) -print | sort)" - count="$(printf '%s\n' "${matches}" | sed '/^$/d' | wc -l | tr -d ' ')" - if [[ "${count}" != "1" ]]; then - echo "expected exactly one *.qcow2, *.raw, or *.img file in ${input}; found ${count}" >&2 - exit 1 - fi - printf '%s\n' "${matches}" - return - fi - - if [[ ! -f "${input}" ]]; then - echo "image does not exist: ${input}" >&2 - exit 1 - fi - - printf '%s\n' "${input}" -} - -normalize_arch() { - case "${1}" in - x86_64 | amd64) - echo x86_64 - ;; - aarch64 | arm64) - echo aarch64 - ;; - *) - echo "${1}" - ;; - esac -} - -qemu_binary_arch() { - case "$(basename "${1}")" in - qemu-system-x86_64) - echo x86_64 - ;; - qemu-system-aarch64) - echo aarch64 - ;; - *) - echo "" - ;; - esac -} - -# detect_accelerator picks a default accelerator for the given QEMU binary. -# hvf and kvm both require the QEMU binary's target architecture to match the -# host architecture; e.g. running qemu-system-x86_64 on an arm64 macOS host to -# boot an amd64 image cannot use hvf and must fall back to tcg. -detect_accelerator() { - local qemu_binary="${1}" - local host_arch - local binary_arch +qemu_guest_require_command "${QEMU_BINARY}" +qemu_guest_require_command "${QEMU_IMG}" +qemu_guest_require_command ssh - host_arch="$(normalize_arch "$(uname -m)")" - binary_arch="$(qemu_binary_arch "${qemu_binary}")" - - case "$(uname -s)" in - Linux) - if [[ -z "${binary_arch}" || "${binary_arch}" != "${host_arch}" ]]; then - echo tcg - elif [[ -r /dev/kvm && -w /dev/kvm ]]; then - echo kvm - else - echo tcg - fi - ;; - Darwin) - if [[ -z "${binary_arch}" || "${binary_arch}" != "${host_arch}" ]]; then - echo tcg - else - echo hvf - fi - ;; - *) - echo tcg - ;; - esac -} - -detect_image_format() { - local image="${1}" - local format - - require_command python3 - format="$("${QEMU_IMG}" info --output=json "${image}" | python3 -c 'import json, sys; print(json.load(sys.stdin).get("format", ""))')" - if [[ -z "${format}" ]]; then - echo "could not detect image format for ${image}; set QEMU_IMAGE_FORMAT" >&2 - exit 1 - fi - echo "${format}" -} - -write_seed_iso() { - local seed_dir="${1}" - local seed_iso="${2}" - local public_key="${3}" - - mkdir -p "${seed_dir}" - cat >"${seed_dir}/meta-data" <"${seed_dir}/user-data" </dev/null 2>&1; then - cloud-localds "${seed_iso}" "${seed_dir}/user-data" "${seed_dir}/meta-data" - elif command -v genisoimage >/dev/null 2>&1; then - (cd "${seed_dir}" && genisoimage -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) - elif command -v mkisofs >/dev/null 2>&1; then - (cd "${seed_dir}" && mkisofs -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) - elif command -v xorriso >/dev/null 2>&1; then - (cd "${seed_dir}" && xorriso -as mkisofs -output "${seed_iso}" -volid cidata -joliet -rock user-data meta-data >/dev/null) - elif command -v hdiutil >/dev/null 2>&1; then - hdiutil makehybrid -o "${seed_iso}" -hfs -joliet -iso -default-volume-name cidata "${seed_dir}" >/dev/null - else - echo "cloud-localds, genisoimage, mkisofs, xorriso, or hdiutil is required to create the seed ISO" >&2 - exit 1 - fi -} - -# shellcheck disable=SC2329 # Called from the EXIT trap. -stop_qemu() { - local pid="${1:-}" - - if [[ -z "${pid}" ]]; then - return - fi - if ! kill -0 "${pid}" >/dev/null 2>&1; then - return - fi - kill "${pid}" >/dev/null 2>&1 || true - sleep 2 - if kill -0 "${pid}" >/dev/null 2>&1; then - kill -9 "${pid}" >/dev/null 2>&1 || true - fi -} - -require_command "${QEMU_BINARY}" -require_command "${QEMU_IMG}" -require_command ssh - -image="$(abs_path "$(resolve_image "${image_arg}")")" -if is_flatcar_requested; then +if ! image="$(qemu_guest_resolve_image_path "${image_arg}")"; then + exit 1 +fi +if [[ "${QEMU_IMAGE_OS}" == "flatcar" ]]; then echo "qemu-boot-smoke.sh does not support Flatcar images: Flatcar uses Ignition, not cloud-init, and the build removes the SSH user before shutdown, so neither QEMU_SEED=cloud-init nor QEMU_SEED=none can authenticate. Image: ${image}" >&2 exit 1 fi @@ -285,95 +116,58 @@ if [[ ! -r "${QEMU_SSH_PRIVATE_KEY}" ]]; then fi tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/qemu-boot-smoke.XXXXXX")" -qemu_pid="" +QEMU_GUEST_PID="" # shellcheck disable=SC2329 # Called from the EXIT trap. cleanup() { - stop_qemu "${qemu_pid}" + qemu_guest_stop "${QEMU_GUEST_PID}" rm -rf "${tmp_dir}" } trap cleanup EXIT +# Ctrl-C during the SSH wait must stop the guest and remove the overlay, so turn +# the signal into an exit that runs the EXIT trap. +trap 'exit 130' INT TERM -ssh_key="${tmp_dir}/ssh_key" -cp "${QEMU_SSH_PRIVATE_KEY}" "${ssh_key}" -chmod 0600 "${ssh_key}" +QEMU_GUEST_SSH_KEY="${tmp_dir}/ssh_key" +cp "${QEMU_SSH_PRIVATE_KEY}" "${QEMU_GUEST_SSH_KEY}" +chmod 0600 "${QEMU_GUEST_SSH_KEY}" if [[ -r "${QEMU_SSH_PUBLIC_KEY}" ]]; then public_key="$(cat "${QEMU_SSH_PUBLIC_KEY}")" else - require_command ssh-keygen - public_key="$(ssh-keygen -y -f "${ssh_key}")" + qemu_guest_require_command ssh-keygen + public_key="$(ssh-keygen -y -f "${QEMU_GUEST_SSH_KEY}")" fi -backing_format="${QEMU_IMAGE_FORMAT:-$(detect_image_format "${image}")}" -runtime_disk="${tmp_dir}/disk.qcow2" -"${QEMU_IMG}" create -f qcow2 -F "${backing_format}" -b "${image}" "${runtime_disk}" >/dev/null +backing_format="${QEMU_IMAGE_FORMAT:-$(qemu_guest_detect_image_format "${QEMU_IMG}" "${image}")}" +QEMU_GUEST_DISK="${tmp_dir}/disk.qcow2" +qemu_guest_create_overlay "${QEMU_IMG}" "${image}" "${backing_format}" "${QEMU_GUEST_DISK}" -seed_args=() +QEMU_GUEST_SEED_ARGS=() case "${QEMU_SEED}" in cloud-init) seed_iso="${tmp_dir}/cidata.iso" - write_seed_iso "${tmp_dir}/seed" "${seed_iso}" "${public_key}" - seed_args=(-drive "file=${seed_iso},media=cdrom,readonly=on") - ;; -none) + qemu_guest_write_seed_iso "${tmp_dir}/seed" "${seed_iso}" "${QEMU_SSH_USER}" "${public_key}" qemu-boot-smoke + QEMU_GUEST_SEED_ARGS=(-drive "file=${seed_iso},media=cdrom,readonly=on") ;; +none) ;; *) echo "unsupported QEMU_SEED=${QEMU_SEED}; expected cloud-init or none" >&2 exit 1 ;; esac -QEMU_ACCELERATOR="${QEMU_ACCELERATOR:-$(detect_accelerator "${QEMU_BINARY}")}" -serial_log="${tmp_dir}/serial.log" -pidfile="${tmp_dir}/qemu.pid" - -"${QEMU_BINARY}" \ - -accel "${QEMU_ACCELERATOR}" \ - -machine "${QEMU_MACHINE}" \ - -m "${QEMU_MEMORY}" \ - -smp "${QEMU_CPUS}" \ - -drive "file=${runtime_disk},if=virtio,format=qcow2" \ - "${seed_args[@]}" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${QEMU_SSH_PORT}-:22" \ - -device "virtio-net-pci,netdev=net0" \ - -display none \ - -serial "file:${serial_log}" \ - -monitor none \ - -no-reboot \ - -pidfile "${pidfile}" \ - -daemonize \ - "${qemu_extra_args[@]}" - -qemu_pid="$(cat "${pidfile}")" -deadline=$((SECONDS + QEMU_SSH_TIMEOUT)) - -echo "Waiting up to ${QEMU_SSH_TIMEOUT}s for SSH on 127.0.0.1:${QEMU_SSH_PORT}..." -while ((SECONDS < deadline)); do - if ! kill -0 "${qemu_pid}" >/dev/null 2>&1; then - echo "QEMU exited before SSH became available" >&2 - sed -n '1,160p' "${serial_log}" >&2 || true - exit 1 - fi - - if ssh \ - -F /dev/null \ - -o BatchMode=yes \ - -o ConnectTimeout=5 \ - -o IdentitiesOnly=yes \ - -o LogLevel=ERROR \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -i "${ssh_key}" \ - -p "${QEMU_SSH_PORT}" \ - "${QEMU_SSH_USER}@127.0.0.1" \ - "${QEMU_SMOKE_COMMAND}" >/dev/null; then - echo "QEMU boot smoke succeeded for ${image}" - exit 0 - fi - - sleep "${QEMU_SSH_INTERVAL}" -done - -echo "Timed out waiting for SSH on 127.0.0.1:${QEMU_SSH_PORT}" >&2 -sed -n '1,160p' "${serial_log}" >&2 || true -exit 1 +QEMU_GUEST_BINARY="${QEMU_BINARY}" +QEMU_GUEST_ACCELERATOR="${QEMU_ACCELERATOR:-$(qemu_guest_detect_accelerator "${QEMU_BINARY}")}" +QEMU_GUEST_MACHINE="${QEMU_MACHINE}" +QEMU_GUEST_MEMORY="${QEMU_MEMORY}" +QEMU_GUEST_CPUS="${QEMU_CPUS}" +QEMU_GUEST_SSH_PORT="${QEMU_SSH_PORT}" +QEMU_GUEST_SSH_USER="${QEMU_SSH_USER}" +QEMU_GUEST_SSH_TIMEOUT="${QEMU_SSH_TIMEOUT}" +QEMU_GUEST_SSH_INTERVAL="${QEMU_SSH_INTERVAL}" +QEMU_GUEST_SERIAL_LOG="${tmp_dir}/serial.log" +QEMU_GUEST_PIDFILE="${tmp_dir}/qemu.pid" + +qemu_guest_start +qemu_guest_wait_for_ssh "${QEMU_SMOKE_COMMAND}" +echo "QEMU boot smoke succeeded for ${image}" diff --git a/images/capi/packer/qemu/scripts/qemu_guest_lib_test.py b/images/capi/packer/qemu/scripts/qemu_guest_lib_test.py new file mode 100644 index 0000000000..e3dd555668 --- /dev/null +++ b/images/capi/packer/qemu/scripts/qemu_guest_lib_test.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import subprocess +import tempfile +import unittest + + +CAPI_DIR = pathlib.Path(__file__).resolve().parents[3] +QEMU_GUEST_LIB = CAPI_DIR / "hack" / "lib" / "qemu-guest.sh" +BOOT_SMOKE = CAPI_DIR / "hack" / "qemu-boot-smoke.sh" + + +def write_stub(path, body, mode=0o755): + path.write_text(body, encoding="utf-8") + path.chmod(mode) + return path + + +class ResolveImageTests(unittest.TestCase): + def resolve(self, argument): + command = ( + f"set -euo pipefail\n" + f"source {str(QEMU_GUEST_LIB)!r}\n" + f'if ! image="$(qemu_guest_resolve_image_path {argument!r})"; then exit 3; fi\n' + f"printf 'RESOLVED %s\\n' \"$image\"\n" + ) + return subprocess.run(["bash", "-c", command], text=True, capture_output=True) + + def test_unresolvable_image_stops_the_caller(self): + # An inner command substitution's status is discarded once the outer + # command runs, so resolving must not be nested inside abs_path. + with tempfile.TemporaryDirectory() as tmp: + result = self.resolve(tmp) + + self.assertEqual(3, result.returncode, result.stderr) + self.assertNotIn("RESOLVED", result.stdout) + self.assertIn("expected exactly one", result.stderr) + + def test_missing_image_stops_the_caller(self): + with tempfile.TemporaryDirectory() as tmp: + result = self.resolve(str(pathlib.Path(tmp) / "absent.qcow2")) + + self.assertEqual(3, result.returncode, result.stderr) + self.assertNotIn("RESOLVED", result.stdout) + self.assertIn("image does not exist", result.stderr) + + def test_output_directory_resolves_to_its_single_image(self): + with tempfile.TemporaryDirectory() as tmp: + image = pathlib.Path(tmp) / "disk.qcow2" + image.touch() + + result = self.resolve(tmp) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(f"RESOLVED {image.resolve()}\n", result.stdout) + + def test_relative_image_resolves_to_an_absolute_path(self): + with tempfile.TemporaryDirectory() as tmp: + image = pathlib.Path(tmp) / "disk.qcow2" + image.touch() + command = ( + f"set -euo pipefail\n" + f"source {str(QEMU_GUEST_LIB)!r}\n" + f"cd {tmp!r}\n" + f"qemu_guest_resolve_image_path disk.qcow2\n" + ) + result = subprocess.run(["bash", "-c", command], text=True, capture_output=True) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(f"{image.resolve()}\n", result.stdout) + + def test_callers_do_not_nest_resolve_inside_abs_path(self): + self.assertNotIn( + 'qemu_guest_abs_path "$(qemu_guest_resolve_image', + BOOT_SMOKE.read_text(encoding="utf-8"), + "the resolve status must not be discarded", + ) + + +class ArgumentHandlingTests(unittest.TestCase): + """A trailing "--" leaves no positional parameters, and bash before 4.4 + treats "${@}" as unset under nounset, which aborts before QEMU starts.""" + + def test_boot_smoke_accepts_a_trailing_separator(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + for name in ("qemu-system-x86_64", "qemu-img", "ssh", "scp"): + write_stub(fake_bin / name, "#!/usr/bin/env bash\nexit 0\n") + image = tmp_path / "image.qcow2" + image.touch() + + result = subprocess.run( + ["bash", str(BOOT_SMOKE), str(image), "--"], + text=True, + capture_output=True, + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + # Stop right after argument parsing. + "QEMU_IMAGE_OS": "flatcar", + }, + ) + + self.assertNotIn("unbound variable", result.stderr) + self.assertIn("does not support Flatcar images", result.stderr) + + def test_no_bare_positional_expansion_after_a_shift(self): + for script in (BOOT_SMOKE, QEMU_GUEST_LIB): + text = script.read_text(encoding="utf-8") + self.assertNotIn('=("${@}")', text, f"{script} needs the ${{@+...}} guard") + self.assertNotIn('in "${@}"', text, f"{script} needs the ${{@+...}} guard") + + +class SignalHandlingTests(unittest.TestCase): + def test_interrupts_run_the_exit_cleanup(self): + text = BOOT_SMOKE.read_text(encoding="utf-8") + + self.assertIn("trap cleanup EXIT", text) + self.assertIn("trap 'exit 130' INT TERM", text) + + +if __name__ == "__main__": + unittest.main() From 07d04f63d7b3abb1ce54099ab15bd6b092c70736 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Fri, 4 Sep 2026 21:06:55 +0200 Subject: [PATCH 2/2] capi: run node conformance against a disposable image overlay Node conformance ran as Packer provisioners inside the QEMU build, so the shipped image had to be snapshotted and restored around the test, and two of those provisioners ran on every QEMU build whether or not conformance was enabled. Run it after the build instead. hack/qemu-node-conformance.sh boots the built image from a throwaway qcow2 copy-on-write overlay with a NoCloud seed ISO, copies the guest hook in over SSH, copies the results back out, and discards the overlay. The image is only ever read from, so the guest-side snapshot, restore and cleanup machinery, the node_conformance Packer variables and packer/config/node-conformance.json are all removed, and packer/qemu/packer.json.tmpl is back to its previous contents. Fixes carried into the new flow: - drop --container-runtime, which e2e_node.test does not accept in 1.33 to 1.35, so pflag exited before any spec ran - default --standalone-mode to false, since standalone mode starts the kubelet without a --kubeconfig and conformance pods cannot schedule - run e2e_node.test from the work dir, so the kubeconfig, kubelet-config and static-pod files the framework writes do not land in the SSH user's home directory - treat a missing or unparsable summary.env as a failure, not a pass - accept the checksum served by dl.k8s.io, which has no trailing newline and made read report EOF after it had already assigned the digest Results are copied into a fresh timestamped subdirectory of NODE_CONFORMANCE_OUTPUT_DIR. That path is caller supplied, so nothing under it is ever removed and repeated runs accumulate side by side. Downloads retry and are time capped so that a stalled transfer fails the run instead of hanging it until the Ginkgo timeout, and INT and TERM stop the guest and remove the overlay. --- docs/book/src/SUMMARY.md | 1 + docs/book/src/capi/node-conformance.md | 123 +++++ images/capi/.gitignore | 1 + images/capi/Makefile | 7 +- images/capi/hack/qemu-node-conformance.sh | 275 ++++++++++++ images/capi/hack/run-e2e-node-conformance.sh | 392 ++++++++++++++++ images/capi/packer/qemu/README.md | 26 ++ .../scripts/node_conformance_hook_test.py | 423 ++++++++++++++++++ .../capi/scripts/ci-qemu-node-conformance.sh | 66 +++ 9 files changed, 1313 insertions(+), 1 deletion(-) create mode 100644 docs/book/src/capi/node-conformance.md create mode 100755 images/capi/hack/qemu-node-conformance.sh create mode 100644 images/capi/hack/run-e2e-node-conformance.sh create mode 100644 images/capi/packer/qemu/scripts/node_conformance_hook_test.py create mode 100755 images/capi/scripts/ci-qemu-node-conformance.sh diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index df54d96377..8da045e5ad 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -27,6 +27,7 @@ - [MaaS](./capi/providers/maas.md) - [Including ECR Credential Provider](./capi/ecr-credential-provider.md) - [Testing the Images](./capi/goss/goss.md) + - [Kubernetes Node Conformance](./capi/node-conformance.md) - [Using Container Images](./capi/container-image.md) - [Customizing containerd](./capi/containerd/customizing-containerd.md) - [Kubernetes version matrix](./capi/kubernetes-version-matrix.md) diff --git a/docs/book/src/capi/node-conformance.md b/docs/book/src/capi/node-conformance.md new file mode 100644 index 0000000000..b1fe676b60 --- /dev/null +++ b/docs/book/src/capi/node-conformance.md @@ -0,0 +1,123 @@ +# Kubernetes Node Conformance + +Image Builder can run the Kubernetes `e2e_node.test` conformance subset against +an already built QEMU image. It is a post-build validation step, not part of the +image build. + +`hack/qemu-node-conformance.sh` boots the built image from a throwaway qcow2 +copy-on-write overlay with a temporary NoCloud seed ISO, copies the conformance +hook into the guest over SSH, runs it, copies the results back out, and then +discards the overlay. The built image is only ever read from, so a conformance +run cannot leave kubelet, CNI, runtime, or test state in the shipped artifact. + +Conformance is not run by default. It downloads the version-matched Kubernetes +test tarball and adds significant runtime, so it is meant for release or +periodic image validation jobs rather than every local or presubmit build. + +## Usage + +Build an image, then validate it: + +```bash +cd images/capi +make build-qemu-ubuntu-2404 +make test-qemu-node-conformance QEMU_NODE_CONFORMANCE_IMAGE=output/ubuntu-2404-kube-v1.33.0 +``` + +`QEMU_NODE_CONFORMANCE_IMAGE` accepts either a Packer output directory holding +exactly one disk image or a path to the image itself. Use +`QEMU_NODE_CONFORMANCE_ARGS='-- ...'` to pass additional QEMU arguments. + +The helper script can also be called directly: + +```bash +cd images/capi +hack/qemu-node-conformance.sh output/ubuntu-2404-kube-v1.33.0 +``` + +From the repository root, the CI entry point builds and validates in one step +with defaults suitable for a nested-virtualization runner: + +```bash +images/capi/scripts/ci-qemu-node-conformance.sh +``` + +It builds `build-qemu-ubuntu-2404-cloudimg` by default, then runs conformance +against the produced artifact with KVM acceleration, 4 CPUs, and 8 GiB of +memory. Override `NODE_CONFORMANCE_TARGET`, `NODE_CONFORMANCE_CPUS`, +`NODE_CONFORMANCE_MEMORY`, or `NODE_CONFORMANCE_ACCELERATOR` to tune a run. It +requires `/dev/kvm` unless `NODE_CONFORMANCE_ACCELERATOR=tcg` is set explicitly +for slower local debugging. + +Inside the guest, the hook downloads `kubernetes-test-linux-${ARCH}.tar.gz` for +the Kubernetes version reported by the image's own kubelet, starts the local CRI +runtime, stops the system kubelet, and runs `e2e_node.test` with a default focus +of `[Conformance]`. + +Each run writes its results into a fresh timestamped subdirectory of +`node-conformance-results/`, before the exit status is evaluated, so logs and +JUnit reports are preserved even when the run fails. Nothing under +`NODE_CONFORMANCE_OUTPUT_DIR` is ever deleted, so repeated runs accumulate side +by side and pointing the variable at an existing directory is safe. A missing or +unparsable `summary.env` is treated as a failure. + +Flatcar targets are excluded. Flatcar uses Ignition rather than cloud-init and +the build removes the SSH user before shutdown, so the guest cannot be reached +over SSH. Set `QEMU_IMAGE_OS=flatcar` to fail fast. + +## Configuration + +Both scripts are configured with environment variables. + +`hack/qemu-node-conformance.sh` shares the QEMU and SSH variables documented by +`hack/qemu-boot-smoke.sh` (`QEMU_BINARY`, `QEMU_IMG`, `QEMU_ACCELERATOR`, +`QEMU_MACHINE`, `QEMU_SSH_PORT`, `QEMU_SSH_USER`, ...), with these defaults +raised for a conformance workload: + +| Variable | Default | Description | +| --- | --- | --- | +| `QEMU_CPUS` | `4` | vCPUs given to the guest. | +| `QEMU_MEMORY` | `4096` | Guest memory in MiB. | +| `QEMU_SSH_TIMEOUT` | `900` | Seconds to wait for SSH after boot. | +| `NODE_CONFORMANCE_OUTPUT_DIR` | `node-conformance-results` | Host directory that per-run result subdirectories are created in. | + +The conformance run itself is tuned with the following variables, which are +forwarded into the guest: + +| Variable | Default | Description | +| --- | --- | --- | +| `KUBERNETES_VERSION` | detected from the guest kubelet | Version of the test tarball to download. | +| `NODE_CONFORMANCE_FOCUS` | `\[Conformance\]` | Ginkgo focus expression. | +| `NODE_CONFORMANCE_SKIP` | `\[Flaky\]\|\[Slow\]` | Ginkgo skip expression. | +| `NODE_CONFORMANCE_PARALLELISM` | `1` | Ginkgo parallel node count. | +| `NODE_CONFORMANCE_FLAKE_ATTEMPTS` | `1` | Ginkgo flake attempts. | +| `NODE_CONFORMANCE_TIMEOUT` | `2h` | Ginkgo timeout for the e2e-node run. | +| `NODE_CONFORMANCE_STANDALONE_MODE` | `false` | Passes `--standalone-mode=true` to `e2e_node.test`. | +| `NODE_CONFORMANCE_KUBELET_FLAGS` | `--fail-swap-on=false --runtime-cgroups=/system.slice/containerd.service` | Extra kubelet flags passed to `e2e_node.test`. | +| `NODE_CONFORMANCE_ETCD_VERSION` | `v3.5.32` | etcd version downloaded when `etcd` is not already installed. | +| `NODE_CONFORMANCE_DOWNLOAD_TIMEOUT` | `1800` | Seconds any single large download may take before it fails. | +| `NODE_CONFORMANCE_RESULTS_DIR` | `/tmp/kubernetes-node-conformance-results` | Guest result directory that is downloaded. | + +`NODE_CONFORMANCE_STANDALONE_MODE` defaults to `false` because standalone mode +starts the kubelet without a `--kubeconfig`, so it never joins the test +apiserver and conformance pods cannot be scheduled. + +Example with a custom focus and two parallel nodes: + +```bash +cd images/capi +NODE_CONFORMANCE_PARALLELISM=2 \ + NODE_CONFORMANCE_FOCUS='\[Conformance\]' \ + hack/qemu-node-conformance.sh output/ubuntu-2404-kube-v1.33.0 +``` + +## Scope + +Node conformance validates a node image in isolation. It complements Goss image +checks, but it does not replace Cluster API provider e2e tests or Kubernetes +cluster conformance suites that need a bootstrapped cluster. + +References: + +- Kubernetes node conformance: +- SIG Node e2e-node tests: diff --git a/images/capi/.gitignore b/images/capi/.gitignore index d3d946cd88..0934891ae2 100644 --- a/images/capi/.gitignore +++ b/images/capi/.gitignore @@ -11,3 +11,4 @@ manifest.json # Goss test droppings debug-goss-spec.yaml goss-spec.yaml +node-conformance-results/ diff --git a/images/capi/Makefile b/images/capi/Makefile index 553103e4f8..a746c6d45a 100644 --- a/images/capi/Makefile +++ b/images/capi/Makefile @@ -240,6 +240,11 @@ test-qemu-boot-smoke: ## Boots a local QEMU image and verifies SSH @test -n "$(QEMU_BOOT_SMOKE_IMAGE)" || (echo "QEMU_BOOT_SMOKE_IMAGE is required" >&2; exit 1) QEMU_IMAGE_OS="$(QEMU_BOOT_SMOKE_OS)" hack/qemu-boot-smoke.sh "$(QEMU_BOOT_SMOKE_IMAGE)" $(QEMU_BOOT_SMOKE_ARGS) +.PHONY: test-qemu-node-conformance +test-qemu-node-conformance: ## Runs Kubernetes node conformance against a built QEMU image + @test -n "$(QEMU_NODE_CONFORMANCE_IMAGE)" || (echo "QEMU_NODE_CONFORMANCE_IMAGE is required" >&2; exit 1) + QEMU_IMAGE_OS="$(QEMU_NODE_CONFORMANCE_OS)" hack/qemu-node-conformance.sh "$(QEMU_NODE_CONFORMANCE_IMAGE)" $(QEMU_NODE_CONFORMANCE_ARGS) + ## -------------------------------------- ## Container variables ## -------------------------------------- @@ -621,7 +626,7 @@ $(QEMU_VALIDATE_TARGETS): deps-qemu set-ssh-password $(PACKER) validate $(PACKER_NODE_FLAGS) -var-file="$(abspath packer/qemu/$(subst validate-,,$@).json)" $(ABSOLUTE_PACKER_VAR_FILES) packer/qemu/packer.json .PHONY: test-qemu-immutable -test-qemu-immutable: ## Runs immutable QEMU helper unit tests +test-qemu-immutable: ## Runs QEMU helper unit tests python3 -m unittest discover -s packer/qemu/scripts -p '*_test.py' validate-qemu-ubuntu-2404-immutable: test-qemu-immutable diff --git a/images/capi/hack/qemu-node-conformance.sh b/images/capi/hack/qemu-node-conformance.sh new file mode 100755 index 0000000000..c5543dec45 --- /dev/null +++ b/images/capi/hack/qemu-node-conformance.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +[[ -n ${DEBUG:-} ]] && set -o xtrace + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +capi_dir="$(cd "${script_dir}/.." && pwd -P)" + +# shellcheck source-path=SCRIPTDIR +# shellcheck source=lib/qemu-guest.sh +source "${script_dir}/lib/qemu-guest.sh" + +usage() { + cat <<'EOF' >&2 +usage: qemu-node-conformance.sh IMAGE_OR_OUTPUT_DIR [-- QEMU_ARGS...] + +Run the Kubernetes e2e_node.test conformance subset against an already built +QEMU image. The image is booted from a throwaway qcow2 copy-on-write overlay +with a NoCloud seed ISO, the conformance hook is copied in over SSH, results are +copied back out, and the overlay is discarded. The built image is only ever read +from, so a conformance run cannot leave test state in the shipped artifact. + +Flatcar (qemu-flatcar) images are not supported: Flatcar uses Ignition rather +than cloud-init, and the build removes the SSH user before shutdown, so the +guest cannot be reached over SSH. + +Environment: + QEMU_BINARY QEMU binary. Default: qemu-system-x86_64 + QEMU_IMG qemu-img binary. Default: qemu-img + QEMU_IMAGE_FORMAT Backing image format. Default: detected + QEMU_ACCELERATOR QEMU accelerator. Default: kvm on Linux with + /dev/kvm, hvf on macOS when the QEMU target + architecture matches the host, else tcg + QEMU_MACHINE QEMU machine type. Default: pc + QEMU_CPUS vCPU count. Default: 4 + QEMU_MEMORY Guest memory in MiB. Default: 4096 + QEMU_SSH_PORT Host port forwarded to guest 22. Default: 2222 + QEMU_SSH_TIMEOUT Seconds to wait for SSH. Default: 900 + QEMU_SSH_INTERVAL Seconds between SSH checks. Default: 5 + QEMU_SSH_USER SSH user. Default: capi + QEMU_SSH_PRIVATE_KEY SSH private key. Default: cloudinit/id_rsa.capi + QEMU_SSH_PUBLIC_KEY SSH public key. Default: cloudinit/id_rsa.capi.pub + QEMU_IMAGE_OS Set to flatcar to fail fast. Default: unset + NODE_CONFORMANCE_OUTPUT_DIR Host directory for downloaded results. + Default: node-conformance-results + NODE_CONFORMANCE_RESULTS_DIR Guest results directory. + Default: /tmp/kubernetes-node-conformance-results + KUBERNETES_VERSION Version of the test tarball to download. + Default: detected from the guest kubelet + NODE_CONFORMANCE_FOCUS Ginkgo focus. Default: \[Conformance\] + NODE_CONFORMANCE_SKIP Ginkgo skip. Default: \[Flaky\]|\[Slow\] + NODE_CONFORMANCE_PARALLELISM Ginkgo nodes. Default: 1 + NODE_CONFORMANCE_FLAKE_ATTEMPTS Ginkgo flake attempts. Default: 1 + NODE_CONFORMANCE_TIMEOUT Ginkgo timeout. Default: 2h + NODE_CONFORMANCE_STANDALONE_MODE Run kubelet without a test apiserver. + Default: false + NODE_CONFORMANCE_KUBELET_FLAGS Extra kubelet flags. Default: + --fail-swap-on=false + --runtime-cgroups=/system.slice/containerd.service + NODE_CONFORMANCE_ETCD_VERSION etcd to download when absent. Default: v3.5.32 +EOF +} + +# node_conformance_summary_exit_code prints the exit code the guest hook +# recorded. A summary that is missing or that does not report an exit code is a +# failure, never an implicit pass. +node_conformance_summary_exit_code() { + local summary_file="${1}" + local exit_code + + if [[ ! -f "${summary_file}" ]]; then + echo "missing node conformance summary: ${summary_file}" >&2 + return 1 + fi + + exit_code="$(sed -n 's/^exit_code=\([0-9][0-9]*\)$/\1/p' "${summary_file}" | tail -n 1)" + if [[ -z "${exit_code}" ]]; then + echo "node conformance summary does not report an exit_code: ${summary_file}" >&2 + return 1 + fi + + printf '%s\n' "${exit_code}" +} + +# node_conformance_guest_env prints the shell-quoted environment assignments +# forwarded into the guest. Only variables the caller set are forwarded, so the +# hook keeps its own documented defaults. +node_conformance_guest_env() { + local -a assignments=("NODE_CONFORMANCE_RESULTS_DIR=${1}") + local name + + for name in \ + KUBERNETES_VERSION \ + NODE_CONFORMANCE_ETCD_VERSION \ + NODE_CONFORMANCE_FLAKE_ATTEMPTS \ + NODE_CONFORMANCE_FOCUS \ + NODE_CONFORMANCE_KUBELET_FLAGS \ + NODE_CONFORMANCE_PARALLELISM \ + NODE_CONFORMANCE_SKIP \ + NODE_CONFORMANCE_STANDALONE_MODE \ + NODE_CONFORMANCE_TIMEOUT; do + if [[ -n "${!name:-}" ]]; then + assignments+=("${name}=${!name}") + fi + done + + printf '%q ' "${assignments[@]}" +} + +# shellcheck disable=SC2329 # Called from the EXIT trap. +cleanup() { + qemu_guest_stop "${QEMU_GUEST_PID:-}" + rm -rf "${tmp_dir}" +} + +main() { + local image_arg + local image + local public_key + local backing_format + local seed_iso + local guest_results_dir + local hook_script + local output_dir + local run_dir + local remote_hook + local remote_env_args + local run_status=0 + local download_status=0 + local exit_code + + if [[ $# -lt 1 ]]; then + usage + return 1 + fi + + image_arg="${1}" + shift + + QEMU_GUEST_EXTRA_ARGS=() + if [[ ${1:-} == "--" ]]; then + shift + # A trailing "--" with nothing after it leaves no positional parameters, + # and bash before 4.4 treats "${@}" as unset under nounset. + QEMU_GUEST_EXTRA_ARGS=(${@+"${@}"}) + elif [[ $# -gt 0 ]]; then + usage + return 1 + fi + + QEMU_BINARY="${QEMU_BINARY:-qemu-system-x86_64}" + QEMU_IMG="${QEMU_IMG:-qemu-img}" + guest_results_dir="${NODE_CONFORMANCE_RESULTS_DIR:-/tmp/kubernetes-node-conformance-results}" + output_dir="${NODE_CONFORMANCE_OUTPUT_DIR:-${capi_dir}/node-conformance-results}" + hook_script="${script_dir}/run-e2e-node-conformance.sh" + + qemu_guest_require_command "${QEMU_BINARY}" + qemu_guest_require_command "${QEMU_IMG}" + qemu_guest_require_command ssh + qemu_guest_require_command scp + + if ! image="$(qemu_guest_resolve_image_path "${image_arg}")"; then + return 1 + fi + if [[ "${QEMU_IMAGE_OS:-}" == "flatcar" ]]; then + echo "qemu-node-conformance.sh does not support Flatcar images: Flatcar uses Ignition, not cloud-init, and the build removes the SSH user before shutdown, so the guest cannot be reached over SSH. Image: ${image}" >&2 + return 1 + fi + + QEMU_SSH_PRIVATE_KEY="${QEMU_SSH_PRIVATE_KEY:-${capi_dir}/cloudinit/id_rsa.capi}" + QEMU_SSH_PUBLIC_KEY="${QEMU_SSH_PUBLIC_KEY:-${capi_dir}/cloudinit/id_rsa.capi.pub}" + if [[ ! -r "${QEMU_SSH_PRIVATE_KEY}" ]]; then + echo "SSH private key is not readable: ${QEMU_SSH_PRIVATE_KEY}" >&2 + return 1 + fi + if [[ ! -r "${hook_script}" ]]; then + echo "conformance hook is not readable: ${hook_script}" >&2 + return 1 + fi + + tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/qemu-node-conformance.XXXXXX")" + QEMU_GUEST_PID="" + trap cleanup EXIT + # Ctrl-C during the SSH wait or the conformance run must stop the guest and + # remove the overlay, so turn the signal into an exit that runs the EXIT trap. + trap 'exit 130' INT TERM + + QEMU_GUEST_SSH_KEY="${tmp_dir}/ssh_key" + cp "${QEMU_SSH_PRIVATE_KEY}" "${QEMU_GUEST_SSH_KEY}" + chmod 0600 "${QEMU_GUEST_SSH_KEY}" + + if [[ -r "${QEMU_SSH_PUBLIC_KEY}" ]]; then + public_key="$(cat "${QEMU_SSH_PUBLIC_KEY}")" + else + qemu_guest_require_command ssh-keygen + public_key="$(ssh-keygen -y -f "${QEMU_GUEST_SSH_KEY}")" + fi + + backing_format="${QEMU_IMAGE_FORMAT:-$(qemu_guest_detect_image_format "${QEMU_IMG}" "${image}")}" + QEMU_GUEST_DISK="${tmp_dir}/disk.qcow2" + qemu_guest_create_overlay "${QEMU_IMG}" "${image}" "${backing_format}" "${QEMU_GUEST_DISK}" + + seed_iso="${tmp_dir}/cidata.iso" + QEMU_GUEST_SSH_USER="${QEMU_SSH_USER:-capi}" + qemu_guest_write_seed_iso \ + "${tmp_dir}/seed" "${seed_iso}" "${QEMU_GUEST_SSH_USER}" "${public_key}" qemu-node-conformance + QEMU_GUEST_SEED_ARGS=(-drive "file=${seed_iso},media=cdrom,readonly=on") + + QEMU_GUEST_BINARY="${QEMU_BINARY}" + QEMU_GUEST_ACCELERATOR="${QEMU_ACCELERATOR:-$(qemu_guest_detect_accelerator "${QEMU_BINARY}")}" + QEMU_GUEST_MACHINE="${QEMU_MACHINE:-pc}" + QEMU_GUEST_MEMORY="${QEMU_MEMORY:-4096}" + QEMU_GUEST_CPUS="${QEMU_CPUS:-4}" + QEMU_GUEST_SSH_PORT="${QEMU_SSH_PORT:-2222}" + QEMU_GUEST_SSH_TIMEOUT="${QEMU_SSH_TIMEOUT:-900}" + QEMU_GUEST_SSH_INTERVAL="${QEMU_SSH_INTERVAL:-5}" + QEMU_GUEST_SERIAL_LOG="${tmp_dir}/serial.log" + QEMU_GUEST_PIDFILE="${tmp_dir}/qemu.pid" + + echo "Booting ${image} on a throwaway overlay for node conformance" + qemu_guest_start + qemu_guest_wait_for_ssh true + + remote_hook="run-e2e-node-conformance.sh" + qemu_guest_scp "${hook_script}" "guest:${remote_hook}" + + remote_env_args="$(node_conformance_guest_env "${guest_results_dir}")" + qemu_guest_ssh "env ${remote_env_args}bash ${remote_hook}" || run_status=$? + + # Download the results before evaluating them so logs and JUnit reports + # survive a failing run. The download lands in the throwaway directory first, + # then it is copied into a fresh per-run subdirectory of the output + # directory. NODE_CONFORMANCE_OUTPUT_DIR is caller supplied, so nothing under + # it is ever deleted and repeated runs accumulate side by side. + qemu_guest_scp "guest:${guest_results_dir}" "${tmp_dir}/results" || download_status=$? + if [[ "${download_status}" -ne 0 ]]; then + echo "failed to download node conformance results from the guest" >&2 + qemu_guest_dump_serial_log + return 1 + fi + + mkdir -p "${output_dir}" + run_dir="$(mktemp -d "${output_dir}/$(date -u +%Y%m%dT%H%M%SZ).XXXXXX")" + cp -R "${tmp_dir}/results/." "${run_dir}/" + echo "Node conformance results downloaded to ${run_dir}" + + exit_code="$(node_conformance_summary_exit_code "${run_dir}/summary.env")" || return 1 + if [[ "${exit_code}" != "0" || "${run_status}" -ne 0 ]]; then + echo "node conformance failed: hook exit_code=${exit_code}, ssh status=${run_status}" >&2 + return 1 + fi + + echo "Node conformance succeeded for ${image}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/images/capi/hack/run-e2e-node-conformance.sh b/images/capi/hack/run-e2e-node-conformance.sh new file mode 100644 index 0000000000..678528176a --- /dev/null +++ b/images/capi/hack/run-e2e-node-conformance.sh @@ -0,0 +1,392 @@ +#!/usr/bin/env bash + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Guest-side Kubernetes node conformance hook. +# +# This script is not run during an image build. hack/qemu-node-conformance.sh +# boots the already built image from a throwaway qcow2 overlay, copies this +# script in, runs it, and copies ${NODE_CONFORMANCE_RESULTS_DIR} back out before +# discarding the overlay. Everything it writes is therefore confined to a disk +# that is deleted afterwards, and the shipped image is only ever read from. + +set -euo pipefail + +log() { + printf '[node-conformance] %s\n' "$*" >&2 +} + +die() { + printf '[node-conformance] ERROR: %s\n' "$*" >&2 + exit 1 +} + +is_true() { + local value="${1:-false}" + + value="$(printf '%s' "${value}" | tr '[:upper:]' '[:lower:]')" + case "${value}" in + true | 1 | yes) return 0 ;; + *) return 1 ;; + esac +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +detect_go_arch() { + case "$(uname -m)" in + x86_64 | amd64) printf 'amd64\n' ;; + aarch64 | arm64) printf 'arm64\n' ;; + *) die "unsupported architecture: $(uname -m)" ;; + esac +} + +normalize_kubernetes_version() { + local version="$1" + version="${version#v}" + [[ "${version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + die "KUBERNETES_VERSION must look like 1.36.2 or v1.36.2" + printf '%s\n' "${version}" +} + +# detect_kubernetes_version reads the version out of the image under test so the +# downloaded test tarball always matches the kubelet that is being validated. +detect_kubernetes_version() { + local version + + command -v kubelet >/dev/null 2>&1 || + die "KUBERNETES_VERSION is unset and kubelet is not installed in this image" + version="$(kubelet --version 2>/dev/null | awk '{print $2}')" + [[ -n "${version}" ]] || die "could not read the Kubernetes version from kubelet" + printf '%s\n' "${version}" +} + +verify_sha256_file() { + local file="$1" + local sha_file="$2" + local expected="" + + # dl.k8s.io serves the checksum without a trailing newline, so read reports + # EOF even though it assigned the digest. Validate the value it read rather + # than its exit status. + read -r expected _ <"${sha_file}" || true + [[ "${expected}" =~ ^[A-Fa-f0-9]{64}$ ]] || + die "invalid or unreadable SHA256 file: ${sha_file}" + printf '%s %s\n' "${expected}" "${file}" | sha256sum --check --strict +} + +is_flatcar() ( + local os_release_file="${NODE_CONFORMANCE_OS_RELEASE_FILE:-/etc/os-release}" + local id="" + local id_like="" + + set +u + if [[ -r "${os_release_file}" ]]; then + # shellcheck disable=SC1090 + . "${os_release_file}" + id="${ID:-}" + id_like="${ID_LIKE:-}" + fi + + id="$(printf '%s' "${id}" | tr '[:upper:]' '[:lower:]')" + id_like="$(printf '%s' "${id_like}" | tr '[:upper:]' '[:lower:]')" + [[ "${id}" == "flatcar" || " ${id_like} " == *" flatcar "* ]] +) + +# node_conformance_download fetches a URL to a file. dl.k8s.io and the GitHub +# release CDN both fail intermittently, so retry, and cap each transfer so that +# a stalled download fails the run instead of hanging it until the Ginkgo +# timeout. Arguments: max seconds, output path, url. +node_conformance_download() { + local max_time="$1" + local output="$2" + local url="$3" + + curl --fail --silent --show-error --location \ + --retry 3 --retry-delay 5 --retry-connrefused \ + --connect-timeout 30 --max-time "${max_time}" \ + --output "${output}" "${url}" +} + +download_kubernetes_tests() { + local kubernetes_version="$1" + local go_arch="$2" + local tarball_url="${NODE_CONFORMANCE_TARBALL_URL:-https://dl.k8s.io/v${kubernetes_version}/kubernetes-test-linux-${go_arch}.tar.gz}" + local download_timeout="${NODE_CONFORMANCE_DOWNLOAD_TIMEOUT:-1800}" + + log "downloading Kubernetes test tarball: ${tarball_url}" + node_conformance_download "${download_timeout}" \ + "${work_dir}/kubernetes-test.tar.gz" "${tarball_url}" + node_conformance_download 120 \ + "${work_dir}/kubernetes-test.tar.gz.sha256" "${tarball_url}.sha256" + verify_sha256_file \ + "${work_dir}/kubernetes-test.tar.gz" \ + "${work_dir}/kubernetes-test.tar.gz.sha256" + + tar -xzf "${work_dir}/kubernetes-test.tar.gz" -C "${work_dir}" \ + kubernetes/test/bin/e2e_node.test \ + kubernetes/test/bin/ginkgo + + e2e_node_test="${work_dir}/kubernetes/test/bin/e2e_node.test" + ginkgo_bin="${work_dir}/kubernetes/test/bin/ginkgo" + chmod +x "${e2e_node_test}" "${ginkgo_bin}" +} + +ensure_etcd() { + local go_arch="$1" + local etcd_version="${NODE_CONFORMANCE_ETCD_VERSION:-v3.5.32}" + local download_timeout="${NODE_CONFORMANCE_DOWNLOAD_TIMEOUT:-1800}" + local etcd_url + + if command -v etcd >/dev/null 2>&1; then + log "using etcd from PATH: $(command -v etcd)" + return + fi + + mkdir -p "${work_dir}/bin" + etcd_url="https://github.com/etcd-io/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-${go_arch}.tar.gz" + log "downloading etcd ${etcd_version}: ${etcd_url}" + node_conformance_download "${download_timeout}" "${work_dir}/etcd.tar.gz" "${etcd_url}" + tar -xzf "${work_dir}/etcd.tar.gz" -C "${work_dir}" + install -m 0755 \ + "${work_dir}/etcd-${etcd_version}-linux-${go_arch}/etcd" \ + "${work_dir}/bin/etcd" + export PATH="${work_dir}/bin:${PATH}" +} + +runtime_endpoint() { + local sock + + for sock in \ + /run/containerd/containerd.sock \ + /var/run/containerd/containerd.sock \ + /var/run/crio/crio.sock; do + if [[ -S "${sock}" ]]; then + printf 'unix://%s\n' "${sock}" + return + fi + done + + die "no CRI runtime socket found" +} + +runtime_process_name() { + local runtime_binary + + case "$1" in + unix:///run/containerd/containerd.sock | unix:///var/run/containerd/containerd.sock) + runtime_binary="$(command -v containerd || true)" + printf '%s\n' "${runtime_binary:-/usr/local/bin/containerd}" + ;; + unix:///var/run/crio/crio.sock) + runtime_binary="$(command -v crio || true)" + printf '%s\n' "${runtime_binary:-/usr/bin/crio}" + ;; + *) + printf 'containerd\n' + ;; + esac +} + +ensure_container_runtime() { + if command -v systemctl >/dev/null 2>&1; then + sudo systemctl start containerd >/dev/null 2>&1 || true + sudo systemctl start crio >/dev/null 2>&1 || true + fi +} + +ensure_cni_config() { + local plugin + + for plugin in bridge host-local loopback portmap; do + [[ -x "${cni_bin_dir}/${plugin}" ]] || + die "required CNI plugin is missing or not executable: ${cni_bin_dir}/${plugin}" + done + + sudo mkdir -p "${cni_conf_dir}" + if sudo find "${cni_conf_dir}" -mindepth 1 -maxdepth 1 -type f -print -quit | + grep -q .; then + log "using existing CNI config in ${cni_conf_dir}" + return + fi + + sudo mkdir -p "${cni_data_dir}" + log "creating CNI config: ${cni_conf_dir}/10-node-conformance.conflist" + cat </dev/null +{ + "cniVersion": "1.0.0", + "name": "node-conformance", + "plugins": [ + { + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "promiscMode": true, + "ipam": { + "type": "host-local", + "dataDir": "${cni_data_dir}", + "ranges": [ + [{ "subnet": "10.88.0.0/16" }] + ], + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true } + } + ] +} +JSON +} + +stop_system_kubelet() { + if command -v systemctl >/dev/null 2>&1 && + systemctl list-unit-files kubelet.service >/dev/null 2>&1; then + log "stopping system kubelet before e2e-node starts its own kubelet" + sudo systemctl stop kubelet || true + fi +} + +# publish_results records the final exit code and hands the results directory to +# the invoking SSH user so hack/qemu-node-conformance.sh can copy it out. +publish_results() { + local exit_code=$? + + set +e + if [[ -n "${results_dir:-}" ]]; then + sudo mkdir -p "${results_dir}" + printf 'exit_code=%s\n' "${exit_code}" | sudo tee "${results_dir}/summary.env" >/dev/null + sudo chown -R "$(id -u):$(id -g)" "${results_dir}" + fi + + exit "${exit_code}" +} + +run_e2e_node() { + local endpoint="$1" + local process_name="$2" + local node_name="${NODE_CONFORMANCE_NODE_NAME:-$(hostname)}" + local k8s_bin_dir="${NODE_CONFORMANCE_K8S_BIN_DIR:-/usr/bin}" + local focus="${NODE_CONFORMANCE_FOCUS:-\\[Conformance\\]}" + local skip="${NODE_CONFORMANCE_SKIP:-\\[Flaky\\]|\\[Slow\\]}" + local timeout="${NODE_CONFORMANCE_TIMEOUT:-2h}" + local parallelism="${NODE_CONFORMANCE_PARALLELISM:-1}" + local flake_attempts="${NODE_CONFORMANCE_FLAKE_ATTEMPTS:-1}" + local kubelet_flags="${NODE_CONFORMANCE_KUBELET_FLAGS:---fail-swap-on=false --runtime-cgroups=/system.slice/containerd.service}" + local kubelet_root_dir="${NODE_CONFORMANCE_KUBELET_ROOT_DIR:-${work_dir}/kubelet}" + local standalone_mode="${NODE_CONFORMANCE_STANDALONE_MODE:-false}" + local -a ginkgo_args + local -a test_args + local exit_code=0 + + if [[ " ${kubelet_flags} " != *" --root-dir="* ]]; then + kubelet_flags+=" --root-dir=${kubelet_root_dir}" + fi + if [[ " ${kubelet_flags} " != *" --cert-dir="* ]]; then + kubelet_flags+=" --cert-dir=${kubelet_root_dir}/pki" + fi + + ginkgo_args=( + "--nodes=${parallelism}" + "--flake-attempts=${flake_attempts}" + "--focus=${focus}" + "--skip=${skip}" + "--timeout=${timeout}" + "--v" + ) + + test_args=( + "--node-name=${node_name}" + "--k8s-bin-dir=${k8s_bin_dir}" + "--container-runtime-endpoint=${endpoint}" + "--container-runtime-process-name=${process_name}" + "--container-runtime-pid-file=" + "--kubelet-flags=${kubelet_flags}" + "--report-dir=${results_dir}" + "--report-prefix=node-conformance" + ) + + if is_true "${standalone_mode}"; then + test_args+=("--standalone-mode=true") + fi + + log "running e2e_node.test focus=${focus} skip=${skip} parallelism=${parallelism}" + # The e2e framework writes kubeconfig, kubelet-config and static-pod manifests + # into its working directory, so run it from the work dir instead of the SSH + # user's home. + ( + cd "${work_dir}" + set +e + sudo -E env "PATH=${PATH}" "${ginkgo_bin}" "${ginkgo_args[@]}" \ + "${e2e_node_test}" -- "${test_args[@]}" 2>&1 | + tee "${results_dir}/e2e_node.log" + exit "${PIPESTATUS[0]}" + ) || exit_code=$? + + return "${exit_code}" +} + +main() { + local kubernetes_version + local go_arch + local endpoint + local process_name + + results_dir="${NODE_CONFORMANCE_RESULTS_DIR:-/tmp/kubernetes-node-conformance-results}" + mkdir -p "${results_dir}" + trap publish_results EXIT + + if is_flatcar; then + die "node conformance is not supported on Flatcar images" + fi + + require_cmd curl + require_cmd sha256sum + require_cmd sudo + require_cmd tar + + work_dir="${NODE_CONFORMANCE_WORK_DIR:-$(mktemp -d /tmp/node-conformance.XXXXXX)}" + cni_conf_dir="${NODE_CONFORMANCE_CNI_CONF_DIR:-/etc/cni/net.d}" + cni_bin_dir="${NODE_CONFORMANCE_CNI_BIN_DIR:-/opt/cni/bin}" + cni_data_dir="${NODE_CONFORMANCE_CNI_DATA_DIR:-${work_dir}/cni/networks}" + + kubernetes_version="${KUBERNETES_VERSION:-}" + if [[ -z "${kubernetes_version}" ]]; then + kubernetes_version="$(detect_kubernetes_version)" + fi + kubernetes_version="$(normalize_kubernetes_version "${kubernetes_version}")" + go_arch="$(detect_go_arch)" + + download_kubernetes_tests "${kubernetes_version}" "${go_arch}" + ensure_etcd "${go_arch}" + ensure_container_runtime + endpoint="$(runtime_endpoint)" + process_name="$(runtime_process_name "${endpoint}")" + ensure_cni_config + stop_system_kubelet + + log "Kubernetes v${kubernetes_version}; runtime endpoint ${endpoint}" + run_e2e_node "${endpoint}" "${process_name}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/images/capi/packer/qemu/README.md b/images/capi/packer/qemu/README.md index f216193e5a..e12ef8ac94 100644 --- a/images/capi/packer/qemu/README.md +++ b/images/capi/packer/qemu/README.md @@ -182,3 +182,29 @@ make test-qemu-boot-smoke QEMU_BOOT_SMOKE_IMAGE=/path/to/image.qcow2 The smoke helper does not support `qemu-flatcar` images because they use Ignition instead of cloud-init. Set `QEMU_BOOT_SMOKE_OS=flatcar` when invoking the Make target to fail fast before attempting an unsupported SSH check. + +## Node conformance + +`hack/qemu-node-conformance.sh` runs the Kubernetes `e2e_node.test` conformance +subset against an already built image. Like the boot smoke test it boots a +throwaway copy-on-write overlay with a temporary NoCloud seed ISO, so the built +artifact is only ever read from and cannot end up carrying conformance state: + +```bash +make test-qemu-node-conformance QEMU_NODE_CONFORMANCE_IMAGE=output/ubuntu-2404-kube-v1.33.0 +``` + +The hook downloads the `e2e_node.test` binary matching the kubelet in the image. +Results are copied back into a fresh timestamped subdirectory of +`node-conformance-results/` before the exit status is evaluated; nothing under +that directory is removed, so runs accumulate side by side. + +Node conformance is opt-in and is not wired into required CI. It is intended for +release or periodic image validation jobs where the added runtime is acceptable, +not for every local or presubmit image build. + +Flatcar targets are excluded, for the same reason the boot smoke test excludes +them. Set `QEMU_NODE_CONFORMANCE_OS=flatcar` to fail fast. + +See [Kubernetes Node Conformance](../../../../docs/book/src/capi/node-conformance.md) +for the full list of configuration variables. diff --git a/images/capi/packer/qemu/scripts/node_conformance_hook_test.py b/images/capi/packer/qemu/scripts/node_conformance_hook_test.py new file mode 100644 index 0000000000..6c01db6d56 --- /dev/null +++ b/images/capi/packer/qemu/scripts/node_conformance_hook_test.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import pathlib +import re +import subprocess +import tempfile +import unittest + + +CAPI_DIR = pathlib.Path(__file__).resolve().parents[3] +REPO_ROOT = pathlib.Path(__file__).resolve().parents[5] +HOOK = CAPI_DIR / "hack" / "run-e2e-node-conformance.sh" +RUNNER = CAPI_DIR / "hack" / "qemu-node-conformance.sh" +BOOT_SMOKE = CAPI_DIR / "hack" / "qemu-boot-smoke.sh" +QEMU_GUEST_LIB = CAPI_DIR / "hack" / "lib" / "qemu-guest.sh" +CI_HELPER = CAPI_DIR / "scripts" / "ci-qemu-node-conformance.sh" +PACKER_TEMPLATE = CAPI_DIR / "packer" / "qemu" / "packer.json.tmpl" +DOC = REPO_ROOT / "docs" / "book" / "src" / "capi" / "node-conformance.md" + +SUDO_STUB = '''#!/usr/bin/env bash +# Drop sudo options such as -E, then run the command directly. +while [[ "${1:-}" == -* ]]; do shift; done +exec "$@" +''' + + +def write_stub(path, body, mode=0o755): + path.write_text(body, encoding="utf-8") + path.chmod(mode) + return path + + +def shell_default(script_text, name): + """Returns the literal default of a "${NAME:-DEFAULT}" expansion. + + The expansions live inside double quotes, so bash collapses a doubled + backslash into a single one before the value is used. + """ + match = re.search(r'\$\{' + re.escape(name) + r':-(.*?)\}"', script_text) + if match is None: + raise AssertionError(f"no default found for {name}") + return match.group(1).replace("\\\\", "\\") + + +def documented_defaults(): + defaults = {} + for line in DOC.read_text(encoding="utf-8").splitlines(): + match = re.match(r"^\| `([A-Z_]+)` \| `(.*?)` \| ", line) + if match: + defaults[match.group(1)] = match.group(2).replace("\\|", "|") + return defaults + + +class GuestHookTests(unittest.TestCase): + def test_flatcar_is_explicitly_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + write_stub(fake_bin / "sudo", SUDO_STUB) + os_release = tmp_path / "os-release" + os_release.write_text('ID="flatcar"\n', encoding="utf-8") + results_dir = tmp_path / "results" + + result = subprocess.run( + ["bash", str(HOOK)], + text=True, + capture_output=True, + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "NODE_CONFORMANCE_RESULTS_DIR": str(results_dir), + "NODE_CONFORMANCE_OS_RELEASE_FILE": str(os_release), + }, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("not supported on Flatcar", result.stderr) + self.assertIn("exit_code=1", (results_dir / "summary.env").read_text(encoding="utf-8")) + + def test_e2e_node_runs_from_the_work_dir_without_the_container_runtime_flag(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + write_stub(fake_bin / "sudo", SUDO_STUB) + + work_dir = tmp_path / "work" + results_dir = tmp_path / "results" + work_dir.mkdir() + results_dir.mkdir() + invocation = tmp_path / "invocation.txt" + ginkgo = write_stub( + tmp_path / "ginkgo", + f"""#!/usr/bin/env bash +{{ + printf 'pwd=%s\\n' "$PWD" + printf 'arg=%s\\n' "$@" +}} > {str(invocation)!r} +""", + ) + + command = f""" +set -euo pipefail +source {str(HOOK)!r} +work_dir={str(work_dir)!r} +results_dir={str(results_dir)!r} +ginkgo_bin={str(ginkgo)!r} +e2e_node_test={str(tmp_path / 'e2e_node.test')!r} +run_e2e_node unix:///run/containerd/containerd.sock /usr/bin/containerd +""" + result = subprocess.run( + ["bash", "-c", command], + text=True, + capture_output=True, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + ) + + self.assertEqual(0, result.returncode, result.stderr) + recorded = invocation.read_text(encoding="utf-8") + self.assertIn(f"pwd={work_dir}\n", recorded) + # e2e_node.test has no --container-runtime flag in 1.33 to 1.35, so + # passing it makes pflag exit before any spec runs. + self.assertNotIn("--container-runtime=", recorded) + self.assertIn("--container-runtime-endpoint=", recorded) + # Standalone mode never joins the test apiserver, so it is off by + # default and must not be requested here. + self.assertNotIn("--standalone-mode", recorded) + + def test_checksum_without_a_trailing_newline_is_accepted(self): + # dl.k8s.io serves the digest with no trailing newline, which makes + # bash read report EOF even though it assigned the value. + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + checked = tmp_path / "checked.txt" + write_stub( + fake_bin / "sha256sum", + f"""#!/usr/bin/env bash +cat > {str(checked)!r} +""", + ) + payload = tmp_path / "kubernetes-test.tar.gz" + payload.write_text("payload", encoding="utf-8") + digest = "a" * 64 + sha_file = tmp_path / "kubernetes-test.tar.gz.sha256" + sha_file.write_text(digest, encoding="utf-8") + + command = f""" +set -euo pipefail +source {str(HOOK)!r} +verify_sha256_file {str(payload)!r} {str(sha_file)!r} +""" + result = subprocess.run( + ["bash", "-c", command], + text=True, + capture_output=True, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(f"{digest} {payload}\n", checked.read_text(encoding="utf-8")) + + def test_invalid_checksum_file_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + sha_file = pathlib.Path(tmp) / "sha256" + sha_file.write_text("not-a-digest\n", encoding="utf-8") + + result = subprocess.run( + ["bash", "-c", f"source {str(HOOK)!r}\nverify_sha256_file /dev/null {str(sha_file)!r}"], + text=True, + capture_output=True, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("invalid or unreadable SHA256 file", result.stderr) + + def test_standalone_mode_is_opt_in(self): + self.assertEqual("false", shell_default(HOOK.read_text(encoding="utf-8"), + "NODE_CONFORMANCE_STANDALONE_MODE")) + + def test_snapshot_and_restore_helpers_are_gone(self): + script = HOOK.read_text(encoding="utf-8") + + for removed in ( + "snapshot_node_state", + "restore_node_state", + "snapshot_runtime_state", + "cleanup_cri_runtime_state", + "cleanup_ctr_runtime_state", + "restore_service_state", + ): + self.assertNotIn(removed, script) + + +class RunnerTests(unittest.TestCase): + def source_runner(self, command, env=None): + return subprocess.run( + ["bash", "-c", f"source {str(RUNNER)!r}\n{command}"], + text=True, + capture_output=True, + env={**os.environ, **(env or {})}, + ) + + def test_missing_summary_is_a_failure(self): + with tempfile.TemporaryDirectory() as tmp: + missing = pathlib.Path(tmp) / "summary.env" + + result = self.source_runner(f"node_conformance_summary_exit_code {str(missing)!r}") + + self.assertNotEqual(0, result.returncode) + self.assertIn("missing node conformance summary", result.stderr) + + def test_summary_without_an_exit_code_is_a_failure(self): + with tempfile.TemporaryDirectory() as tmp: + summary = pathlib.Path(tmp) / "summary.env" + summary.write_text("skipped=true\n", encoding="utf-8") + + result = self.source_runner(f"node_conformance_summary_exit_code {str(summary)!r}") + + self.assertNotEqual(0, result.returncode) + self.assertIn("does not report an exit_code", result.stderr) + + def test_summary_exit_code_is_reported(self): + with tempfile.TemporaryDirectory() as tmp: + summary = pathlib.Path(tmp) / "summary.env" + summary.write_text("exit_code=7\n", encoding="utf-8") + + result = self.source_runner(f"node_conformance_summary_exit_code {str(summary)!r}") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("7\n", result.stdout) + + def test_only_explicitly_set_variables_are_forwarded_to_the_guest(self): + result = self.source_runner( + "node_conformance_guest_env /tmp/results", + {"NODE_CONFORMANCE_FOCUS": r"\[Conformance\]", "NODE_CONFORMANCE_TIMEOUT": ""}, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("NODE_CONFORMANCE_RESULTS_DIR=/tmp/results", result.stdout) + self.assertIn("NODE_CONFORMANCE_FOCUS=", result.stdout) + self.assertNotIn("NODE_CONFORMANCE_TIMEOUT", result.stdout) + + def test_flatcar_images_are_rejected_before_boot(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + for name in ("qemu-system-x86_64", "qemu-img", "ssh", "scp"): + write_stub(fake_bin / name, "#!/usr/bin/env bash\nexit 0\n") + image = tmp_path / "image.qcow2" + image.write_text("", encoding="utf-8") + + result = subprocess.run( + ["bash", str(RUNNER), str(image)], + text=True, + capture_output=True, + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "QEMU_IMAGE_OS": "flatcar", + }, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("does not support Flatcar images", result.stderr) + + def test_ci_helper_rejects_flatcar_target(self): + result = subprocess.run( + ["bash", str(CI_HELPER)], + text=True, + capture_output=True, + env={ + **os.environ, + "NODE_CONFORMANCE_TARGET": "build-qemu-flatcar", + "NODE_CONFORMANCE_ACCELERATOR": "tcg", + }, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("not supported for node conformance", result.stderr) + + +class ArgumentHandlingTests(unittest.TestCase): + def test_conformance_runner_accepts_a_trailing_separator(self): + # A trailing "--" leaves no positional parameters, and bash before 4.4 + # treats "${@}" as unset under nounset, aborting before QEMU starts. + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + for name in ("qemu-system-x86_64", "qemu-img", "ssh", "scp"): + write_stub(fake_bin / name, "#!/usr/bin/env bash\nexit 0\n") + image = tmp_path / "image.qcow2" + image.touch() + + result = subprocess.run( + ["bash", str(RUNNER), str(image), "--"], + text=True, + capture_output=True, + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + # Stop right after argument parsing. + "QEMU_IMAGE_OS": "flatcar", + }, + ) + + self.assertNotIn("unbound variable", result.stderr) + self.assertIn("does not support Flatcar images", result.stderr) + + def test_no_bare_positional_expansion_after_a_shift(self): + text = RUNNER.read_text(encoding="utf-8") + + self.assertNotIn('=("${@}")', text, "the ${@+...} guard is required") + + +class OutputDirectoryTests(unittest.TestCase): + def test_the_caller_supplied_output_directory_is_never_removed(self): + # NODE_CONFORMANCE_OUTPUT_DIR is caller supplied, so an rm -rf on it + # would delete whatever the caller pointed at, including a cwd. + runner = RUNNER.read_text(encoding="utf-8") + + self.assertNotIn('rm -rf "${output_dir}"', runner) + self.assertIn('mkdir -p "${output_dir}"', runner) + self.assertIn('run_dir="$(mktemp -d "${output_dir}/', runner) + + def test_results_are_evaluated_from_the_per_run_directory(self): + runner = RUNNER.read_text(encoding="utf-8") + + self.assertIn('node_conformance_summary_exit_code "${run_dir}/summary.env"', runner) + + +class SignalHandlingTests(unittest.TestCase): + def test_interrupts_run_the_exit_cleanup(self): + text = RUNNER.read_text(encoding="utf-8") + + self.assertIn("trap cleanup EXIT", text) + self.assertIn("trap 'exit 130' INT TERM", text) + + +class DownloadHardeningTests(unittest.TestCase): + def test_downloads_retry_and_are_time_capped(self): + hook = HOOK.read_text(encoding="utf-8") + + self.assertIn("--retry 3 --retry-delay 5 --retry-connrefused", hook) + self.assertIn('--max-time "${max_time}"', hook) + # Every download goes through the hardened helper. + self.assertEqual(1, hook.count("curl --fail")) + + +class ImageIsNotModifiedTests(unittest.TestCase): + def test_packer_template_has_no_node_conformance_provisioners(self): + template = json.loads(PACKER_TEMPLATE.read_text(encoding="utf-8")) + serialized = json.dumps(template) + + self.assertNotIn("node_conformance", serialized) + self.assertNotIn("run-e2e-node-conformance", serialized) + + def test_conformance_runs_on_a_copy_on_write_overlay(self): + runner = RUNNER.read_text(encoding="utf-8") + + self.assertIn("qemu_guest_create_overlay", runner) + self.assertIn("qemu_guest_create_overlay", QEMU_GUEST_LIB.read_text(encoding="utf-8")) + + def test_boot_smoke_and_conformance_share_the_qemu_guest_library(self): + for script in (RUNNER, BOOT_SMOKE): + self.assertIn( + 'source "${script_dir}/lib/qemu-guest.sh"', + script.read_text(encoding="utf-8"), + f"{script} should reuse the shared QEMU guest helpers", + ) + + +class DocumentationTests(unittest.TestCase): + def test_documented_hook_defaults_match_the_script(self): + script = HOOK.read_text(encoding="utf-8") + documented = documented_defaults() + + for name in ( + "NODE_CONFORMANCE_FOCUS", + "NODE_CONFORMANCE_SKIP", + "NODE_CONFORMANCE_PARALLELISM", + "NODE_CONFORMANCE_FLAKE_ATTEMPTS", + "NODE_CONFORMANCE_TIMEOUT", + "NODE_CONFORMANCE_STANDALONE_MODE", + "NODE_CONFORMANCE_KUBELET_FLAGS", + "NODE_CONFORMANCE_ETCD_VERSION", + "NODE_CONFORMANCE_DOWNLOAD_TIMEOUT", + "NODE_CONFORMANCE_RESULTS_DIR", + ): + self.assertIn(name, documented) + self.assertEqual(shell_default(script, name), documented[name], name) + + def test_documented_runner_defaults_match_the_script(self): + script = RUNNER.read_text(encoding="utf-8") + documented = documented_defaults() + + for name in ("QEMU_CPUS", "QEMU_MEMORY", "QEMU_SSH_TIMEOUT"): + self.assertIn(name, documented) + self.assertEqual(shell_default(script, name), documented[name], name) + + +if __name__ == "__main__": + unittest.main() diff --git a/images/capi/scripts/ci-qemu-node-conformance.sh b/images/capi/scripts/ci-qemu-node-conformance.sh new file mode 100755 index 0000000000..c1d2b7f1dd --- /dev/null +++ b/images/capi/scripts/ci-qemu-node-conformance.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds a QEMU node image and then runs Kubernetes node conformance against the +# built artifact from a throwaway copy-on-write overlay, so the shipped image is +# never modified by the test. + +set -o errexit +set -o nounset +set -o pipefail + +[[ -n ${DEBUG:-} ]] && set -o xtrace + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +capi_dir="$(cd -- "${script_dir}/.." && pwd)" + +target="${NODE_CONFORMANCE_TARGET:-build-qemu-ubuntu-2404-cloudimg}" +cpus="${NODE_CONFORMANCE_CPUS:-4}" +memory="${NODE_CONFORMANCE_MEMORY:-8192}" +accelerator="${NODE_CONFORMANCE_ACCELERATOR:-kvm}" + +case "${target}" in +*flatcar*) + echo "NODE_CONFORMANCE_TARGET=${target} is not supported for node conformance; Flatcar uses Ignition and is explicitly excluded." >&2 + exit 1 + ;; +esac + +if [[ "${accelerator}" == "kvm" && ! -e /dev/kvm ]]; then + echo "NODE_CONFORMANCE_ACCELERATOR=kvm requires /dev/kvm in the CI container." >&2 + echo "Use a nested-virtualization capable runner, or set NODE_CONFORMANCE_ACCELERATOR=tcg for slow local debugging." >&2 + exit 1 +fi + +cd "${capi_dir}" +make deps-qemu "${target}" + +# The build target writes to output/-kube-. Pick +# the directory the build just produced. +output_dirs=() +while IFS= read -r output_dir; do + output_dirs+=("${output_dir}") +done < <(find output -mindepth 1 -maxdepth 1 -type d | sort) +if [[ "${#output_dirs[@]}" -ne 1 ]]; then + echo "expected exactly one build output directory under output/; found ${#output_dirs[@]}" >&2 + printf '%s\n' "${output_dirs[@]+"${output_dirs[@]}"}" >&2 + exit 1 +fi + +QEMU_ACCELERATOR="${accelerator}" \ + QEMU_CPUS="${cpus}" \ + QEMU_MEMORY="${memory}" \ + hack/qemu-node-conformance.sh "${output_dirs[0]}"