From 1e67f096eefabc9ae33f5caf08fb97ebb2e8034c Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 10:16:48 -0500 Subject: [PATCH 1/6] Add Ubuntu 24.04.5 (kernel 7.0) support with dual-compat layer Raise the DKMS compat ceiling from 6.17 to 7.0 so the 6.9.5 source tree builds on Noble HWE without changing PACKAGE_VERSION. --- .github/workflows/ci.yml | 17 +-- ARCHITECTURE.md | 10 +- README.md | 15 ++- include/dkms_compat.h | 15 ++- include/linux/dpll.h | 192 +++++++++++++++++++++++++++++++++ include/nsim_rename.h | 2 + netdevsim/dev.c | 1 + netdevsim/health.c | 14 +++ scripts/test-libvirt-ubuntu.sh | 6 +- scripts/test-utm-ubuntu.sh | 8 +- 10 files changed, 262 insertions(+), 18 deletions(-) create mode 100644 include/linux/dpll.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05db1d8..b6ea8e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,11 +40,12 @@ jobs: run: | sudo apt-get update sudo apt-get install -y dkms gcc make ethtool linuxptp \ - linux-headers-$(uname -r) \ - linux-modules-extra-$(uname -r) || { - echo "::error::Could not install kernel headers/modules for $(uname -r)" - exit 1 - } + linux-headers-$(uname -r) + # linux-modules-extra is not published for every HWE ABI + # (e.g. 7.0 generic); extras live in linux-modules-* instead. + if ! sudo apt-get install -y linux-modules-extra-$(uname -r); then + echo "linux-modules-extra-$(uname -r) not available — continuing" + fi - name: Install DKMS source tree run: | @@ -250,8 +251,10 @@ jobs: - name: Install DKMS dependencies run: | sudo apt-get install -y dkms gcc make ethtool linuxptp \ - linux-headers-$(uname -r) \ - linux-modules-extra-$(uname -r) + linux-headers-$(uname -r) + if ! sudo apt-get install -y linux-modules-extra-$(uname -r); then + echo "linux-modules-extra-$(uname -r) not available — continuing" + fi - name: Build and install DKMS modules run: | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 27b4c23..c810021 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -137,7 +137,7 @@ renaming happens at the preprocessor level. ## Kernel Version Compatibility Layer `include/dkms_compat.h` provides shims so the 6.9.5 source compiles on -kernels from **6.8** (Ubuntu 22.04) through **6.17** (Ubuntu 24.04 HWE): +kernels from **6.8** (Ubuntu 22.04) through **7.0** (Ubuntu 24.04.5 HWE): | Shim | Kernel boundary | What changed | |------|----------------|--------------| @@ -154,7 +154,9 @@ kernels from **6.8** (Ubuntu 22.04) through **6.17** (Ubuntu 24.04 HWE): | `HAVE_DEBUGFS_GET_AUX` | >= 6.16 | `debugfs_real_fops` removed, new aux API | | `HAVE_XFRMDEV_OPS_DEV_PARAM` | >= 6.16 | xfrm callbacks gained `net_device *` param | | `UDP_TUNNEL_NIC_INFO_MAY_SLEEP` | >= 6.17 → stub 0 | Flag removed from tunnel infrastructure | +| `HAVE_DEVLINK_HEALTH_REPORTER_OPS_PERIOD` | >= 6.18 | `devl_health_reporter_create` dropped graceful_period | | `NSIM_ETHTOOL_TS_INFO` | >= 6.11 | `ethtool_ts_info` → `kernel_ethtool_ts_info` | +| vendored `include/linux/dpll.h` | >= 7.0 | Host DPLL kAPI added `dpll_tracker *`; keep 6.9.5 get/put | Each block is guarded by `LINUX_VERSION_CODE` so the code compiles cleanly on the native 6.9.x tree as well (no shims active). @@ -221,9 +223,10 @@ Format: ` [num_ports]` netdevsim-dkms/ ├── .github/workflows/ci.yml # GitHub Actions CI ├── include/ -│ ├── dkms_compat.h # Kernel version shims (6.8 → 6.17) +│ ├── dkms_compat.h # Kernel version shims (6.8 → 7.0) │ ├── nsim_rename.h # nsim_ symbol prefix macros │ └── linux/ +│ ├── dpll.h # 6.9.5 DPLL kAPI on >= 7.0; include_next otherwise │ ├── ptp_clock_kernel.h # PTP kernel API header │ └── ptp_mock.h # Mock PHC API header ├── ptp/ # nsim_ptp.ko + nsim_ptp_mock.ko @@ -337,5 +340,6 @@ cloud credentials needed). | Ubuntu | Kernel | Status | |--------|--------|--------| | 22.04 (HWE) | 6.8.x | Tested in CI | -| 24.04 (HWE) | 6.17.x | Tested in CI | +| 24.04.4 (HWE) | 6.17.x | Dual-compat layer | +| 24.04.5 (HWE) | 7.0.x | Tested in CI (ubuntu-24.04) | | 6.9.x (native) | 6.9.x | Source origin — no shims needed | diff --git a/README.md b/README.md index 6457cc1..94e874a 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,16 @@ Based on Linux 6.9.5 kernel sources. ## Kernel Compatibility -This package targets **Linux 6.9.x** kernels. Internal kernel APIs (devlink, -dpll, netdevice, PTP) may differ in other kernel versions. Building against a -substantially different kernel version will likely require source modifications. +This package is based on **Linux 6.9.5** sources. `include/dkms_compat.h` +shims the same tree onto nearby kernels: + +| Ubuntu | Kernel | +|--------|--------| +| 22.04 HWE | 6.8.x | +| 24.04.4 HWE | 6.17.x | +| 24.04.5 HWE | 7.0.x | + +Building against a kernel outside this range will likely need more shims. ## Prerequisites @@ -273,7 +280,7 @@ you to remove it or pick a different `--vm-name`. # Ubuntu 22.04 (kernel 6.8 HWE) — smoke tests only ./scripts/test-utm-ubuntu.sh --release 22.04 --skip-ptp-operator --shell -# Ubuntu 24.04 (kernel 6.17 HWE) — smoke tests only +# Ubuntu 24.04 (kernel 7.0 HWE) — smoke tests only ./scripts/test-utm-ubuntu.sh --release 24.04 --skip-ptp-operator --shell ``` diff --git a/include/dkms_compat.h b/include/dkms_compat.h index 304e56a..a177134 100644 --- a/include/dkms_compat.h +++ b/include/dkms_compat.h @@ -3,8 +3,8 @@ * Kernel version compatibility layer for netdevsim DKMS package. * * The DKMS sources originate from Linux 6.9.5. This header provides - * shims so the same source tree compiles against both 6.8.x (Ubuntu - * 22.04) and 6.17.x (Ubuntu 24.04 HWE) kernels. + * shims so the same source tree compiles against 6.8.x (Ubuntu 22.04), + * 6.17.x (Ubuntu 24.04.4 HWE), and 7.0.x (Ubuntu 24.04.5 HWE) kernels. * * Each compat block is guarded by LINUX_VERSION_CODE so the module * builds cleanly on the native 6.9.x tree as well. @@ -204,6 +204,17 @@ __dkms_hrtimer_dummy(struct hrtimer *t) #define UDP_TUNNEL_NIC_INFO_MAY_SLEEP 0 #endif +/* + * ---- devl_health_reporter_create signature -------------------------------- + * + * Commit d2b007374551 ("devlink: Move graceful period parameter to + * reporter ops"), merged in v6.18, dropped the graceful_period argument. + * The period now lives in ops->default_graceful_period (0 when unset). + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) + #define HAVE_DEVLINK_HEALTH_REPORTER_OPS_PERIOD 1 +#endif + /* * ---- kernel_ethtool_ts_info ----------------------------------------------- * diff --git a/include/linux/dpll.h b/include/linux/dpll.h new file mode 100644 index 0000000..7895c9c --- /dev/null +++ b/include/linux/dpll.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DPLL kAPI used by this DKMS tree. + * + * On kernels < 7.0, defer to the host linux/dpll.h (6.8 / 6.9 / 6.17 + * signatures already match our 6.9.5 sources via dkms_compat.h). + * + * Linux 7.0 added a dpll_tracker * argument to dpll_device_get/put and + * dpll_pin_get/put, and exported __dpll_*_change_ntf from the in-tree + * header. Those collide with nsim_ symbol renaming and with our 6.9.5 + * dpll_core.c / dpll_netlink.c. nsim_dpll implements the 6.9.5 kAPI, so + * on >= 7.0 we compile against that snapshot instead of the host header. + */ + +#ifndef _NSIM_DKMS_LINUX_DPLL_H_ +#define _NSIM_DKMS_LINUX_DPLL_H_ + +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 0, 0) +#include_next +#else + +#ifndef __DPLL_H__ +#define __DPLL_H__ + +/* + * Copyright (c) 2023 Meta Platforms, Inc. and affiliates + * Copyright (c) 2023 Intel and affiliates + * + * Snapshot of include/linux/dpll.h from Linux 6.9.5. + */ + +#include +#include +#include +#include +#include + +struct dpll_device; +struct dpll_pin; + +struct dpll_device_ops { + int (*mode_get)(const struct dpll_device *dpll, void *dpll_priv, + enum dpll_mode *mode, struct netlink_ext_ack *extack); + int (*lock_status_get)(const struct dpll_device *dpll, void *dpll_priv, + enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, + struct netlink_ext_ack *extack); + int (*temp_get)(const struct dpll_device *dpll, void *dpll_priv, + s32 *temp, struct netlink_ext_ack *extack); +}; + +struct dpll_pin_ops { + int (*frequency_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + const u64 frequency, + struct netlink_ext_ack *extack); + int (*frequency_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + u64 *frequency, struct netlink_ext_ack *extack); + int (*direction_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + const enum dpll_pin_direction direction, + struct netlink_ext_ack *extack); + int (*direction_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + enum dpll_pin_direction *direction, + struct netlink_ext_ack *extack); + int (*state_on_pin_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_pin *parent_pin, + void *parent_pin_priv, + enum dpll_pin_state *state, + struct netlink_ext_ack *extack); + int (*state_on_dpll_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, + void *dpll_priv, enum dpll_pin_state *state, + struct netlink_ext_ack *extack); + int (*state_on_pin_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_pin *parent_pin, + void *parent_pin_priv, + const enum dpll_pin_state state, + struct netlink_ext_ack *extack); + int (*state_on_dpll_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, + void *dpll_priv, + const enum dpll_pin_state state, + struct netlink_ext_ack *extack); + int (*prio_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + u32 *prio, struct netlink_ext_ack *extack); + int (*prio_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + const u32 prio, struct netlink_ext_ack *extack); + int (*phase_offset_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + s64 *phase_offset, + struct netlink_ext_ack *extack); + int (*phase_adjust_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + s32 *phase_adjust, + struct netlink_ext_ack *extack); + int (*phase_adjust_set)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + const s32 phase_adjust, + struct netlink_ext_ack *extack); + int (*ffo_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + s64 *ffo, struct netlink_ext_ack *extack); +}; + +struct dpll_pin_frequency { + u64 min; + u64 max; +}; + +#define DPLL_PIN_FREQUENCY_RANGE(_min, _max) \ + { \ + .min = _min, \ + .max = _max, \ + } + +#define DPLL_PIN_FREQUENCY(_val) DPLL_PIN_FREQUENCY_RANGE(_val, _val) +#define DPLL_PIN_FREQUENCY_1PPS \ + DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_1_HZ) +#define DPLL_PIN_FREQUENCY_10MHZ \ + DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_10_MHZ) +#define DPLL_PIN_FREQUENCY_IRIG_B \ + DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_10_KHZ) +#define DPLL_PIN_FREQUENCY_DCF77 \ + DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_77_5_KHZ) + +struct dpll_pin_phase_adjust_range { + s32 min; + s32 max; +}; + +struct dpll_pin_properties { + const char *board_label; + const char *panel_label; + const char *package_label; + enum dpll_pin_type type; + unsigned long capabilities; + u32 freq_supported_num; + struct dpll_pin_frequency *freq_supported; + struct dpll_pin_phase_adjust_range phase_range; +}; + +void dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin); +void dpll_netdev_pin_clear(struct net_device *dev); + +size_t dpll_netdev_pin_handle_size(const struct net_device *dev); +int dpll_netdev_add_pin_handle(struct sk_buff *msg, + const struct net_device *dev); + +struct dpll_device * +dpll_device_get(u64 clock_id, u32 dev_driver_id, struct module *module); + +void dpll_device_put(struct dpll_device *dpll); + +int dpll_device_register(struct dpll_device *dpll, enum dpll_type type, + const struct dpll_device_ops *ops, void *priv); + +void dpll_device_unregister(struct dpll_device *dpll, + const struct dpll_device_ops *ops, void *priv); + +struct dpll_pin * +dpll_pin_get(u64 clock_id, u32 dev_driver_id, struct module *module, + const struct dpll_pin_properties *prop); + +int dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin, + const struct dpll_pin_ops *ops, void *priv); + +void dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin, + const struct dpll_pin_ops *ops, void *priv); + +void dpll_pin_put(struct dpll_pin *pin); + +int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin, + const struct dpll_pin_ops *ops, void *priv); + +void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin, + const struct dpll_pin_ops *ops, void *priv); + +int dpll_device_change_ntf(struct dpll_device *dpll); + +int dpll_pin_change_ntf(struct dpll_pin *pin); + +#endif /* __DPLL_H__ */ +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(7, 0, 0) */ + +#endif /* _NSIM_DKMS_LINUX_DPLL_H_ */ diff --git a/include/nsim_rename.h b/include/nsim_rename.h index 8153a12..3b0f26a 100644 --- a/include/nsim_rename.h +++ b/include/nsim_rename.h @@ -69,6 +69,8 @@ /* ---- DPLL netlink (dpll_netlink.c) ------------------------------------- */ #define dpll_device_change_ntf nsim_dpll_device_change_ntf #define dpll_pin_change_ntf nsim_dpll_pin_change_ntf +#define __dpll_device_change_ntf nsim_dpll_device_change_ntf_internal +#define __dpll_pin_change_ntf nsim_dpll_pin_change_ntf_internal /* ---- DPLL internal (dpll_core.h / dpll_*.c) ---------------------------- */ #define dpll_nl_family nsim_dpll_nl_family diff --git a/netdevsim/dev.c b/netdevsim/dev.c index 18fe57f..56ca8d2 100644 --- a/netdevsim/dev.c +++ b/netdevsim/dev.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/netdevsim/health.c b/netdevsim/health.c index 70e8bdf..ea5307c 100644 --- a/netdevsim/health.c +++ b/netdevsim/health.c @@ -178,17 +178,31 @@ int nsim_dev_health_init(struct nsim_dev *nsim_dev, struct devlink *devlink) struct nsim_dev_health *health = &nsim_dev->health; int err; +#ifdef HAVE_DEVLINK_HEALTH_REPORTER_OPS_PERIOD + health->empty_reporter = + devl_health_reporter_create(devlink, + &nsim_dev_empty_reporter_ops, + health); +#else health->empty_reporter = devl_health_reporter_create(devlink, &nsim_dev_empty_reporter_ops, 0, health); +#endif if (IS_ERR(health->empty_reporter)) return PTR_ERR(health->empty_reporter); +#ifdef HAVE_DEVLINK_HEALTH_REPORTER_OPS_PERIOD + health->dummy_reporter = + devl_health_reporter_create(devlink, + &nsim_dev_dummy_reporter_ops, + health); +#else health->dummy_reporter = devl_health_reporter_create(devlink, &nsim_dev_dummy_reporter_ops, 0, health); +#endif if (IS_ERR(health->dummy_reporter)) { err = PTR_ERR(health->dummy_reporter); goto err_empty_reporter_destroy; diff --git a/scripts/test-libvirt-ubuntu.sh b/scripts/test-libvirt-ubuntu.sh index b9e15b6..3ace878 100755 --- a/scripts/test-libvirt-ubuntu.sh +++ b/scripts/test-libvirt-ubuntu.sh @@ -262,6 +262,7 @@ EOF HWE_PACKAGES="" if [[ "$UBUNTU_RELEASE" == "24.04" ]]; then + # linux-generic-hwe-24.04 tracks the current Noble HWE kernel (7.0 as of 24.04.5). HWE_PACKAGES=" - linux-generic-hwe-24.04 - linux-headers-generic-hwe-24.04" @@ -461,7 +462,10 @@ log "Ensuring kernel headers and extra modules are installed ..." vm_ssh sudo bash -c "' set -euo pipefail apt-get update -qq -apt-get install -y -qq linux-headers-\$(uname -r) linux-modules-extra-\$(uname -r) dkms gcc make 2>&1 | tail -5 +apt-get install -y -qq linux-headers-\$(uname -r) dkms gcc make 2>&1 | tail -5 +if ! apt-get install -y -qq linux-modules-extra-\$(uname -r) 2>/dev/null; then + echo \" linux-modules-extra-\$(uname -r) not available — continuing\" +fi echo \"Headers dir: /lib/modules/\$(uname -r)/build\" ls /lib/modules/\$(uname -r)/build/Makefile >/dev/null 2>&1 && echo \" OK\" || echo \" MISSING\" '" diff --git a/scripts/test-utm-ubuntu.sh b/scripts/test-utm-ubuntu.sh index f417db1..0491a86 100755 --- a/scripts/test-utm-ubuntu.sh +++ b/scripts/test-utm-ubuntu.sh @@ -206,6 +206,7 @@ EOF HWE_PACKAGES="" if [[ "$UBUNTU_RELEASE" == "24.04" ]]; then + # linux-generic-hwe-24.04 tracks the current Noble HWE kernel (7.0 as of 24.04.5). HWE_PACKAGES=" - linux-generic-hwe-24.04 - linux-headers-generic-hwe-24.04" @@ -591,7 +592,12 @@ log "Ensuring kernel headers and extra modules are installed ..." vm_ssh sudo bash -c "' set -euo pipefail apt-get update -qq -apt-get install -y -qq linux-headers-\$(uname -r) linux-modules-extra-\$(uname -r) dkms gcc make 2>&1 | tail -5 +apt-get install -y -qq linux-headers-\$(uname -r) dkms gcc make 2>&1 | tail -5 +# linux-modules-extra-\$(uname -r) is not published for every HWE ABI +# (e.g. 7.0 generic); extras live in linux-modules-* instead. +if ! apt-get install -y -qq linux-modules-extra-\$(uname -r) 2>/dev/null; then + echo \" linux-modules-extra-\$(uname -r) not available — continuing\" +fi echo \"Headers dir: /lib/modules/\$(uname -r)/build\" ls /lib/modules/\$(uname -r)/build/Makefile >/dev/null 2>&1 && echo \" OK\" || echo \" MISSING\" '" From e4e1e61f0e59eb632db76c631c4d119b25ddfd09 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 11:24:41 -0500 Subject: [PATCH 2/6] ci: add podman API wrap and disk/memory cleanup from ptp-operator ptp-test can hang forever on `podman exec switch1` because daemonless Kind execs deadlock on libpod SQLite. Retry uses a fresh runner with the same workflow, so it wedges on the same line. Mirror ptp-operator netdevsim-ci: serialize podman through one API server, drop image tarball and caches before e2e, and drop page cache. --- .github/workflows/ci.yml | 24 +++++++- scripts/start-podman-api-service.sh | 92 +++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) create mode 100755 scripts/start-podman-api-service.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6ea8e5..3f8c960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,6 +171,8 @@ jobs: sudo apt-get update sudo apt-get install -y podman podman --version + # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). + sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" - name: Clone ptp-operator run: | @@ -182,7 +184,7 @@ jobs: sudo bash -l <<'BUILD' set -euo pipefail set -x - export PATH=/usr/local/go/bin:/root/go/bin:$PATH + export PATH=/usr/local/ptp-podman-wrap:/usr/local/bin:/usr/local/go/bin:/root/go/bin:$PATH export KIND_EXPERIMENTAL_PROVIDER=podman export BUILDAH_LAYERS=true export BUILDAH_HISTORY=true @@ -191,6 +193,8 @@ jobs: VM_IP=$(hostname -I | awk '{print $1}') cd /root/ptp-operator/scripts ./run-on-vm.sh --dkms --images "${VM_IP}" + + rm -rf /tmp/go-build* /root/.cache/go-build BUILD - name: Upload image tarball @@ -238,6 +242,8 @@ jobs: sudo apt-get install -y podman fi podman --version + # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). + sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" - name: Install OpenShift CLI (oc) run: | @@ -284,7 +290,7 @@ jobs: sudo bash -l <<'LOAD' set -euo pipefail set -x - export PATH=/usr/local/go/bin:/root/go/bin:$PATH + export PATH=/usr/local/ptp-podman-wrap:/usr/local/bin:/usr/local/go/bin:/root/go/bin:$PATH export KIND_EXPERIMENTAL_PROVIDER=podman VM_IP=$(hostname -I | awk '{print $1}') @@ -292,12 +298,24 @@ jobs: ./run-on-vm.sh --dkms --load /tmp/ptp-images.tar "${VM_IP}" LOAD + - name: "Free disk space before tests" + run: | + sudo bash -l <<'CLEANUP' + set -euo pipefail + rm -rf /tmp/ptp-images.tar /tmp/ptp-images-load + rm -rf /tmp/go-build* /var/cache/apt /root/.cache + sync + echo 3 > /proc/sys/vm/drop_caches || true + df -h + free -h + CLEANUP + - name: "Run scenario: ${{ matrix.mode }}" run: | sudo bash -l <<'DEPLOY' set -euo pipefail set -x - export PATH=/usr/local/go/bin:/root/go/bin:$PATH + export PATH=/usr/local/ptp-podman-wrap:/usr/local/bin:/usr/local/go/bin:/root/go/bin:$PATH export KIND_EXPERIMENTAL_PROVIDER=podman VM_IP=$(hostname -I | awk '{print $1}') diff --git a/scripts/start-podman-api-service.sh b/scripts/start-podman-api-service.sh new file mode 100755 index 0000000..585e8a7 --- /dev/null +++ b/scripts/start-podman-api-service.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Run all podman CLI invocations through a single API server. +# +# Kind fires parallel `podman exec` during "Writing configuration". Each +# daemonless podman process locks the shared libpod SQLite DB; concurrent +# execs into systemd (kindest/node) containers deadlock (futex_wait). A +# single `podman system service` serializes DB access like dockerd does. +# +# Usage: start-podman-api-service.sh +# Idempotent. Safe to call from CI after overlay config / before Kind. +set -euo pipefail + +if [[ "$(id -u)" -ne 0 ]]; then + exec sudo -E bash "$0" "$@" +fi + +SOCK="${PODMAN_API_SOCK:-unix:///run/podman/podman.sock}" +SOCK_PATH="${SOCK#unix://}" +# Dedicated dir so we never overwrite the real podman binary (kubic may +# install it at /usr/local/bin/podman → ETXTBSY if we try to replace it). +WRAP_DIR="${PODMAN_WRAP_DIR:-/usr/local/ptp-podman-wrap}" +WRAPPER="${WRAP_DIR}/podman" + +mkdir -p "$(dirname "${SOCK_PATH}")" "${WRAP_DIR}" + +# Resolve the real binary before we put our wrapper ahead on PATH. +REAL_PODMAN="${REAL_PODMAN:-}" +if [[ -z "${REAL_PODMAN}" ]]; then + for candidate in /usr/bin/podman /usr/local/bin/podman; do + if [[ -x "${candidate}" && "${candidate}" != "${WRAPPER}" ]]; then + # Skip if this path is already our wrapper script. + if head -n1 "${candidate}" 2>/dev/null | grep -q '^#!'; then + if grep -q 'start-podman-api-service' "${candidate}" 2>/dev/null; then + continue + fi + fi + REAL_PODMAN="${candidate}" + break + fi + done +fi +if [[ -z "${REAL_PODMAN}" || ! -x "${REAL_PODMAN}" ]]; then + echo "ERROR: could not find real podman binary" + exit 1 +fi +echo "Using real podman binary: ${REAL_PODMAN}" + +# Prefer systemd socket (podman.socket) when present; else start a service. +if [[ -S "${SOCK_PATH}" ]] && "${REAL_PODMAN}" --remote --url "${SOCK}" info >/dev/null 2>&1; then + echo "podman API already up at ${SOCK}" +else + # Try enabling the packaged socket first. + systemctl enable --now podman.socket 2>/dev/null || true + sleep 0.5 +fi + +if ! "${REAL_PODMAN}" --remote --url "${SOCK}" info >/dev/null 2>&1; then + rm -f "${SOCK_PATH}" + "${REAL_PODMAN}" system service --time=0 "${SOCK}" >/var/log/podman-api-service.log 2>&1 & + echo $! >/run/podman-api-service.pid + echo "Started podman system service pid=$(cat /run/podman-api-service.pid) sock=${SOCK}" + + for _ in $(seq 1 50); do + if [[ -S "${SOCK_PATH}" ]] && "${REAL_PODMAN}" --remote --url "${SOCK}" info >/dev/null 2>&1; then + break + fi + sleep 0.2 + done +fi + +if ! "${REAL_PODMAN}" --remote --url "${SOCK}" info >/dev/null 2>&1; then + echo "ERROR: podman API failed to become ready" + tail -n 50 /var/log/podman-api-service.log || true + systemctl status podman.socket podman.service 2>&1 | head -40 || true + exit 1 +fi + +# Atomic replace of wrapper (never overwrite /usr/bin or /usr/local/bin/podman). +tmp="$(mktemp "${WRAP_DIR}/podman.XXXXXX")" +cat >"${tmp}" < ${REAL_PODMAN} --remote --url ${SOCK}" +echo "Prepend ${WRAP_DIR} to PATH so Kind/CI use the remote client." +# Smoke-test via the wrapper itself. +PATH="${WRAP_DIR}:${PATH}" "${WRAPPER}" info --format 'API ok GraphDriver={{.Store.GraphDriverName}}' \ + || PATH="${WRAP_DIR}:${PATH}" "${WRAPPER}" info | head -n 20 From e8b04a612f583ae82e8a86ac86019c8958c69663 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 11:59:29 -0500 Subject: [PATCH 3/6] ci: timeout optional extra-modules so stalled apt cannot hang jobs linux-modules-extra exists on ubuntu-22.04 Azure runners and apt can sit forever on the 71MB download. Split it into a 3-minute optional step and cap apt Acquire timeouts so dkms-test cannot block ptp-images. --- .github/workflows/ci.yml | 51 +++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f8c960..010b9ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ jobs: # ------------------------------------------------------------------ dkms-test: runs-on: ${{ matrix.os }} + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -37,15 +38,31 @@ jobs: - uses: actions/checkout@v4 - name: Install DKMS dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y dkms gcc make ethtool linuxptp \ + sudo apt-get update \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + -o Acquire::Retries=2 + sudo apt-get install -y \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + -o Acquire::Retries=2 \ + dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) - # linux-modules-extra is not published for every HWE ABI - # (e.g. 7.0 generic); extras live in linux-modules-* instead. - if ! sudo apt-get install -y linux-modules-extra-$(uname -r); then - echo "linux-modules-extra-$(uname -r) not available — continuing" - fi + + # Optional: missing on some HWE ABIs (e.g. 7.0). When the package + # exists (22.04 Azure), apt can stall forever on a 71MB fetch from + # archive.ubuntu.com — cap it so a dead mirror cannot hang the job. + - name: Install optional extra kernel modules + timeout-minutes: 3 + continue-on-error: true + run: | + sudo apt-get install -y \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + -o Acquire::Retries=1 \ + linux-modules-extra-$(uname -r) - name: Install DKMS source tree run: | @@ -255,12 +272,24 @@ jobs: oc version --client - name: Install DKMS dependencies + timeout-minutes: 5 run: | - sudo apt-get install -y dkms gcc make ethtool linuxptp \ + sudo apt-get install -y \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + -o Acquire::Retries=2 \ + dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) - if ! sudo apt-get install -y linux-modules-extra-$(uname -r); then - echo "linux-modules-extra-$(uname -r) not available — continuing" - fi + + - name: Install optional extra kernel modules + timeout-minutes: 3 + continue-on-error: true + run: | + sudo apt-get install -y \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + -o Acquire::Retries=1 \ + linux-modules-extra-$(uname -r) - name: Build and install DKMS modules run: | From cd923f57a605cf3d94854ccdbd795c5c8a0d9f3a Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 13:23:46 -0500 Subject: [PATCH 4/6] ci: drop extra-modules and kill stalled apt under sudo The 3-minute extra-modules step timeout only killed the shell; sudo apt-get kept the dpkg lock, so later jobs hung on podman install or failed with openvswitch-switch status un. Extra-modules is unused by the out-of-tree DKMS build. Wrap remaining apt in sudo timeout and Acquire timeouts so a dead mirror cannot freeze the runner. --- .github/workflows/ci.yml | 61 +++++++++++----------------------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 010b9ff..2132241 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,32 +38,16 @@ jobs: - uses: actions/checkout@v4 - name: Install DKMS dependencies - timeout-minutes: 5 + timeout-minutes: 8 run: | - sudo apt-get update \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - -o Acquire::Retries=2 - sudo apt-get install -y \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - -o Acquire::Retries=2 \ + # sudo timeout so apt children die with the step (GHA timeout + # only kills the shell; leftover apt holds dpkg locks). + APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) + sudo timeout --kill-after=20 180 "${APT[@]}" update + sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) - # Optional: missing on some HWE ABIs (e.g. 7.0). When the package - # exists (22.04 Azure), apt can stall forever on a 71MB fetch from - # archive.ubuntu.com — cap it so a dead mirror cannot hang the job. - - name: Install optional extra kernel modules - timeout-minutes: 3 - continue-on-error: true - run: | - sudo apt-get install -y \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - -o Acquire::Retries=1 \ - linux-modules-extra-$(uname -r) - - name: Install DKMS source tree run: | sudo mkdir -p /usr/src/${DKMS_PKG}-${DKMS_VER} @@ -177,6 +161,7 @@ jobs: - uses: actions/checkout@v4 - name: Install latest podman + timeout-minutes: 8 run: | . /etc/os-release sudo mkdir -p /etc/apt/keyrings @@ -185,8 +170,9 @@ jobs: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg] \ https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/ /" \ | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null - sudo apt-get update - sudo apt-get install -y podman + APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) + sudo timeout --kill-after=20 180 "${APT[@]}" update + sudo timeout --kill-after=20 180 "${APT[@]}" install -y podman podman --version # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" @@ -243,8 +229,10 @@ jobs: - uses: actions/checkout@v4 - name: Install latest podman + timeout-minutes: 8 run: | . /etc/os-release + APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) if [[ "${VERSION_ID}" == "22.04" ]]; then sudo mkdir -p /etc/apt/keyrings curl -fsSL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/Release.key" \ @@ -252,12 +240,9 @@ jobs: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg] \ https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/ /" \ | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null - sudo apt-get update - sudo apt-get install -y podman - else - sudo apt-get update - sudo apt-get install -y podman fi + sudo timeout --kill-after=20 180 "${APT[@]}" update + sudo timeout --kill-after=20 180 "${APT[@]}" install -y podman podman --version # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" @@ -272,25 +257,13 @@ jobs: oc version --client - name: Install DKMS dependencies - timeout-minutes: 5 + timeout-minutes: 8 run: | - sudo apt-get install -y \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - -o Acquire::Retries=2 \ + APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) + sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) - - name: Install optional extra kernel modules - timeout-minutes: 3 - continue-on-error: true - run: | - sudo apt-get install -y \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - -o Acquire::Retries=1 \ - linux-modules-extra-$(uname -r) - - name: Build and install DKMS modules run: | sudo mkdir -p /usr/src/${DKMS_PKG}-${DKMS_VER} From 96b2068a15ef4f6be671cb593be016b0a38b00cb Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 13:37:17 -0500 Subject: [PATCH 5/6] ci: restore extra-modules so netdevsim can resolve gnss symbols Dropping linux-modules-extra left gnss.ko missing on Azure kernels, so modprobe netdevsim failed with unknown symbol and smoke tests died. Install extra-modules again under sudo timeout, and fail the load step if netdevsim does not actually insert. --- .github/workflows/ci.yml | 27 +++++++++++++------- scripts/ci-install-extra-modules.sh | 39 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 9 deletions(-) create mode 100755 scripts/ci-install-extra-modules.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2132241..716d857 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 - name: Install DKMS dependencies - timeout-minutes: 8 + timeout-minutes: 10 run: | # sudo timeout so apt children die with the step (GHA timeout # only kills the shell; leftover apt holds dpkg locks). @@ -47,8 +47,8 @@ jobs: sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) - - - name: Install DKMS source tree + # gnss.ko is in extra-modules on Azure kernels; netdevsim needs it. + sudo bash "$GITHUB_WORKSPACE/scripts/ci-install-extra-modules.sh" run: | sudo mkdir -p /usr/src/${DKMS_PKG}-${DKMS_VER} sudo cp -a Makefile dkms.conf install-udev-rule.sh 99-nsim-ptp.rules \ @@ -75,11 +75,19 @@ jobs: id: load run: | set -e - sudo modprobe gnss && echo "gnss loaded" || echo "gnss skipped" - sudo modprobe nsim_ptp && echo "nsim_ptp loaded" - sudo modprobe nsim_ptp_mock && echo "nsim_ptp_mock loaded" - sudo modprobe nsim_dpll && echo "nsim_dpll loaded" - sudo modprobe netdevsim pci_bus_nr=0x1f && echo "netdevsim loaded" + sudo modprobe gnss && echo "gnss loaded" || echo "gnss skipped" + sudo modprobe nsim_ptp + echo "nsim_ptp loaded" + sudo modprobe nsim_ptp_mock + echo "nsim_ptp_mock loaded" + sudo modprobe nsim_dpll + echo "nsim_dpll loaded" + if ! sudo modprobe netdevsim pci_bus_nr=0x1f; then + echo "::error::netdevsim failed to load (needs gnss from linux-modules-extra)" + sudo dmesg | tail -n 40 || true + exit 1 + fi + echo "netdevsim loaded" lsmod | grep -E 'nsim_ptp|nsim_dpll|netdevsim|gnss' || true sudo chmod 666 /dev/nsim_ptp* 2>/dev/null || true echo "load_ok=true" >> "$GITHUB_OUTPUT" @@ -257,12 +265,13 @@ jobs: oc version --client - name: Install DKMS dependencies - timeout-minutes: 8 + timeout-minutes: 10 run: | APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) + sudo bash "$GITHUB_WORKSPACE/scripts/ci-install-extra-modules.sh" - name: Build and install DKMS modules run: | diff --git a/scripts/ci-install-extra-modules.sh b/scripts/ci-install-extra-modules.sh new file mode 100755 index 0000000..ccd8816 --- /dev/null +++ b/scripts/ci-install-extra-modules.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Install linux-modules-extra-$(uname -r) when apt has the package. +# +# gnss.ko lives in extra-modules on Azure/GHA kernels; netdevsim will not +# load without those GNSS symbols. Extra-modules is missing on some HWE +# ABIs (e.g. 7.0 generic) — skip in that case. +# +# apt is run under `sudo timeout` so a stalled 71MB fetch cannot outlive +# this script. GitHub's step timeout only kills the shell; leftover sudo +# apt-get holds /var/lib/dpkg/lock-frontend. +set -euo pipefail + +extra="linux-modules-extra-$(uname -r)" +APT=(apt-get + -o Acquire::http::Timeout=20 + -o Acquire::https::Timeout=20 + -o Acquire::Retries=2 +) + +recover_dpkg() { + sudo killall -9 apt-get apt dpkg 2>/dev/null || true + sleep 1 + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock \ + /var/cache/apt/archives/lock /var/lib/apt/lists/lock + sudo dpkg --configure -a || true +} + +if ! apt-cache show "${extra}" >/dev/null 2>&1; then + echo "${extra} not in apt — continuing (gnss may already be in linux-modules)" + exit 0 +fi + +if sudo timeout --kill-after=20 240 "${APT[@]}" install -y "${extra}"; then + exit 0 +fi + +echo "::error::${extra} install timed out or failed; recovering dpkg" +recover_dpkg +exit 1 From aea654a12b5be5fa2ae6322f16dca1c0fa9e97d2 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Tue, 18 Aug 2026 13:43:57 -0500 Subject: [PATCH 6/6] ci: timeout, retry, and fallback mirror for stalled apt fetches Apt's own Retries never fire on a hung Get:. Kill apt-get after 120s, recover dpkg, retry up to three times, force IPv4, disable pipelining, and switch azure.archive.ubuntu.com to archive.ubuntu.com on retry. --- .github/workflows/ci.yml | 38 +++++++++-------- scripts/ci-apt.sh | 63 +++++++++++++++++++++++++++++ scripts/ci-install-extra-modules.sh | 28 ++----------- 3 files changed, 88 insertions(+), 41 deletions(-) create mode 100755 scripts/ci-apt.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 716d857..38446ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,17 +38,20 @@ jobs: - uses: actions/checkout@v4 - name: Install DKMS dependencies - timeout-minutes: 10 + timeout-minutes: 12 run: | - # sudo timeout so apt children die with the step (GHA timeout - # only kills the shell; leftover apt holds dpkg locks). - APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) - sudo timeout --kill-after=20 180 "${APT[@]}" update - sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" update + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" install \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) + + - name: Install extra kernel modules + timeout-minutes: 12 + run: | # gnss.ko is in extra-modules on Azure kernels; netdevsim needs it. sudo bash "$GITHUB_WORKSPACE/scripts/ci-install-extra-modules.sh" + + - name: Install DKMS source tree run: | sudo mkdir -p /usr/src/${DKMS_PKG}-${DKMS_VER} sudo cp -a Makefile dkms.conf install-udev-rule.sh 99-nsim-ptp.rules \ @@ -169,7 +172,7 @@ jobs: - uses: actions/checkout@v4 - name: Install latest podman - timeout-minutes: 8 + timeout-minutes: 10 run: | . /etc/os-release sudo mkdir -p /etc/apt/keyrings @@ -178,9 +181,8 @@ jobs: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg] \ https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/ /" \ | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null - APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) - sudo timeout --kill-after=20 180 "${APT[@]}" update - sudo timeout --kill-after=20 180 "${APT[@]}" install -y podman + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" update + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" install podman podman --version # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" @@ -237,10 +239,9 @@ jobs: - uses: actions/checkout@v4 - name: Install latest podman - timeout-minutes: 8 + timeout-minutes: 10 run: | . /etc/os-release - APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) if [[ "${VERSION_ID}" == "22.04" ]]; then sudo mkdir -p /etc/apt/keyrings curl -fsSL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/Release.key" \ @@ -249,8 +250,8 @@ jobs: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_${VERSION_ID}/ /" \ | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null fi - sudo timeout --kill-after=20 180 "${APT[@]}" update - sudo timeout --kill-after=20 180 "${APT[@]}" install -y podman + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" update + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" install podman podman --version # Serialize podman CLI via one API server (avoids Kind exec SQLite deadlock). sudo bash "$GITHUB_WORKSPACE/scripts/start-podman-api-service.sh" @@ -265,12 +266,15 @@ jobs: oc version --client - name: Install DKMS dependencies - timeout-minutes: 10 + timeout-minutes: 12 run: | - APT=(apt-get -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=2) - sudo timeout --kill-after=20 180 "${APT[@]}" install -y \ + sudo bash "$GITHUB_WORKSPACE/scripts/ci-apt.sh" install \ dkms gcc make ethtool linuxptp \ linux-headers-$(uname -r) + + - name: Install extra kernel modules + timeout-minutes: 12 + run: | sudo bash "$GITHUB_WORKSPACE/scripts/ci-install-extra-modules.sh" - name: Build and install DKMS modules diff --git a/scripts/ci-apt.sh b/scripts/ci-apt.sh new file mode 100755 index 0000000..7d117d7 --- /dev/null +++ b/scripts/ci-apt.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Run apt-get with a hard timeout and retries. +# +# A stalled Azure mirror fetch prints Get: then waits forever. apt's own +# Acquire::Retries does not fire because the download never fails. Kill +# the apt-get process (not just the shell), recover dpkg, and retry. +# Attempt 2+ switches azure.archive.ubuntu.com -> archive.ubuntu.com. +# +# Usage: ci-apt.sh update +# ci-apt.sh install pkg [pkg...] +set -euo pipefail + +ATTEMPTS="${APT_ATTEMPTS:-3}" +TIMEOUT_SECS="${APT_TIMEOUT_SECS:-120}" +cmd="$1" +shift + +APT_OPTS=( + -o Acquire::ForceIPv4=true + -o Acquire::http::Pipeline-Depth=0 + -o Acquire::http::Timeout=20 + -o Acquire::https::Timeout=20 + -o Acquire::Retries=0 +) + +recover_dpkg() { + sudo killall -9 apt-get apt dpkg 2>/dev/null || true + sleep 1 + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock \ + /var/cache/apt/archives/lock /var/lib/apt/lists/lock + sudo dpkg --configure -a || true +} + +switch_to_archive_ubuntu() { + echo "switching apt mirror azure.archive.ubuntu.com -> archive.ubuntu.com" + sudo grep -rl 'azure.archive.ubuntu.com' /etc/apt 2>/dev/null \ + | while read -r f; do + sudo sed -i 's/azure.archive.ubuntu.com/archive.ubuntu.com/g' "$f" + done +} + +for i in $(seq 1 "${ATTEMPTS}"); do + echo "apt ${cmd} attempt ${i}/${ATTEMPTS} (timeout ${TIMEOUT_SECS}s)" + if [[ "${i}" -ge 2 ]]; then + switch_to_archive_ubuntu + fi + + set +e + sudo timeout --kill-after=15 "${TIMEOUT_SECS}" \ + apt-get "${APT_OPTS[@]}" "${cmd}" -y "$@" + rc=$? + set -e + + if [[ "${rc}" -eq 0 ]]; then + exit 0 + fi + echo "apt ${cmd} attempt ${i} failed rc=${rc}; recovering dpkg" + recover_dpkg + sleep $((i * 5)) +done + +echo "::error::apt ${cmd} failed after ${ATTEMPTS} attempts" +exit 1 diff --git a/scripts/ci-install-extra-modules.sh b/scripts/ci-install-extra-modules.sh index ccd8816..e81ddad 100755 --- a/scripts/ci-install-extra-modules.sh +++ b/scripts/ci-install-extra-modules.sh @@ -4,36 +4,16 @@ # gnss.ko lives in extra-modules on Azure/GHA kernels; netdevsim will not # load without those GNSS symbols. Extra-modules is missing on some HWE # ABIs (e.g. 7.0 generic) — skip in that case. -# -# apt is run under `sudo timeout` so a stalled 71MB fetch cannot outlive -# this script. GitHub's step timeout only kills the shell; leftover sudo -# apt-get holds /var/lib/dpkg/lock-frontend. set -euo pipefail extra="linux-modules-extra-$(uname -r)" -APT=(apt-get - -o Acquire::http::Timeout=20 - -o Acquire::https::Timeout=20 - -o Acquire::Retries=2 -) - -recover_dpkg() { - sudo killall -9 apt-get apt dpkg 2>/dev/null || true - sleep 1 - sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock \ - /var/cache/apt/archives/lock /var/lib/apt/lists/lock - sudo dpkg --configure -a || true -} +here="$(cd "$(dirname "$0")" && pwd)" if ! apt-cache show "${extra}" >/dev/null 2>&1; then echo "${extra} not in apt — continuing (gnss may already be in linux-modules)" exit 0 fi -if sudo timeout --kill-after=20 240 "${APT[@]}" install -y "${extra}"; then - exit 0 -fi - -echo "::error::${extra} install timed out or failed; recovering dpkg" -recover_dpkg -exit 1 +# 71MB package: 180s per attempt, 3 tries, fallback mirror on retry. +APT_TIMEOUT_SECS=180 APT_ATTEMPTS=3 \ + bash "${here}/ci-apt.sh" install "${extra}"