diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3f385..1142337 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ All notable changes to RigForge are documented here. The format is based on ## [Unreleased] +### Fixed + +- **Runtime HugePages reservation is grow-only (#328).** `tune_kernel`'s runtime sysctl wrote the + miner's computed requirement absolutely, which SHRINKS a pool another consumer already reserved + (a co-hosted pithead stack's p2pool/monerod share the same 2MB pool) down to its in-use floor — + zero free pages on both sides, measured live on the pithead#797 appliance bench. The write is now + availability-based: pages the miner can draw on = free + whatever a running miner already holds; + the pool grows by the shortfall only and never shrinks. Fresh single-purpose rigs see the same + reservation as before; re-runs that find enough available write nothing. This is the runtime half + of #305's co-resident keep-existing guard, and it applies with or without + `hugepages_reserve_extra_mb` set. The `setup --dry-run` plan previews the same decision. + ## [1.12.0] - 2026-07-19 The pithead#597 producer release: the control-upgrade `/status` contract a one-click worker diff --git a/rigforge.sh b/rigforge.sh index 276077c..330c17f 100755 --- a/rigforge.sh +++ b/rigforge.sh @@ -108,6 +108,7 @@ MEMINFO="${MEMINFO:-/proc/meminfo}" MSR_MODULE_DIR="${MSR_MODULE_DIR:-/sys/module/msr}" GOVERNOR_FILE="${GOVERNOR_FILE:-/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor}" HUGEPAGES_1G_NR="${HUGEPAGES_1G_NR:-/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages}" +NR_HUGEPAGES_FILE="${NR_HUGEPAGES_FILE:-/proc/sys/vm/nr_hugepages}" # Hashrate-capping-hardware diagnostics (#67): RAM layout (dmidecode) + effective CPU clock under load. DMIDECODE="${DMIDECODE:-dmidecode}" RDMSR_BIN="${RDMSR_BIN:-rdmsr}" # msr-tools, for doctor's register-level MSR verification (#66) @@ -1337,6 +1338,48 @@ _cmdline_reserved_2mb() { # -> reserved 2MB-equivalent pages awk '{ cur=""; def=""; t=0; for (i=1;i<=NF;i++) { if ($i ~ /^hugepagesz=/) { split($i,a,"="); cur=a[2] } else if ($i ~ /^default_hugepagesz=/) { split($i,a,"="); def=a[2] } else if ($i ~ /^hugepages=/) { split($i,a,"="); n=a[2]+0; s=(cur!=""?cur:def); if (s=="1G"||s=="1073741824"||s=="1048576K") t+=n*512; else t+=n } } print t+0 }' <<<"$1" } +# 2MB pages the running miner already holds (kB -> pages), 0 when stopped: pages it holds now are +# pages it re-uses across a restart, so they count as available when sizing the pool. HugetlbPages +# spans every page size, so a dataset sitting in 1GB pages overstates this credit — in exactly the +# case where the 2MB pool is not needed for the dataset, so nothing actually used goes unreserved. +_miner_held_hugepages() { + local pid kb + pid=$(systemctl show "$SERVICE_NAME.service" -p MainPID --value 2>/dev/null) || pid="" + if [ -n "$pid" ] && [ "$pid" -gt 0 ] 2>/dev/null && [ -r "/proc/$pid/status" ]; then + kb=$(awk '/^HugetlbPages:/ { print $2; exit }' "/proc/$pid/status" 2>/dev/null) + echo $((${kb:-0} / 2048)) + return + fi + echo 0 +} + +# Grow-only runtime reservation (#328, the runtime half of #305's keep-existing guard): another +# consumer may hold pages from the same pool (a co-hosted pithead stack's p2pool/monerod), and +# writing our raw requirement shrinks that pool to its in-use floor — 0 free pages on both sides, +# measured live on the pithead#797 appliance bench. Ensure the miner's pages are AVAILABLE (free +# now, or already held by the running miner) and grow the pool by the shortfall only; never shrink +# a reservation someone else made. Idempotent: a re-run that finds enough available writes nothing. +# ponytail: with headroom configured AND the co-resident already running, its live pages and the +# headroom both count toward the target — a bounded over-reserve in the safe direction; tighten to +# max(headroom, live use) if a real box ever needs the difference. +_hugepages_avail() { # -> 2MB pages the miner could draw on right now (free + already held) + local free held + free=$(awk '/^HugePages_Free:/ { print $2; exit }' "$MEMINFO" 2>/dev/null) || free="" + held=$(_miner_held_hugepages) + echo $((${free:-0} + held)) +} + +_ensure_hugepages() { # + local required=$1 current avail + current=$(cat "$NR_HUGEPAGES_FILE" 2>/dev/null) || current=0 + avail=$(_hugepages_avail) + if [ "$avail" -lt "$required" ]; then + sudo sysctl -w vm.nr_hugepages=$((current + required - avail)) + else + log "HugePages pool already covers the miner ($avail of $required pages available; pool: $current) — leaving it as-is (#328)." + fi +} + tune_kernel() { if [ "$OS_TYPE" != "Linux" ]; then log "Skipping kernel tuning (Not supported on $OS_TYPE)." @@ -1361,13 +1404,13 @@ tune_kernel() { # Calculate exact requirement based on hardware, the tuned thread count, and 1GB page status REQUIRED_PAGES=$(RX_THREADS="$RX_SETUP_THREADS" RESERVE_EXTRA_MB="${HUGEPAGES_RESERVE_EXTRA_MB:-0}" THREADS_CAP="${THREADS_CAP:-}" "$SCRIPT_DIR/util/proposed-grub.sh" --runtime) log "Hardware-optimized HugePages: $REQUIRED_PAGES (2MB pages) calculated." - sudo sysctl -w vm.nr_hugepages="$REQUIRED_PAGES" + _ensure_hugepages "$REQUIRED_PAGES" else # Fallback when proposed-grub.sh is missing: 3072 × 2MB = 6 GB of huge pages — enough for the # ~2.3 GB RandomX dataset plus per-thread scratchpads on a large desktop/server, without over- # reserving on smaller hosts. proposed-grub.sh computes an exact, hardware-sized value instead. warn "Utility script not found. Fallback to safe default (3072)." - sudo sysctl -w vm.nr_hugepages=3072 + _ensure_hugepages 3072 fi log "Configuring bootloader (GRUB) for persistent HugePages..." @@ -1580,7 +1623,12 @@ _setup_plan() { fi fi fi - _p "tuning the kernel" "$_msr; reserve $_pages 2MB HugePages (runtime sysctl); $_grubline$_reboot" + # Grow-only preview (#328): same availability check _ensure_hugepages runs, read-only. + local _hpline="grow the pool so $_pages 2MB HugePages are available (grow-only runtime sysctl)" + if [[ "$_pages" =~ ^[0-9]+$ ]] && [ "$(_hugepages_avail)" -ge "$_pages" ] 2>/dev/null; then + _hpline="HugePages pool already covers the miner ($_pages pages needed) — no change" + fi + _p "tuning the kernel" "$_msr; $_hpline; $_grubline$_reboot" local _f1="hugetlbfs /dev/hugepages hugetlbfs defaults 0 0" _f2="hugetlbfs_1g $HUGEPAGES_1G_DIR hugetlbfs pagesize=1G 0 0" _add="" grep -qxF "$_f1" "$FSTAB" 2>/dev/null || _add=" '$_f1'" grep -qxF "$_f2" "$FSTAB" 2>/dev/null || _add="$_add '$_f2'" diff --git a/tests/run.sh b/tests/run.sh index b7cac82..6d028eb 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -2276,13 +2276,16 @@ case "${1:-}" in esac EOF chmod +x "$DR/util/proposed-grub.sh" +# MEMINFO is pinned (0 free) so the grow-only preview (#328) renders the same on any host — +# a runner with a real, large HugePages_Free would otherwise flip the plan line to "no change". +printf 'HugePages_Free: 0\n' >"$DR/etc/meminfo" dr_out="$(cd "$DR" && PATH="$STUBS:$PATH" CALL_LOG="$DR/calls.log" GRUB_DEFAULT="$DR/etc/grub" FSTAB="$DR/etc/fstab" \ - RIGFORGE_HOME="$PWD" bash "$SCRIPT" setup --dry-run &1)" + MEMINFO="$DR/etc/meminfo" RIGFORGE_HOME="$PWD" bash "$SCRIPT" setup --dry-run &1)" dr_rc=$? assert_rc "dry-run exits 0 (#146)" "$dr_rc" "0" assert_contains "plan: build line names the pinned version (#146)" "$dr_out" "build XMRig" assert_contains "plan: dependency probe ran (#146)" "$dr_out" "installing dependencies" -assert_contains "plan: computed HugePages count (#146)" "$dr_out" "reserve 2514 2MB HugePages" +assert_contains "plan: computed HugePages count, grow-only wording (#146/#328)" "$dr_out" "grow the pool so 2514 2MB HugePages are available" assert_contains "plan: GRUB before -> after diff (#146)" "$dr_out" "GRUB cmdline: 'quiet splash' -> 'quiet splash default_hugepagesz=2M hugepages=2514 msr.allow_writes=on'" assert_contains "plan: reboot called out when GRUB changes (#146)" "$dr_out" "a reboot WILL be required" assert_contains "plan: fstab append lines (#146)" "$dr_out" "hugetlbfs /dev/hugepages" @@ -2290,6 +2293,12 @@ assert_contains "plan: autotune timer with the configured target (#146)" "$dr_ou assert_contains "plan: sister API units (#146)" "$dr_out" "install rigforge-api.service" assert_contains "plan: add_to_path symlink (#146)" "$dr_out" "symlink" assert_contains "plan: footer says nothing changed (#146)" "$dr_out" "Dry run — nothing was changed" +# Grow-only preview, covered branch (#328): with plenty of free pages the plan says "no change" +# instead of the grow line — same availability check the mutation runs. +printf 'HugePages_Free: 99999\n' >"$DR/etc/meminfo_full" +dr_out_full="$(cd "$DR" && PATH="$STUBS:$PATH" CALL_LOG="$DR/calls2.log" GRUB_DEFAULT="$DR/etc/grub" FSTAB="$DR/etc/fstab" \ + MEMINFO="$DR/etc/meminfo_full" RIGFORGE_HOME="$PWD" bash "$SCRIPT" setup --dry-run &1)" +assert_contains "plan: covered pool renders the no-change line (#328)" "$dr_out_full" "HugePages pool already covers the miner (2514 pages needed) — no change" # No-mutation guard: none of the stubbed mutating commands were invoked (the stubs log every call). for mut in apt-get modprobe tee mount sysctl; do assert_absent "dry-run never invokes $mut (#146)" "$(cat "$DR/calls.log" 2>/dev/null)" "[$mut]" @@ -4925,6 +4934,93 @@ out="$(RIGFORGE_THREADS=abc run_tunekernel "$PGC")" assert_absent "garbage RIGFORGE_THREADS is sanitized away (#65)" "$out" "Sizing the HugePages reservation" assert_contains "sanitized RIGFORGE_THREADS -> empty RX_THREADS to proposed-grub (#65)" "$(cat "$PGC")" "RX_THREADS=[]" +# #328: the runtime HugePages write is grow-only — it must never shrink a pool another consumer +# (a co-hosted pithead stack) already reserved. The proposed-grub stub above says the miner needs +# 200 pages; MEMINFO/NR_HUGEPAGES_FILE fake the live pool and a recording sysctl captures writes. +echo "== black-box: runtime HugePages reservation is grow-only (#328) ==" +HP="$(mktemp -d "$SANDBOX/hp328.XXXXXX")" +mkdir -p "$HP/bin" +cat >"$HP/bin/sysctl" <<'EOF' +#!/usr/bin/env bash +echo "$*" >>"$SYSCTL_CALLS" +EOF +chmod +x "$HP/bin/sysctl" +run_tk328() { # [miner_held_pages] + ( + source "$SCRIPT" + OS_TYPE=Linux + SCRIPT_DIR="$TK" + WORKER_ROOT="$TK/home/worker" + MODULES_LOAD_DIR="$TK/nope" + MODULES_FILE="$TK/nope/modules" + GRUB_DEFAULT="$TK/nope/grub" # nonexistent -> the GRUB block is skipped + printf 'HugePages_Free: %s\n' "$2" >"$HP/meminfo" + printf '%s\n' "$3" >"$HP/nr_hugepages" + MEMINFO="$HP/meminfo" + NR_HUGEPAGES_FILE="$HP/nr_hugepages" + # The held-pages credit has its own seam: a running miner is out of scope for a sandbox. + [ -n "${4:-}" ] && eval "_miner_held_hugepages() { echo $4; }" + export PG_CALLS="$HP/pg_calls" SYSCTL_CALLS="$1" + set +e + PATH="$HP/bin:$STUBS:$PATH" tune_kernel 2>&1 + ) +} +SC="$HP/calls1" +: >"$SC" +out="$(run_tk328 "$SC" 0 3072)" +assert_contains "co-resident pool grows by the shortfall, never shrinks (#328)" "$(cat "$SC")" "vm.nr_hugepages=3272" +SC="$HP/calls2" +: >"$SC" +out="$(run_tk328 "$SC" 500 3072)" +assert_absent "enough free pages -> no write at all (#328)" "$(cat "$SC")" "vm.nr_hugepages" +assert_contains "enough free pages -> says it left the pool alone (#328)" "$out" "leaving it as-is" +SC="$HP/calls3" +: >"$SC" +out="$(run_tk328 "$SC" 0 0)" +assert_contains "fresh box -> plain requirement, unchanged behavior (#328)" "$(cat "$SC")" "vm.nr_hugepages=200" +SC="$HP/calls4" +: >"$SC" +out="$(run_tk328 "$SC" 50 1300 180)" +assert_absent "a running miner's held pages count as available — idempotent re-run (#328)" "$(cat "$SC")" "vm.nr_hugepages" +# The real held-pages probe (no seam): a stub systemctl reports this test shell as the miner's +# MainPID. On Linux /proc/$$/status exists with HugetlbPages: 0 kB — the probe reads it and +# credits 0; on macOS there is no /proc, the guard falls through to the same 0. Either way the +# outcome matches calls1: grow by the full shortfall. +cat >"$HP/bin/systemctl" <"$SC" +out="$(run_tk328 "$SC" 0 3072)" +assert_contains "real MainPID probe -> zero credit for a page-less process (#328)" "$(cat "$SC")" "vm.nr_hugepages=3272" +rm -f "$HP/bin/systemctl" +# proposed-grub.sh missing -> the 3072 fallback goes through the same grow-only path. +run_tk328_nopg() { # + ( + source "$SCRIPT" + OS_TYPE=Linux + SCRIPT_DIR="$HP/empty" # no util/proposed-grub.sh here + WORKER_ROOT="$TK/home/worker" + MODULES_LOAD_DIR="$TK/nope" + MODULES_FILE="$TK/nope/modules" + GRUB_DEFAULT="$TK/nope/grub" + printf 'HugePages_Free: 0\n' >"$HP/meminfo" + printf '0\n' >"$HP/nr_hugepages" + MEMINFO="$HP/meminfo" + NR_HUGEPAGES_FILE="$HP/nr_hugepages" + export SYSCTL_CALLS="$1" + set +e + PATH="$HP/bin:$STUBS:$PATH" tune_kernel 2>&1 + ) +} +mkdir -p "$HP/empty" +SC="$HP/calls6" +: >"$SC" +out="$(run_tk328_nopg "$SC")" +assert_contains "fallback (no proposed-grub.sh) is grow-only too (#328)" "$(cat "$SC")" "vm.nr_hugepages=3072" + # tune with no built worker fails clearly. TN2="$(mktemp -d "$SANDBOX/tune2.XXXXXX")" cp "$ROOT/VERSION" "$TN2/"