From 19388f42d82db5deb08cb79cb8be464b06a95d55 Mon Sep 17 00:00:00 2001 From: Travis Wu Date: Wed, 16 Sep 2026 16:00:06 +0800 Subject: [PATCH] fix(ceph): generate the cluster fsid once per install instead of compiling it in Every CubeCOS cluster reported the same Ceph fsid, because it was a compile-time constant that the ceph.fsid tunable merely defaulted to. Two installs were indistinguishable to anything that asks "is this the same cluster", which is what cubecos#1490 is about. The fsid is now minted once, by the bootstrap node, and recorded at /etc/cube/cos/ceph/fsid -- inside a directory the ceph module already declares to CONFIG_MIGRATE, so it survives the partition switch an upgrade performs. Every later commit reads the record rather than recomputing, which matters because the fsid reaches monmaptool: a value that drifted between commits would build a monmap the mons reject. Generated, not derived. Deriving it from cubesys.seed would have matched what rbd_secret_uuid already does, but the seed also generates the cluster's ssh key, and the fsid is published in every ceph.conf -- a v5 UUID would hand anyone an offline oracle for a low-entropy secret. The seed is separately unsafe to derive from while cubecos#1500 stands. An existing cluster never moves. On a node that predates the record, ceph_fsid_current reads the fsid already in ceph.conf, else extracts it from this node's mon store -- /var/lib/ceph is migrated, and the extract works with ceph down, which an upgrade commit needs. Whatever it finds is recorded and wins over anything that would be generated, so an upgraded cluster keeps the fsid its mons and OSDs already carry, the old constant included; LEGACY_FSID is kept only to name it. A joining node takes the bootstrap node's fsid over ssh, inside the retry loop that already waits for the master's mon bootstrap -- the same wait, for the same reason, since neither answer exists until the master has made it. Only the bootstrap node may mint, so a cluster gets exactly one. Nothing falls back to a constant on failure: an unresolved fsid refuses the commit, and sync_ceph_config keeps the existing ceph.conf rather than rewriting it with a worse one. Every value is shape-checked as a UUID before use, because HexUtilPOpen discards exit status and the shape is the only signal a C++ caller gets (cubecos#1486). Validate() now rejects a malformed operator-pinned ceph.fsid, which ValidateRegex did not. Refs bigstack-oss/cubecos#1490 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013cE5vKGkQQn3nAnn9TLKAU Signed-off-by: Travis Wu --- core/modules/config_ceph.cpp | 108 ++++++++++++++++++++++++++-- core/sdk_sh/modules/sdk_ceph.sh | 85 ++++++++++++++++++++++ core/sdk_sh/tests/test_ceph_fsid.sh | 93 ++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 core/sdk_sh/tests/test_ceph_fsid.sh diff --git a/core/modules/config_ceph.cpp b/core/modules/config_ceph.cpp index 2aad3c1a..71957fed 100644 --- a/core/modules/config_ceph.cpp +++ b/core/modules/config_ceph.cpp @@ -78,7 +78,10 @@ const static char ADMIN_KEYRING[] = "/etc/ceph/ceph.client.admin.keyring"; const static char K8S_KEYRING[] = "/etc/ceph/ceph.client.k8s.keyring"; const static char CEPHFS_CLIENT_AUTHKEY[] = "/etc/ceph/admin.key"; -static const char FSID[] = "c6e64c49-09cf-463b-9d1c-b6645b4b3b85"; +// The fsid every install shipped with before it was generated per cluster +// (cubecos#1490). Kept only so an upgraded cluster can be recognised as already +// carrying it -- nothing configures this value any more. +static const char LEGACY_FSID[] = "c6e64c49-09cf-463b-9d1c-b6645b4b3b85"; static const char CEPH_CACHE_POOL[] = "cachepool"; static const char K8S_VOLUME[] = "k8s-volumes"; @@ -129,7 +132,9 @@ CONFIG_TUNING_BOOL(CEPH_MIRROR_META_SYNC, "ceph.mirror.meta.sync", TUNING_PUB, " CONFIG_TUNING_BOOL(CEPH_ENABLED, "ceph.enabled", TUNING_UNPUB, "Set to true to enable ceph service.", true); CONFIG_TUNING_BOOL(CEPH_MON_ENABLED, "ceph.mon.enabled", TUNING_UNPUB, "Enable ceph monitor on this host.", false); CONFIG_TUNING_BOOL(CEPH_PERF_TUNED, "ceph.perf.tuned", TUNING_UNPUB, "Enable ceph performance tuning on this host.", true); -CONFIG_TUNING_STR(CEPH_FSID, "ceph.fsid", TUNING_UNPUB, "Set the UUID of the ceph cluster.", FSID, ValidateRegex, DFT_REGEX_STR); +// Empty by default: the fsid is generated once per cluster and recorded, not +// compiled in. Setting this pins an explicit one and overrides that. +CONFIG_TUNING_STR(CEPH_FSID, "ceph.fsid", TUNING_UNPUB, "Set the UUID of the ceph cluster.", "", ValidateRegex, DFT_REGEX_STR); CONFIG_TUNING_BOOL(CEPH_MIRROR_ENABLED, "ceph.mirror.enabled", TUNING_UNPUB, "Enable ceph rbd mirror.", false); CONFIG_TUNING_STR(CEPH_MIRROR_NAME, "ceph.mirror.name", TUNING_UNPUB, "Set local site name.", "", ValidateRegex, DFT_REGEX_STR); CONFIG_TUNING_STR(CEPH_MIRROR_PEER_NAME, "ceph.mirror.peer.%d.name", TUNING_UNPUB, "Set peer site name.", "", ValidateRegex, DFT_REGEX_STR); @@ -1533,9 +1538,78 @@ NotifyKeystone(bool modified) s_bKeystoneModified = IsModifiedTune(2); } +// An fsid is a UUID and nothing else; see ResolveFsid on why the shape is the +// only thing that can be trusted here. +static bool +IsFsid(const std::string& s) +{ + if (s.length() != 36) + return false; + for (size_t i = 0; i < s.length(); i++) { + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (s[i] != '-') + return false; + } + else if (!isxdigit((unsigned char)s[i]) || isupper((unsigned char)s[i])) { + return false; + } + } + return true; +} + +// The fsid to configure this node with, or empty when it is not knowable yet. +// +// Resolution order, and the reason for it: +// 1. ceph.fsid, when an operator has pinned one explicitly. +// 2. Whatever hex_sdk can already establish -- the recorded value, else the +// live cluster, its ceph.conf, or this node's mon store. An existing +// cluster's own fsid always wins, which is what keeps an upgrade on the +// fsid its mons and OSDs already carry, LEGACY_FSID included. +// 3. Only the bootstrap node mints a new one, so a cluster gets exactly one. +// Every other node takes the bootstrap node's over ssh -- the same way it +// already learns the bootstrap mon ip a few lines below. +// +// Returns empty rather than guessing. HexUtilPOpen discards exit status, so the +// shape of the answer is the only signal there is: anything that is not a UUID +// is a failed probe, not an fsid (cubecos#1486). +static std::string +ResolveFsid(bool isMaster, const std::string& peer) +{ + if (!s_fsid.newValue().empty()) + return s_fsid.newValue(); + + std::string fsid = HexUtilPOpen(HEX_SDK " ceph_fsid_resolve %d 2>/dev/null", + isMaster ? 1 : 0); + if (IsFsid(fsid)) + return fsid; + + if (isMaster) + return ""; + + // a joining node: the bootstrap node owns the answer, and may not have + // minted it yet -- the caller retries + fsid = HexUtilPOpen("ssh root@%s " HEX_SDK " ceph_fsid_resolve 1 2>/dev/null", + peer.c_str()); + if (!IsFsid(fsid)) + return ""; + + // record it locally so every later commit on this node is answered from disk + HexUtilSystemF(0, 0, HEX_SDK " ceph_fsid_record %s", fsid.c_str()); + return fsid; +} + static bool Validate() { + // An operator-pinned fsid must be a UUID: ValidateRegex on the tunable only + // says "a string", and a malformed one would reach monmaptool. + if (s_fsid.modified() && !s_fsid.newValue().empty() && + !IsFsid(s_fsid.newValue())) { + HexLogError("ceph.fsid must be a lowercase UUID: %s", s_fsid.newValue().c_str()); + printf("ceph.fsid must be a lowercase UUID: %s\n", s_fsid.newValue().c_str()); + return false; + } + if (!IsBootstrap()) { if (!s_mirrorEnabled) return true; @@ -1702,7 +1776,7 @@ Commit(bool modified, int dryLevel) bool monEnabled = IsMonEnabled(s_ha, s_monEnabled, s_eCubeRole, s_ctrlHosts); bool isMaster = G(IS_MASTER); - std::string fsid = s_fsid; + std::string fsid; std::string myIp = G(MGMT_ADDR); std::string ctrl = G(CTRL); std::string master = GetMaster(s_ha, ctrl, s_ctrlHosts); @@ -1725,8 +1799,13 @@ Commit(bool modified, int dryLevel) masterIp = HexUtilPOpen("ssh root@%s %s ceph_bootstrap_mon_ip 2>/dev/null", peer.c_str(), HEX_SDK); } + // Same wait, same reason: a joining node cannot know the fsid until + // the master has minted it, and the master mints it here. + if (fsid.length() == 0) + fsid = ResolveFsid(isMaster, peer); + HexLogInfo("got ceph monintor bootstrap ip %s from %s", masterIp.c_str(), peer.c_str()); - if (HexParseIP(masterIp.c_str(), AF_INET, &v4addr)) + if (HexParseIP(masterIp.c_str(), AF_INET, &v4addr) && fsid.length() > 0) break; else { // wait out the master's mon bootstrap (~10 min) instead of failing @@ -1740,6 +1819,14 @@ Commit(bool modified, int dryLevel) return false; } + // Refuse rather than fall back to a constant. A node that configured + // itself with the wrong fsid would build a monmap its peers reject, and + // the old behaviour -- everyone sharing one -- is the defect being fixed. + if (!IsFsid(fsid)) { + HexLogError("failed to resolve the ceph cluster fsid"); + return false; + } + if (access(CONTROL_REJOIN, F_OK) == 0) { peer = GetControllerPeers(s_hostname, s_ctrlHosts)[0]; master = HexUtilPOpen("ssh root@%s %s ceph_mon_map_hosts %s 2>/dev/null", peer.c_str(), HEX_SDK, CONF); @@ -2041,8 +2128,10 @@ SyncConfigMain(int argc, char* argv[]) return EXIT_FAILURE; } - // update ceph.conf for using current monmap hosts - std::string fsid = s_fsid; + // update ceph.conf for using current monmap hosts. Reads the record and + // nothing else -- this resyncs an already-bootstrapped node, so the answer + // exists; asking a peer for it here would ssh to self on a single node. + std::string fsid = HexUtilPOpen(HEX_SDK " ceph_fsid_resolve 0 2>/dev/null"); std::string ctrl = G(CTRL); std::string ctrlIp = G(CTRL_IP); std::string sharedId = G(SHARED_ID); @@ -2053,7 +2142,12 @@ SyncConfigMain(int argc, char* argv[]) std::string mdsIplist = HexUtilPOpen(HEX_SDK " ceph_mds_map_iplist %s", CONF); std::string adminCliPass = GetSaltKey(s_saltkey, s_adminCliPass.newValue(), s_seed.newValue()); - // no usable monmap (sdk returns "1" on failure): keep the existing conf + // no usable monmap (sdk returns "1" on failure), or no fsid: keep the + // existing conf rather than rewriting it with a worse one + if (!IsFsid(fsid)) { + HexLogWarning("sync_ceph_config: no fsid available, keeping %s", CONF); + return EXIT_SUCCESS; + } if (monHosts.length() == 0 || monIplist.find('.') == std::string::npos) { HexLogWarning("sync_ceph_config: no monmap available, keeping %s", CONF); return EXIT_SUCCESS; diff --git a/core/sdk_sh/modules/sdk_ceph.sh b/core/sdk_sh/modules/sdk_ceph.sh index ca70796e..7c7e9b8d 100644 --- a/core/sdk_sh/modules/sdk_ceph.sh +++ b/core/sdk_sh/modules/sdk_ceph.sh @@ -299,6 +299,91 @@ ceph_mon_map_iplist() echo -n $iplist } +# Where the cluster's fsid is recorded. /etc/cube/cos/ceph is CONFIG_MIGRATE'd by +# the ceph module, so this survives the partition switch an upgrade performs. +CEPH_FSID_FILE=${CEPH_FSID_FILE:-/etc/cube/cos/ceph/fsid} + +# An fsid is a UUID and nothing else. Shape-check every value before it is +# recorded or printed: HexUtilPOpen discards exit status, so a caller in C++ +# cannot tell a failure from an answer except by looking at what came back +# (cubecos#1486 is the same lesson). +_ceph_fsid_is_uuid() +{ + [[ "$1" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] +} + +# This cluster's fsid as it already exists, or nothing. Only reached on a node +# that predates the record, which means an upgrade. +# +# Not asked of a running cluster: `ceph fsid` needs $CONF to find the mons, so +# it can never answer where reading $CONF could not. +ceph_fsid_current() +{ + local fsid="" + + if [ -f "$CONF" ] ; then + fsid=$(awk -F= '/^[[:space:]]*fsid[[:space:]]*=/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' "$CONF") + _ceph_fsid_is_uuid "$fsid" && { echo -n "$fsid" ; return 0 ; } + fi + + # the local mon store, which /var/lib/ceph being CONFIG_MIGRATE'd keeps + # across an upgrade; ceph_mon_map_create extracts it with ceph down + ceph_mon_map_create >/dev/null 2>&1 + fsid=$(monmaptool --print $MAPFILE 2>/dev/null | awk '/^fsid/ {print $2; exit}') + _ceph_fsid_is_uuid "$fsid" && { echo -n "$fsid" ; return 0 ; } + + return 1 +} + +# Record $1 as this node's fsid, once. Never overwrites: the recorded value is +# what every later commit reads, so changing it would repoint a live cluster's +# monmap at an fsid its mons do not have. +ceph_fsid_record() +{ + local fsid=$1 + _ceph_fsid_is_uuid "$fsid" || return 1 + [ -s "$CEPH_FSID_FILE" ] && return 0 + mkdir -p "$(dirname "$CEPH_FSID_FILE")" + echo -n "$fsid" > "$CEPH_FSID_FILE" + chmod 0644 "$CEPH_FSID_FILE" +} + +# The fsid to configure this node with. Prints it, or nothing and returns +# non-zero when the answer is not knowable yet -- a joining node before the +# bootstrap node has minted one. Never guesses. +# +# $1: 1 to mint a new fsid when there is no cluster and nothing recorded. +# Only the bootstrap node passes 1, so a cluster mints exactly one. +# +# Order matters. An existing cluster's own fsid outranks anything we would +# generate, which is what keeps an upgraded cluster on the fsid its mons and +# OSDs already carry -- including the old hardcoded constant. +ceph_fsid_resolve() +{ + local may_generate=${1:-0} + local fsid="" + + if [ -s "$CEPH_FSID_FILE" ] ; then + fsid=$(tr -d '[:space:]' < "$CEPH_FSID_FILE") + _ceph_fsid_is_uuid "$fsid" || return 1 + echo -n "$fsid" + return 0 + fi + + if fsid=$(ceph_fsid_current) ; then + ceph_fsid_record "$fsid" || return 1 + echo -n "$fsid" + return 0 + fi + + [ "$may_generate" == "1" ] || return 1 + + fsid=$(uuidgen | tr -d '[:space:]') + _ceph_fsid_is_uuid "$fsid" || return 1 + ceph_fsid_record "$fsid" || return 1 + echo -n "$fsid" +} + ceph_mon_map_hosts() { ceph_mon_map_create $1 diff --git a/core/sdk_sh/tests/test_ceph_fsid.sh b/core/sdk_sh/tests/test_ceph_fsid.sh new file mode 100644 index 00000000..71d8b68c --- /dev/null +++ b/core/sdk_sh/tests/test_ceph_fsid.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# ceph_fsid_resolve: which fsid a node ends up with, and -- the part that +# matters -- that an existing cluster's own fsid always outranks a new one. +# +# Run: bash test_ceph_fsid.sh +# +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +SRC=$(dirname "${BASH_SOURCE[0]}")/../modules/sdk_ceph.sh +for fn in _ceph_fsid_is_uuid ceph_fsid_current ceph_fsid_record ceph_fsid_resolve ; do + sed -n "/^$fn()/,/^}/p" $SRC >> $T/fn.sh +done +CEPH_FSID_FILE=$T/fsid +CONF=$T/ceph.conf +MAPFILE=$T/monmap + +source $T/fn.sh + +pass=0 fail=0 +chk(){ # description actual expected + if [ "$2" = "$3" ] ; then + pass=$((pass+1)); printf 'PASS %-46s -> %s\n' "$1" "$2" + else + fail=$((fail+1)); printf 'FAIL %-46s -> got "%s", want "%s"\n' "$1" "$2" "$3" + fi +} + +# stubs for the two sources, each switched off by default +export T +ceph_mon_map_create(){ :; } +monmaptool(){ [ -s $T/monmap_fsid ] || return 1; echo "fsid $(cat $T/monmap_fsid)"; } + +OLD=c6e64c49-09cf-463b-9d1c-b6645b4b3b85 +reset_all(){ rm -f $T/fsid $T/ceph.conf $T/monmap_fsid; } + +# --- a fresh bootstrap node mints one, and only when allowed to --- +reset_all +out=$(ceph_fsid_resolve 0); rc=$? +chk "joining node with no cluster yet fails" "$rc" "1" +chk " ...and records nothing" "$(cat $T/fsid 2>/dev/null)" "" + +reset_all +out=$(ceph_fsid_resolve 1) +chk "bootstrap node mints a uuid" "$(_ceph_fsid_is_uuid "$out" && echo yes)" "yes" +chk " ...and records it" "$(cat $T/fsid)" "$out" + +# minting is once: the recorded value wins on every later call +again=$(ceph_fsid_resolve 1) +chk "second call returns the same fsid" "$again" "$out" + +# --- an existing cluster's fsid outranks a new one: the upgrade case --- +reset_all +printf '[global]\n fsid = %s\n' "$OLD" > $T/ceph.conf +out=$(ceph_fsid_resolve 1) +chk "ceph.conf's fsid is adopted" "$out" "$OLD" +chk " ...and recorded, not regenerated" "$(cat $T/fsid)" "$OLD" + +reset_all +echo "$OLD" > $T/monmap_fsid +out=$(ceph_fsid_resolve 1) +chk "mon store answers with ceph down" "$out" "$OLD" + +# --- a recorded fsid is never overwritten, whatever the cluster says --- +reset_all +echo "$OLD" > $T/fsid +echo "11111111-2222-3333-4444-555555555555" > $T/monmap_fsid +out=$(ceph_fsid_resolve 1) +chk "recorded fsid outranks the cluster's" "$out" "$OLD" +ceph_fsid_record "11111111-2222-3333-4444-555555555555" +chk " ...and record refuses to overwrite" "$(cat $T/fsid)" "$OLD" + +# --- garbage in is never garbage out --- +reset_all +echo "not-a-uuid" > $T/fsid +out=$(ceph_fsid_resolve 1); rc=$? +chk "a corrupt record fails loudly" "$rc" "1" +chk " ...and prints nothing" "$out" "" + +reset_all +echo "1" > $T/monmap_fsid # the sdk's own failure sentinel +out=$(ceph_fsid_resolve 0); rc=$? +chk "sentinel from a failed probe is refused" "$rc" "1" + +reset_all +ceph_fsid_record "" ; rc=$? +chk "record refuses an empty fsid" "$rc" "1" +chk " ...writing no file" "$(ls $T/fsid 2>/dev/null)" "" + +echo "----" +echo "PASS=$pass FAIL=$fail" +[ $fail -eq 0 ] || exit 1 +echo "OK: ceph_fsid_resolve"