diff --git a/core/modules/config_ceph.cpp b/core/modules/config_ceph.cpp index 2aad3c1a..71957fed 100644 --- a/core/modules/config_ceph.cpp +++ b/core/modules/config_ceph.cpp @@ -78,7 +78,10 @@ const static char ADMIN_KEYRING[] = "/etc/ceph/ceph.client.admin.keyring"; const static char K8S_KEYRING[] = "/etc/ceph/ceph.client.k8s.keyring"; const static char CEPHFS_CLIENT_AUTHKEY[] = "/etc/ceph/admin.key"; -static const char FSID[] = "c6e64c49-09cf-463b-9d1c-b6645b4b3b85"; +// The fsid every install shipped with before it was generated per cluster +// (cubecos#1490). Kept only so an upgraded cluster can be recognised as already +// carrying it -- nothing configures this value any more. +static const char LEGACY_FSID[] = "c6e64c49-09cf-463b-9d1c-b6645b4b3b85"; static const char CEPH_CACHE_POOL[] = "cachepool"; static const char K8S_VOLUME[] = "k8s-volumes"; @@ -129,7 +132,9 @@ CONFIG_TUNING_BOOL(CEPH_MIRROR_META_SYNC, "ceph.mirror.meta.sync", TUNING_PUB, " CONFIG_TUNING_BOOL(CEPH_ENABLED, "ceph.enabled", TUNING_UNPUB, "Set to true to enable ceph service.", true); CONFIG_TUNING_BOOL(CEPH_MON_ENABLED, "ceph.mon.enabled", TUNING_UNPUB, "Enable ceph monitor on this host.", false); CONFIG_TUNING_BOOL(CEPH_PERF_TUNED, "ceph.perf.tuned", TUNING_UNPUB, "Enable ceph performance tuning on this host.", true); -CONFIG_TUNING_STR(CEPH_FSID, "ceph.fsid", TUNING_UNPUB, "Set the UUID of the ceph cluster.", FSID, ValidateRegex, DFT_REGEX_STR); +// Empty by default: the fsid is generated once per cluster and recorded, not +// compiled in. Setting this pins an explicit one and overrides that. +CONFIG_TUNING_STR(CEPH_FSID, "ceph.fsid", TUNING_UNPUB, "Set the UUID of the ceph cluster.", "", ValidateRegex, DFT_REGEX_STR); CONFIG_TUNING_BOOL(CEPH_MIRROR_ENABLED, "ceph.mirror.enabled", TUNING_UNPUB, "Enable ceph rbd mirror.", false); CONFIG_TUNING_STR(CEPH_MIRROR_NAME, "ceph.mirror.name", TUNING_UNPUB, "Set local site name.", "", ValidateRegex, DFT_REGEX_STR); CONFIG_TUNING_STR(CEPH_MIRROR_PEER_NAME, "ceph.mirror.peer.%d.name", TUNING_UNPUB, "Set peer site name.", "", ValidateRegex, DFT_REGEX_STR); @@ -1533,9 +1538,78 @@ NotifyKeystone(bool modified) s_bKeystoneModified = IsModifiedTune(2); } +// An fsid is a UUID and nothing else; see ResolveFsid on why the shape is the +// only thing that can be trusted here. +static bool +IsFsid(const std::string& s) +{ + if (s.length() != 36) + return false; + for (size_t i = 0; i < s.length(); i++) { + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (s[i] != '-') + return false; + } + else if (!isxdigit((unsigned char)s[i]) || isupper((unsigned char)s[i])) { + return false; + } + } + return true; +} + +// The fsid to configure this node with, or empty when it is not knowable yet. +// +// Resolution order, and the reason for it: +// 1. ceph.fsid, when an operator has pinned one explicitly. +// 2. Whatever hex_sdk can already establish -- the recorded value, else the +// live cluster, its ceph.conf, or this node's mon store. An existing +// cluster's own fsid always wins, which is what keeps an upgrade on the +// fsid its mons and OSDs already carry, LEGACY_FSID included. +// 3. Only the bootstrap node mints a new one, so a cluster gets exactly one. +// Every other node takes the bootstrap node's over ssh -- the same way it +// already learns the bootstrap mon ip a few lines below. +// +// Returns empty rather than guessing. HexUtilPOpen discards exit status, so the +// shape of the answer is the only signal there is: anything that is not a UUID +// is a failed probe, not an fsid (cubecos#1486). +static std::string +ResolveFsid(bool isMaster, const std::string& peer) +{ + if (!s_fsid.newValue().empty()) + return s_fsid.newValue(); + + std::string fsid = HexUtilPOpen(HEX_SDK " ceph_fsid_resolve %d 2>/dev/null", + isMaster ? 1 : 0); + if (IsFsid(fsid)) + return fsid; + + if (isMaster) + return ""; + + // a joining node: the bootstrap node owns the answer, and may not have + // minted it yet -- the caller retries + fsid = HexUtilPOpen("ssh root@%s " HEX_SDK " ceph_fsid_resolve 1 2>/dev/null", + peer.c_str()); + if (!IsFsid(fsid)) + return ""; + + // record it locally so every later commit on this node is answered from disk + HexUtilSystemF(0, 0, HEX_SDK " ceph_fsid_record %s", fsid.c_str()); + return fsid; +} + static bool Validate() { + // An operator-pinned fsid must be a UUID: ValidateRegex on the tunable only + // says "a string", and a malformed one would reach monmaptool. + if (s_fsid.modified() && !s_fsid.newValue().empty() && + !IsFsid(s_fsid.newValue())) { + HexLogError("ceph.fsid must be a lowercase UUID: %s", s_fsid.newValue().c_str()); + printf("ceph.fsid must be a lowercase UUID: %s\n", s_fsid.newValue().c_str()); + return false; + } + if (!IsBootstrap()) { if (!s_mirrorEnabled) return true; @@ -1702,7 +1776,7 @@ Commit(bool modified, int dryLevel) bool monEnabled = IsMonEnabled(s_ha, s_monEnabled, s_eCubeRole, s_ctrlHosts); bool isMaster = G(IS_MASTER); - std::string fsid = s_fsid; + std::string fsid; std::string myIp = G(MGMT_ADDR); std::string ctrl = G(CTRL); std::string master = GetMaster(s_ha, ctrl, s_ctrlHosts); @@ -1725,8 +1799,13 @@ Commit(bool modified, int dryLevel) masterIp = HexUtilPOpen("ssh root@%s %s ceph_bootstrap_mon_ip 2>/dev/null", peer.c_str(), HEX_SDK); } + // Same wait, same reason: a joining node cannot know the fsid until + // the master has minted it, and the master mints it here. + if (fsid.length() == 0) + fsid = ResolveFsid(isMaster, peer); + HexLogInfo("got ceph monintor bootstrap ip %s from %s", masterIp.c_str(), peer.c_str()); - if (HexParseIP(masterIp.c_str(), AF_INET, &v4addr)) + if (HexParseIP(masterIp.c_str(), AF_INET, &v4addr) && fsid.length() > 0) break; else { // wait out the master's mon bootstrap (~10 min) instead of failing @@ -1740,6 +1819,14 @@ Commit(bool modified, int dryLevel) return false; } + // Refuse rather than fall back to a constant. A node that configured + // itself with the wrong fsid would build a monmap its peers reject, and + // the old behaviour -- everyone sharing one -- is the defect being fixed. + if (!IsFsid(fsid)) { + HexLogError("failed to resolve the ceph cluster fsid"); + return false; + } + if (access(CONTROL_REJOIN, F_OK) == 0) { peer = GetControllerPeers(s_hostname, s_ctrlHosts)[0]; master = HexUtilPOpen("ssh root@%s %s ceph_mon_map_hosts %s 2>/dev/null", peer.c_str(), HEX_SDK, CONF); @@ -2041,8 +2128,10 @@ SyncConfigMain(int argc, char* argv[]) return EXIT_FAILURE; } - // update ceph.conf for using current monmap hosts - std::string fsid = s_fsid; + // update ceph.conf for using current monmap hosts. Reads the record and + // nothing else -- this resyncs an already-bootstrapped node, so the answer + // exists; asking a peer for it here would ssh to self on a single node. + std::string fsid = HexUtilPOpen(HEX_SDK " ceph_fsid_resolve 0 2>/dev/null"); std::string ctrl = G(CTRL); std::string ctrlIp = G(CTRL_IP); std::string sharedId = G(SHARED_ID); @@ -2053,7 +2142,12 @@ SyncConfigMain(int argc, char* argv[]) std::string mdsIplist = HexUtilPOpen(HEX_SDK " ceph_mds_map_iplist %s", CONF); std::string adminCliPass = GetSaltKey(s_saltkey, s_adminCliPass.newValue(), s_seed.newValue()); - // no usable monmap (sdk returns "1" on failure): keep the existing conf + // no usable monmap (sdk returns "1" on failure), or no fsid: keep the + // existing conf rather than rewriting it with a worse one + if (!IsFsid(fsid)) { + HexLogWarning("sync_ceph_config: no fsid available, keeping %s", CONF); + return EXIT_SUCCESS; + } if (monHosts.length() == 0 || monIplist.find('.') == std::string::npos) { HexLogWarning("sync_ceph_config: no monmap available, keeping %s", CONF); return EXIT_SUCCESS; diff --git a/core/sdk_sh/modules/sdk_ceph.sh b/core/sdk_sh/modules/sdk_ceph.sh index ca70796e..7c7e9b8d 100644 --- a/core/sdk_sh/modules/sdk_ceph.sh +++ b/core/sdk_sh/modules/sdk_ceph.sh @@ -299,6 +299,91 @@ ceph_mon_map_iplist() echo -n $iplist } +# Where the cluster's fsid is recorded. /etc/cube/cos/ceph is CONFIG_MIGRATE'd by +# the ceph module, so this survives the partition switch an upgrade performs. +CEPH_FSID_FILE=${CEPH_FSID_FILE:-/etc/cube/cos/ceph/fsid} + +# An fsid is a UUID and nothing else. Shape-check every value before it is +# recorded or printed: HexUtilPOpen discards exit status, so a caller in C++ +# cannot tell a failure from an answer except by looking at what came back +# (cubecos#1486 is the same lesson). +_ceph_fsid_is_uuid() +{ + [[ "$1" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] +} + +# This cluster's fsid as it already exists, or nothing. Only reached on a node +# that predates the record, which means an upgrade. +# +# Not asked of a running cluster: `ceph fsid` needs $CONF to find the mons, so +# it can never answer where reading $CONF could not. +ceph_fsid_current() +{ + local fsid="" + + if [ -f "$CONF" ] ; then + fsid=$(awk -F= '/^[[:space:]]*fsid[[:space:]]*=/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' "$CONF") + _ceph_fsid_is_uuid "$fsid" && { echo -n "$fsid" ; return 0 ; } + fi + + # the local mon store, which /var/lib/ceph being CONFIG_MIGRATE'd keeps + # across an upgrade; ceph_mon_map_create extracts it with ceph down + ceph_mon_map_create >/dev/null 2>&1 + fsid=$(monmaptool --print $MAPFILE 2>/dev/null | awk '/^fsid/ {print $2; exit}') + _ceph_fsid_is_uuid "$fsid" && { echo -n "$fsid" ; return 0 ; } + + return 1 +} + +# Record $1 as this node's fsid, once. Never overwrites: the recorded value is +# what every later commit reads, so changing it would repoint a live cluster's +# monmap at an fsid its mons do not have. +ceph_fsid_record() +{ + local fsid=$1 + _ceph_fsid_is_uuid "$fsid" || return 1 + [ -s "$CEPH_FSID_FILE" ] && return 0 + mkdir -p "$(dirname "$CEPH_FSID_FILE")" + echo -n "$fsid" > "$CEPH_FSID_FILE" + chmod 0644 "$CEPH_FSID_FILE" +} + +# The fsid to configure this node with. Prints it, or nothing and returns +# non-zero when the answer is not knowable yet -- a joining node before the +# bootstrap node has minted one. Never guesses. +# +# $1: 1 to mint a new fsid when there is no cluster and nothing recorded. +# Only the bootstrap node passes 1, so a cluster mints exactly one. +# +# Order matters. An existing cluster's own fsid outranks anything we would +# generate, which is what keeps an upgraded cluster on the fsid its mons and +# OSDs already carry -- including the old hardcoded constant. +ceph_fsid_resolve() +{ + local may_generate=${1:-0} + local fsid="" + + if [ -s "$CEPH_FSID_FILE" ] ; then + fsid=$(tr -d '[:space:]' < "$CEPH_FSID_FILE") + _ceph_fsid_is_uuid "$fsid" || return 1 + echo -n "$fsid" + return 0 + fi + + if fsid=$(ceph_fsid_current) ; then + ceph_fsid_record "$fsid" || return 1 + echo -n "$fsid" + return 0 + fi + + [ "$may_generate" == "1" ] || return 1 + + fsid=$(uuidgen | tr -d '[:space:]') + _ceph_fsid_is_uuid "$fsid" || return 1 + ceph_fsid_record "$fsid" || return 1 + echo -n "$fsid" +} + ceph_mon_map_hosts() { ceph_mon_map_create $1 diff --git a/core/sdk_sh/tests/test_ceph_fsid.sh b/core/sdk_sh/tests/test_ceph_fsid.sh new file mode 100644 index 00000000..71d8b68c --- /dev/null +++ b/core/sdk_sh/tests/test_ceph_fsid.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# ceph_fsid_resolve: which fsid a node ends up with, and -- the part that +# matters -- that an existing cluster's own fsid always outranks a new one. +# +# Run: bash test_ceph_fsid.sh +# +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +SRC=$(dirname "${BASH_SOURCE[0]}")/../modules/sdk_ceph.sh +for fn in _ceph_fsid_is_uuid ceph_fsid_current ceph_fsid_record ceph_fsid_resolve ; do + sed -n "/^$fn()/,/^}/p" $SRC >> $T/fn.sh +done +CEPH_FSID_FILE=$T/fsid +CONF=$T/ceph.conf +MAPFILE=$T/monmap + +source $T/fn.sh + +pass=0 fail=0 +chk(){ # description actual expected + if [ "$2" = "$3" ] ; then + pass=$((pass+1)); printf 'PASS %-46s -> %s\n' "$1" "$2" + else + fail=$((fail+1)); printf 'FAIL %-46s -> got "%s", want "%s"\n' "$1" "$2" "$3" + fi +} + +# stubs for the two sources, each switched off by default +export T +ceph_mon_map_create(){ :; } +monmaptool(){ [ -s $T/monmap_fsid ] || return 1; echo "fsid $(cat $T/monmap_fsid)"; } + +OLD=c6e64c49-09cf-463b-9d1c-b6645b4b3b85 +reset_all(){ rm -f $T/fsid $T/ceph.conf $T/monmap_fsid; } + +# --- a fresh bootstrap node mints one, and only when allowed to --- +reset_all +out=$(ceph_fsid_resolve 0); rc=$? +chk "joining node with no cluster yet fails" "$rc" "1" +chk " ...and records nothing" "$(cat $T/fsid 2>/dev/null)" "" + +reset_all +out=$(ceph_fsid_resolve 1) +chk "bootstrap node mints a uuid" "$(_ceph_fsid_is_uuid "$out" && echo yes)" "yes" +chk " ...and records it" "$(cat $T/fsid)" "$out" + +# minting is once: the recorded value wins on every later call +again=$(ceph_fsid_resolve 1) +chk "second call returns the same fsid" "$again" "$out" + +# --- an existing cluster's fsid outranks a new one: the upgrade case --- +reset_all +printf '[global]\n fsid = %s\n' "$OLD" > $T/ceph.conf +out=$(ceph_fsid_resolve 1) +chk "ceph.conf's fsid is adopted" "$out" "$OLD" +chk " ...and recorded, not regenerated" "$(cat $T/fsid)" "$OLD" + +reset_all +echo "$OLD" > $T/monmap_fsid +out=$(ceph_fsid_resolve 1) +chk "mon store answers with ceph down" "$out" "$OLD" + +# --- a recorded fsid is never overwritten, whatever the cluster says --- +reset_all +echo "$OLD" > $T/fsid +echo "11111111-2222-3333-4444-555555555555" > $T/monmap_fsid +out=$(ceph_fsid_resolve 1) +chk "recorded fsid outranks the cluster's" "$out" "$OLD" +ceph_fsid_record "11111111-2222-3333-4444-555555555555" +chk " ...and record refuses to overwrite" "$(cat $T/fsid)" "$OLD" + +# --- garbage in is never garbage out --- +reset_all +echo "not-a-uuid" > $T/fsid +out=$(ceph_fsid_resolve 1); rc=$? +chk "a corrupt record fails loudly" "$rc" "1" +chk " ...and prints nothing" "$out" "" + +reset_all +echo "1" > $T/monmap_fsid # the sdk's own failure sentinel +out=$(ceph_fsid_resolve 0); rc=$? +chk "sentinel from a failed probe is refused" "$rc" "1" + +reset_all +ceph_fsid_record "" ; rc=$? +chk "record refuses an empty fsid" "$rc" "1" +chk " ...writing no file" "$(ls $T/fsid 2>/dev/null)" "" + +echo "----" +echo "PASS=$pass FAIL=$fail" +[ $fail -eq 0 ] || exit 1 +echo "OK: ceph_fsid_resolve"