From 917fe9dc0d6a7dc24d9a6948deffb80507653939 Mon Sep 17 00:00:00 2001 From: Travis Wu Date: Wed, 16 Sep 2026 19:50:28 +0800 Subject: [PATCH 1/4] fix(enterprise): carry the provider through, and point cube-cos at the dashboard Two failures seen on the 1cc r630, both of which made a working surface look broken with nothing in a log to explain it. Chat reset itself on every re-run. This script renders the whole release, so a value it does not pass back is one helm removes -- and it passed nothing for the provider, so each deploy reset the endpoint to the chart's placeholder and dropped the key. The surface then reported "chat enabled" and failed on the first question. It now reads the key back from the Secret and the URL from the deployment's arguments, exactly as it already does for the TLS and enrollment material, and takes both as arguments so a first install can set them. The read-back is by argument position rather than a jsonpath range piped through grep: the latter silently produced nothing, which carried the key correctly while letting the URL fall back to the placeholder -- a failure that looks identical to not having tried. cube-cos pointed at http://127.0.0.1:8080, which is httpd and answers 403 to everything; nginx serves the dashboard on the control address. A target pointed at loopback looks configured and refuses every request, which presented as a proxied tab showing "403 Forbidden". It now uses the control address this script already resolves for the kubeconfig. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PZ5umjjCedZwWtbAbiMjfj Signed-off-by: Travis Wu --- internal/enterprise/assets/install-advisor.sh | 51 +++++++++++++++++-- internal/enterprise/assets_test.go | 34 +++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/internal/enterprise/assets/install-advisor.sh b/internal/enterprise/assets/install-advisor.sh index 1d89fec..4767e09 100644 --- a/internal/enterprise/assets/install-advisor.sh +++ b/internal/enterprise/assets/install-advisor.sh @@ -3,7 +3,8 @@ # advisor_register only pushes the chart + prereqs to Harbor; this installs # the chart and confirms the advisor serves. Idempotent: safe to re-run. # -# args: [base_url] [console_pool] [console_account] +# args: [base_url] [console_pool] +# [console_account] [provider_key_file] [provider_url] set -uo pipefail FRAMEWORK="${1:?framework name required}" ADVISOR_LB_IP="${2:?advisor lb ip required}" @@ -27,6 +28,10 @@ CONSOLE_POOL="${5:-}" # provisions "advisor" for exactly this, so that is the default; empty leaves # the console disabled, which is what every install did before. CONSOLE_ACCOUNT="${6:-advisor}" +# The inference endpoint the chat surface calls, and a file holding its key. +# Both optional: given, they set it; omitted, whatever the deployment already +# uses is carried through. +PROVIDER_URL="${8:-}" NS=cube-advisor fail() { echo "ERROR: $*" >&2; exit 1; } @@ -250,15 +255,54 @@ if [ "${#POOL_ADDRS[@]}" -ge 2 ]; then else echo "warning: no ingress-lb on framework $FRAMEWORK; the CMP console origin is not configured" >&2 fi - # The node's own dashboard, which every enrolled node serves locally. + # The node's own dashboard. Addressed at the control VIP rather than + # loopback: nginx serves the UI on the management address, and 127.0.0.1:8080 + # is httpd, which answers 403 to everything -- a target pointed there looks + # configured and refuses every request. WC_ARGS+=(--set "webConsole.origins[$n].address=${POOL_ADDRS[$n]}" \ - --set "webConsole.origins[$n].upstream=http://127.0.0.1:8080" \ + --set "webConsole.origins[$n].upstream=https://$CTRL" \ --set "webConsole.origins[$n].targets[0]=cube-cos") echo "web console enabled on ${#POOL_ADDRS[@]} origin address(es)." elif [ -n "$CONSOLE_POOL" ]; then echo "warning: the console pool has fewer than 2 addresses; leaving the web console off" >&2 fi +# --- the inference provider --- +# Carried through a re-run, like the TLS and enrollment material above, and for +# the same reason: this script renders the whole release, so a value it does not +# pass back is a value helm removes. Leaving it out reset the provider to the +# chart's placeholder and dropped the key, which turns a working chat surface +# into one that reports itself enabled and fails on the first question. +PROVIDER_ARGS=() +PROVIDER_KEY_FILE="${7:-}" +if [ -n "$PROVIDER_KEY_FILE" ]; then + [ -r "$PROVIDER_KEY_FILE" ] || fail "cannot read the provider key file: $PROVIDER_KEY_FILE" + PROVIDER_ARGS+=(--set-file provider.key="$PROVIDER_KEY_FILE") + [ -n "$PROVIDER_URL" ] && PROVIDER_ARGS+=(--set provider.url="$PROVIDER_URL") + echo "provider set to ${PROVIDER_URL:-the chart default}." +else + # Read back what the deployment is already using. The key lives in the + # Secret; the URL is an argument on the container. + PREV_KEY="$($K -n "$NS" get secret cube-advisor-secrets -o jsonpath='{.data.providerKey}' 2>/dev/null | base64 -d)" + # Read the flag's value by position in the argument list. A jsonpath range + # piped through grep looked simpler and silently produced nothing, which is + # the failure that let the URL fall back to the chart's placeholder while the + # key was carried correctly. + PREV_URL="$($K -n "$NS" get deploy cube-advisor -o json 2>/dev/null | jq -r ' + .spec.template.spec.containers[0].args as $a + | ($a | index("-provider-url")) as $i + | if $i == null then empty else $a[$i + 1] end')" + if [ -n "$PREV_KEY" ]; then + PROVIDER_KEEP="$(mktemp)"; TMPDIRS+=("$PROVIDER_KEEP") + printf '%s' "$PREV_KEY" > "$PROVIDER_KEEP" + PROVIDER_ARGS+=(--set-file provider.key="$PROVIDER_KEEP") + echo "carrying the existing provider key through the upgrade." + fi + if [ -n "$PREV_URL" ]; then + PROVIDER_ARGS+=(--set provider.url="$PREV_URL") + fi +fi + # --- the console's SSH user CA --- # advisor-api generates one at startup when the chart passes no key, which # suits a dev run and breaks a deployment: a node pins this CA in sshd and @@ -293,6 +337,7 @@ helm upgrade --install cube-advisor "oci://$RURL/$RPROJ/cube-advisor" --version "${ENROLL_ARGS[@]}" \ "${WC_ARGS[@]}" \ "${CONSOLE_ARGS[@]}" \ + "${PROVIDER_ARGS[@]}" \ --kube-insecure-skip-tls-verify --insecure-skip-tls-verify --timeout 20m --wait=false \ || fail "helm upgrade failed — the release is unchanged; do not uninstall to retry, that deletes the database" diff --git a/internal/enterprise/assets_test.go b/internal/enterprise/assets_test.go index b27c1e7..51bc29a 100644 --- a/internal/enterprise/assets_test.go +++ b/internal/enterprise/assets_test.go @@ -228,3 +228,37 @@ func TestAdvisorInstallPrintsTheConsoleCAForTheNodes(t *testing.T) { "the console CA, so the console cannot work end to end") } } + +// The script renders the whole release, so a value it does not pass back is a +// value helm removes. The provider was not carried, so every re-run reset the +// chat surface to the chart's placeholder endpoint and dropped the key — +// leaving a deployment that reports "chat enabled" and fails on the first +// question. Seen twice on the 1cc r630, the second time caused by deploying an +// unrelated fix. +func TestAdvisorInstallCarriesTheProviderThroughAnUpgrade(t *testing.T) { + for _, want := range []string{ + "get secret cube-advisor-secrets -o jsonpath='{.data.providerKey}'", + "--set-file provider.key=", + "carrying the existing provider key through the upgrade", + } { + if !contains(installAdvisorScript, want) { + t.Errorf("the advisor installer does not %q; a re-run would reset the "+ + "provider and the chat surface would fail on its first question", want) + } + } +} + +// 127.0.0.1:8080 is httpd, which answers 403 to everything. The dashboard is +// nginx on the management address, so a cube-cos target pointed at loopback +// looks configured and refuses every request — which is exactly how it +// presented: a proxied tab showing "403 Forbidden" with nothing in any log. +func TestAdvisorConsolePointsCubeCosAtTheDashboard(t *testing.T) { + if contains(installAdvisorScript, "upstream=http://127.0.0.1:8080") { + t.Error("cube-cos still points at 127.0.0.1:8080, which is httpd and 403s; " + + "the dashboard is served on the control address") + } + if !contains(installAdvisorScript, `webConsole.origins[$n].upstream=https://$CTRL`) { + t.Error("cube-cos does not point at the control address the rest of this " + + "script already resolves") + } +} From 67149e10966b583c1bbee17c5330e01759f0b764 Mon Sep 17 00:00:00 2001 From: Travis Wu Date: Wed, 16 Sep 2026 21:07:01 +0800 Subject: [PATCH 2/4] fix(enterprise): give the dashboard origin its identity provider The dashboard reached through the web console sent the browser to Keycloak on :10443 of the control address, out of the proxy origin and at an address a remote operator cannot reach. The cube-cos origin now declares that endpoint as a companion: its own origin, on :10443 of the same pinned address, opened by the same session. Same address, so it is another port on the load balancer that already exists rather than a second one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PZ5umjjCedZwWtbAbiMjfj Signed-off-by: Travis Wu --- internal/enterprise/assets/install-advisor.sh | 10 +++++++++- internal/enterprise/assets_test.go | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/internal/enterprise/assets/install-advisor.sh b/internal/enterprise/assets/install-advisor.sh index 4767e09..de82a8e 100644 --- a/internal/enterprise/assets/install-advisor.sh +++ b/internal/enterprise/assets/install-advisor.sh @@ -259,9 +259,17 @@ if [ "${#POOL_ADDRS[@]}" -ge 2 ]; then # loopback: nginx serves the UI on the management address, and 127.0.0.1:8080 # is httpd, which answers 403 to everything -- a target pointed there looks # configured and refuses every request. + # + # Keycloak, on :10443 of the same address, is a companion rather than an + # origin of its own: the dashboard sends the browser straight at it, so the + # two have to be reachable from one session, and they share one load + # balancer because they share one address. WC_ARGS+=(--set "webConsole.origins[$n].address=${POOL_ADDRS[$n]}" \ --set "webConsole.origins[$n].upstream=https://$CTRL" \ - --set "webConsole.origins[$n].targets[0]=cube-cos") + --set "webConsole.origins[$n].targets[0]=cube-cos" \ + --set "webConsole.origins[$n].companions[0].address=${POOL_ADDRS[$n]}:10443" \ + --set "webConsole.origins[$n].companions[0].upstream=https://$CTRL:10443" \ + --set "webConsole.origins[$n].companions[0].targets[0]=cube-cos-idp") echo "web console enabled on ${#POOL_ADDRS[@]} origin address(es)." elif [ -n "$CONSOLE_POOL" ]; then echo "warning: the console pool has fewer than 2 addresses; leaving the web console off" >&2 diff --git a/internal/enterprise/assets_test.go b/internal/enterprise/assets_test.go index 51bc29a..804949d 100644 --- a/internal/enterprise/assets_test.go +++ b/internal/enterprise/assets_test.go @@ -262,3 +262,18 @@ func TestAdvisorConsolePointsCubeCosAtTheDashboard(t *testing.T) { "script already resolves") } } + +// The dashboard sends the browser at Keycloak on :10443 of the same address, +// so that endpoint has to be part of the same openable group — a companion, +// not a separate origin the session does not carry. +func TestAdvisorInstallGivesTheDashboardItsIdentityProvider(t *testing.T) { + for _, want := range []string{ + "webConsole.origins[$n].companions[0].upstream=https://$CTRL:10443", + "webConsole.origins[$n].companions[0].targets[0]=cube-cos-idp", + "webConsole.origins[$n].companions[0].address=${POOL_ADDRS[$n]}:10443", + } { + if !contains(installAdvisorScript, want) { + t.Errorf("install-advisor.sh does not set %s", want) + } + } +} From d2f30e5547e8d08c12361dddaba73a9c712d47a9 Mon Sep 17 00:00:00 2001 From: Travis Wu Date: Thu, 17 Sep 2026 02:26:12 +0800 Subject: [PATCH 3/4] fix(enterprise): give the dashboard origin Skyline and the Ceph dashboard too The dashboard's home page links out to three endpoints on other ports of the control address -- Keycloak (:10443), Skyline (:9999) and the Ceph dashboard (:7443) -- and all three are built in script from the cluster address handed over as a bare string. A browser following one of them left the proxy origin for an address a remote operator cannot reach. All three are now companions of the cube-cos origin, so they share its pinned address, its session and the load balancer it already has. The list is a loop rather than three copies of the same three --set flags. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PZ5umjjCedZwWtbAbiMjfj Signed-off-by: Travis Wu --- internal/enterprise/assets/install-advisor.sh | 22 ++++++++++++------- internal/enterprise/assets_test.go | 15 ++++++++----- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/internal/enterprise/assets/install-advisor.sh b/internal/enterprise/assets/install-advisor.sh index de82a8e..2f9c18e 100644 --- a/internal/enterprise/assets/install-advisor.sh +++ b/internal/enterprise/assets/install-advisor.sh @@ -260,16 +260,22 @@ if [ "${#POOL_ADDRS[@]}" -ge 2 ]; then # is httpd, which answers 403 to everything -- a target pointed there looks # configured and refuses every request. # - # Keycloak, on :10443 of the same address, is a companion rather than an - # origin of its own: the dashboard sends the browser straight at it, so the - # two have to be reachable from one session, and they share one load - # balancer because they share one address. + # The dashboard's own links out, each on another port of the same address: + # Keycloak (:10443), Skyline (:9999) and the Ceph dashboard (:7443). They are + # companions rather than origins of their own -- the dashboard sends the + # browser straight at them, so they have to be reachable from one session, + # and they share one load balancer because they share one address. WC_ARGS+=(--set "webConsole.origins[$n].address=${POOL_ADDRS[$n]}" \ --set "webConsole.origins[$n].upstream=https://$CTRL" \ - --set "webConsole.origins[$n].targets[0]=cube-cos" \ - --set "webConsole.origins[$n].companions[0].address=${POOL_ADDRS[$n]}:10443" \ - --set "webConsole.origins[$n].companions[0].upstream=https://$CTRL:10443" \ - --set "webConsole.origins[$n].companions[0].targets[0]=cube-cos-idp") + --set "webConsole.origins[$n].targets[0]=cube-cos") + c=0 + for pair in "10443:cube-cos-idp" "9999:cube-cos-skyline" "7443:cube-cos-ceph"; do + port="${pair%%:*}"; name="${pair#*:}" + WC_ARGS+=(--set "webConsole.origins[$n].companions[$c].address=${POOL_ADDRS[$n]}:$port" \ + --set "webConsole.origins[$n].companions[$c].upstream=https://$CTRL:$port" \ + --set "webConsole.origins[$n].companions[$c].targets[0]=$name") + c=$((c+1)) + done echo "web console enabled on ${#POOL_ADDRS[@]} origin address(es)." elif [ -n "$CONSOLE_POOL" ]; then echo "warning: the console pool has fewer than 2 addresses; leaving the web console off" >&2 diff --git a/internal/enterprise/assets_test.go b/internal/enterprise/assets_test.go index 804949d..ac871c6 100644 --- a/internal/enterprise/assets_test.go +++ b/internal/enterprise/assets_test.go @@ -265,12 +265,17 @@ func TestAdvisorConsolePointsCubeCosAtTheDashboard(t *testing.T) { // The dashboard sends the browser at Keycloak on :10443 of the same address, // so that endpoint has to be part of the same openable group — a companion, -// not a separate origin the session does not carry. -func TestAdvisorInstallGivesTheDashboardItsIdentityProvider(t *testing.T) { +// The dashboard links out to Keycloak, Skyline and the Ceph dashboard, each +// on another port of the same address. Every one has to be part of the same +// openable group -- a companion, not a separate origin the session does not +// carry, which would answer "no such origin" the moment the browser followed +// the link. +func TestAdvisorInstallGivesTheDashboardItsCompanions(t *testing.T) { for _, want := range []string{ - "webConsole.origins[$n].companions[0].upstream=https://$CTRL:10443", - "webConsole.origins[$n].companions[0].targets[0]=cube-cos-idp", - "webConsole.origins[$n].companions[0].address=${POOL_ADDRS[$n]}:10443", + `"10443:cube-cos-idp" "9999:cube-cos-skyline" "7443:cube-cos-ceph"`, + "webConsole.origins[$n].companions[$c].address=${POOL_ADDRS[$n]}:$port", + "webConsole.origins[$n].companions[$c].upstream=https://$CTRL:$port", + "webConsole.origins[$n].companions[$c].targets[0]=$name", } { if !contains(installAdvisorScript, want) { t.Errorf("install-advisor.sh does not set %s", want) From d43a17d3ce3966a28efbf159bc54053a0514faf7 Mon Sep 17 00:00:00 2001 From: Travis Wu Date: Thu, 17 Sep 2026 10:00:20 +0800 Subject: [PATCH 4/4] fix(enterprise): open the CMP portal at /portal The framework ingress serves the portal under /portal and Keycloak under /auth, and claims nothing at the root, so the origin opened at "/" landed on the ingress's own 404 and read as a target that does not work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PZ5umjjCedZwWtbAbiMjfj Signed-off-by: Travis Wu --- internal/enterprise/assets/install-advisor.sh | 31 ++++++++----------- internal/enterprise/assets_test.go | 8 +++++ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/internal/enterprise/assets/install-advisor.sh b/internal/enterprise/assets/install-advisor.sh index 2f9c18e..871fcfd 100644 --- a/internal/enterprise/assets/install-advisor.sh +++ b/internal/enterprise/assets/install-advisor.sh @@ -247,24 +247,23 @@ if [ "${#POOL_ADDRS[@]}" -ge 2 ]; then WC_ARGS+=(--set webConsole.enabled=true --set webConsole.mode=address) n=0 if [ -n "$INGRESS" ]; then + # /portal, not /: the framework ingress serves the portal there and + # Keycloak under /auth, and claims nothing at the root. WC_ARGS+=(--set "webConsole.origins[$n].address=${POOL_ADDRS[$n]}" \ --set "webConsole.origins[$n].upstream=https://$INGRESS" \ + --set "webConsole.origins[$n].path=/portal" \ --set "webConsole.origins[$n].targets[0]=cube-cmp" \ --set "webConsole.origins[$n].targets[1]=app-fw-idp") n=$((n+1)) else echo "warning: no ingress-lb on framework $FRAMEWORK; the CMP console origin is not configured" >&2 fi - # The node's own dashboard. Addressed at the control VIP rather than - # loopback: nginx serves the UI on the management address, and 127.0.0.1:8080 - # is httpd, which answers 403 to everything -- a target pointed there looks - # configured and refuses every request. + # The node's own dashboard, at the control VIP rather than loopback: + # 127.0.0.1:8080 is httpd, which answers 403 to everything. # - # The dashboard's own links out, each on another port of the same address: - # Keycloak (:10443), Skyline (:9999) and the Ceph dashboard (:7443). They are - # companions rather than origins of their own -- the dashboard sends the - # browser straight at them, so they have to be reachable from one session, - # and they share one load balancer because they share one address. + # The three endpoints it links out to are companions rather than origins of + # their own: the dashboard sends the browser straight at them, so they must + # be reachable from one session, and they share its address and its LB. WC_ARGS+=(--set "webConsole.origins[$n].address=${POOL_ADDRS[$n]}" \ --set "webConsole.origins[$n].upstream=https://$CTRL" \ --set "webConsole.origins[$n].targets[0]=cube-cos") @@ -282,11 +281,9 @@ elif [ -n "$CONSOLE_POOL" ]; then fi # --- the inference provider --- -# Carried through a re-run, like the TLS and enrollment material above, and for -# the same reason: this script renders the whole release, so a value it does not -# pass back is a value helm removes. Leaving it out reset the provider to the -# chart's placeholder and dropped the key, which turns a working chat surface -# into one that reports itself enabled and fails on the first question. +# Carried through a re-run, like the TLS and enrollment material above and for +# the same reason: this script renders the whole release, so a value it does +# not pass back is one helm removes. PROVIDER_ARGS=() PROVIDER_KEY_FILE="${7:-}" if [ -n "$PROVIDER_KEY_FILE" ]; then @@ -298,10 +295,8 @@ else # Read back what the deployment is already using. The key lives in the # Secret; the URL is an argument on the container. PREV_KEY="$($K -n "$NS" get secret cube-advisor-secrets -o jsonpath='{.data.providerKey}' 2>/dev/null | base64 -d)" - # Read the flag's value by position in the argument list. A jsonpath range - # piped through grep looked simpler and silently produced nothing, which is - # the failure that let the URL fall back to the chart's placeholder while the - # key was carried correctly. + # By position in the argument list: a jsonpath range piped through grep + # silently produced nothing, carrying the key while the URL fell back. PREV_URL="$($K -n "$NS" get deploy cube-advisor -o json 2>/dev/null | jq -r ' .spec.template.spec.containers[0].args as $a | ($a | index("-provider-url")) as $i diff --git a/internal/enterprise/assets_test.go b/internal/enterprise/assets_test.go index ac871c6..0f1ec95 100644 --- a/internal/enterprise/assets_test.go +++ b/internal/enterprise/assets_test.go @@ -282,3 +282,11 @@ func TestAdvisorInstallGivesTheDashboardItsCompanions(t *testing.T) { } } } + +// The framework ingress serves the portal under /portal and claims nothing at +// the root, so an origin opened at / lands on the ingress's own 404. +func TestAdvisorInstallOpensTheCmpPortalAtItsOwnPath(t *testing.T) { + if !contains(installAdvisorScript, "webConsole.origins[$n].path=/portal") { + t.Error("install-advisor.sh does not give the CMP origin its landing path") + } +}