diff --git a/config/charts/knative-operator/templates/access-providers-config.yaml b/config/charts/knative-operator/templates/access-providers-config.yaml index 7ad811969..eebfdb08d 100644 --- a/config/charts/knative-operator/templates/access-providers-config.yaml +++ b/config/charts/knative-operator/templates/access-providers-config.yaml @@ -18,16 +18,26 @@ {{- fail "knative_operator.multicluster.enabled is true but knative_operator.multicluster.accessProvidersConfig is empty; either provide an access-providers config or disable multi-cluster support" }} {{- end }} {{- $mountPaths := list }} -{{- range ($mc.plugins | default (list)) }} -{{- $mountPaths = append $mountPaths .mountPath }} +{{- range $index, $plugin := ($mc.plugins | default (list)) }} +{{- $mountPath := $plugin.mountPath | default "" | toString | clean }} +{{- if not (hasPrefix "/" $mountPath) }} +{{- fail (printf "knative_operator.multicluster.plugins[%d].mountPath must be an absolute path, got %q" $index ($plugin.mountPath | default "" | toString)) }} +{{- end }} +{{- $mountPaths = append $mountPaths $mountPath }} {{- end }} {{- $cfg := $mc.accessProvidersConfig | default dict }} {{- range ($cfg.providers | default (list)) }} -{{- $cmd := (.execConfig | default dict).command | default "" }} -{{- if $cmd }} -{{- $cmdDir := dir $cmd }} -{{- if not (has $cmdDir $mountPaths) }} -{{- fail (printf "provider %q: command dir %q does not match any plugins[].mountPath %v" .name $cmdDir $mountPaths) }} +{{- $rawCmd := (.execConfig | default dict).command | default "" | toString }} +{{- $cmd := $rawCmd | clean }} +{{- if $rawCmd }} +{{- $matched := false }} +{{- range $mountPath := $mountPaths }} +{{- if hasPrefix (printf "%s/" $mountPath) $cmd }} +{{- $matched = true }} +{{- end }} +{{- end }} +{{- if not $matched }} +{{- fail (printf "provider %q: command %q is not under any plugins[].mountPath %v" .name $cmd $mountPaths) }} {{- end }} {{- end }} {{- end }} diff --git a/config/charts/knative-operator/templates/operator.yaml b/config/charts/knative-operator/templates/operator.yaml index 04d161133..57fe18c5d 100644 --- a/config/charts/knative-operator/templates/operator.yaml +++ b/config/charts/knative-operator/templates/operator.yaml @@ -867,9 +867,13 @@ spec: - name: access-config mountPath: /etc/cluster-inventory readOnly: true -{{- range ($mc.plugins | default list) }} +{{- range $index, $plugin := ($mc.plugins | default list) }} +{{- $mountPath := $plugin.mountPath | default "" | toString | clean }} +{{- if not (hasPrefix "/" $mountPath) }} +{{- fail (printf "knative_operator.multicluster.plugins[%d].mountPath must be an absolute path, got %q" $index ($plugin.mountPath | default "" | toString)) }} +{{- end }} - name: {{ .name }} - mountPath: {{ .mountPath }} + mountPath: {{ $mountPath }} readOnly: true {{- end }} {{- end }} diff --git a/config/charts/knative-operator/values.yaml b/config/charts/knative-operator/values.yaml index a842bd60f..9734015f6 100644 --- a/config/charts/knative-operator/values.yaml +++ b/config/charts/knative-operator/values.yaml @@ -8,21 +8,24 @@ knative_operator: # - name: secretreader # execConfig: # apiVersion: client.authentication.k8s.io/v1 - # command: /access-plugins/secretreader/secretreader-plugin + # command: /access-plugins/secretreader/bin/secretreader-plugin + # interactiveMode: Never # provideClusterInfo: true # - name: kubeconfig-secretreader # execConfig: # apiVersion: client.authentication.k8s.io/v1 - # command: /access-plugins/secretreader/kubeconfig-secretreader-plugin + # command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin + # interactiveMode: Never # provideClusterInfo: true - # plugins[] uses the Kubernetes "image" volume type + # plugins[] uses the Kubernetes "image" volume type. Each mountPath must be + # absolute, and each execConfig.command must point under one of these paths. plugins: [] # plugins: # - name: secretreader - # image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.0 + # image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3 # mountPath: /access-plugins/secretreader # - name: kubeconfig-secretreader - # image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.0 + # image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.3 # mountPath: /access-plugins/kubeconfig-secretreader # Polling cadence for spoke deployment readiness. remoteDeploymentsPollInterval: 10s diff --git a/docs/development/e2e-multicluster.md b/docs/development/e2e-multicluster.md index d660827b0..d14fafce5 100644 --- a/docs/development/e2e-multicluster.md +++ b/docs/development/e2e-multicluster.md @@ -40,7 +40,7 @@ KUBECONFIG="${SPOKE_HOST_KUBECONFIG}" kubectl get nodes ## 2. Install the ClusterProfile CRD on the hub ```bash -: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.0/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}" +: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.3/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}" kubectl apply -f "${CLUSTER_INVENTORY_CRD_URL}" kubectl wait --for=condition=Established --timeout=60s \ crd/clusterprofiles.multicluster.x-k8s.io @@ -54,12 +54,13 @@ Apply the operator from source: ko apply -Rf config/ ``` -Generate a spoke bootstrap token and mount the access provider plumbing. The -helper script does this end to end: +Generate a spoke bootstrap token, store it as `knative-operator/${SPOKE_CLUSTER_NAME}`, +and mount the official `secretreader` image as the access provider. The helper +script does this end to end: ```bash source test/e2e-common.sh -install_access_provider_config # builds and installs the token-exec-plugin +install_access_provider_config # installs the secretreader access provider apply_cluster_profile default # creates the ClusterProfile on the hub ``` @@ -70,11 +71,11 @@ A minimal provider config (written by the helper to { "providers": [ { - "name": "e2e-static-token", + "name": "secretreader", "execConfig": { "apiVersion": "client.authentication.k8s.io/v1", - "command": "/etc/cluster-inventory/plugin/ko-app/token-exec-plugin", - "args": ["/etc/cluster-inventory/access/token"], + "command": "/etc/cluster-inventory/plugin/bin/secretreader-plugin", + "provideClusterInfo": true, "interactiveMode": "Never" } } @@ -205,8 +206,8 @@ which reuses the same helpers this guide calls manually: `AccessProviderFailed`, exec into the operator and run the plugin manually: ```bash kubectl -n knative-operator exec deploy/knative-operator -- \ - /etc/cluster-inventory/plugin/ko-app/token-exec-plugin \ - /etc/cluster-inventory/access/token + env KUBERNETES_EXEC_INFO="{\"apiVersion\":\"client.authentication.k8s.io/v1\",\"kind\":\"ExecCredential\",\"spec\":{\"cluster\":{\"server\":\"https://debug.invalid\",\"config\":{\"clusterName\":\"${SPOKE_CLUSTER_NAME}\"}}}}" \ + /etc/cluster-inventory/plugin/bin/secretreader-plugin ``` ## 8. Cleanup diff --git a/docs/multicluster.md b/docs/multicluster.md index 919d07546..a2e18fcef 100644 --- a/docs/multicluster.md +++ b/docs/multicluster.md @@ -52,19 +52,31 @@ knative_operator: enabled: true accessProvidersConfig: providers: - - name: token-secretreader + - name: secretreader execConfig: apiVersion: client.authentication.k8s.io/v1 - command: /access-plugins/token-secretreader/kubeconfig-secretreader-plugin + command: /access-plugins/secretreader/bin/secretreader-plugin + interactiveMode: Never + provideClusterInfo: true + - name: kubeconfig-secretreader + execConfig: + apiVersion: client.authentication.k8s.io/v1 + command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin + interactiveMode: Never provideClusterInfo: true plugins: - - name: token-secretreader - image: ghcr.io/example/plugin:v1.0.0 - mountPath: /access-plugins/token-secretreader + - name: secretreader + image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3 + mountPath: /access-plugins/secretreader + - name: kubeconfig-secretreader + image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.3 + mountPath: /access-plugins/kubeconfig-secretreader ``` The chart creates a `ConfigMap` with the provider config and mounts each -plugin as a Kubernetes image volume inside the operator pod. +plugin as a Kubernetes image volume inside the operator pod. Each +`plugins[].mountPath` must be absolute, and each `execConfig.command` must +point under a plugin mount path, not at the mount directory itself. ## Namespace configuration diff --git a/go.mod b/go.mod index 5a24a393c..5fcafbd90 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( knative.dev/pkg v0.0.0-20260602142205-ac97e43f6622 knative.dev/reconciler-test v0.0.0-20260602150814-125bf8d48e1c knative.dev/serving v0.49.1-0.20260604132906-516bc43f3667 - sigs.k8s.io/cluster-inventory-api v0.1.0 + sigs.k8s.io/cluster-inventory-api v0.1.3 sigs.k8s.io/controller-tools v0.20.1 sigs.k8s.io/yaml v1.6.0 ) diff --git a/go.sum b/go.sum index 984e8a3ca..f1d0e2f9a 100644 --- a/go.sum +++ b/go.sum @@ -1835,8 +1835,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/cluster-inventory-api v0.1.0 h1:DG/hLTIJkdkKfuyMMA0ybbtBbFNWr7S4QeQcAmlSnGo= -sigs.k8s.io/cluster-inventory-api v0.1.0/go.mod h1:7J3M6srZ1I4snZR+p5zxgEBdXnia3tlHo5ODMHJpEUk= +sigs.k8s.io/cluster-inventory-api v0.1.3 h1:E7GY85hOIIPdALNdTO9Cbs2PXqjotWq6afe/NTwgCJo= +sigs.k8s.io/cluster-inventory-api v0.1.3/go.mod h1:7J3M6srZ1I4snZR+p5zxgEBdXnia3tlHo5ODMHJpEUk= sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= diff --git a/test/cmd/token-exec-plugin/main.go b/test/cmd/token-exec-plugin/main.go deleted file mode 100644 index d2f01180f..000000000 --- a/test/cmd/token-exec-plugin/main.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2025 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// token-exec-plugin reads a bearer token from a file and writes an -// ExecCredential JSON to stdout. -package main - -import ( - "encoding/json" - "fmt" - "os" - "strings" -) - -func main() { - if len(os.Args) != 2 { - fmt.Fprintf(os.Stderr, "usage: %s \n", os.Args[0]) - os.Exit(2) - } - raw, err := os.ReadFile(os.Args[1]) - if err != nil { - fmt.Fprintf(os.Stderr, "read token %q: %v\n", os.Args[1], err) - os.Exit(1) - } - cred := map[string]any{ - "apiVersion": "client.authentication.k8s.io/v1", - "kind": "ExecCredential", - "status": map[string]any{ - "token": strings.TrimSpace(string(raw)), - }, - } - if err := json.NewEncoder(os.Stdout).Encode(cred); err != nil { - fmt.Fprintf(os.Stderr, "encode credential: %v\n", err) - os.Exit(1) - } -} diff --git a/test/config/multicluster/clusterprofile-status.yaml.tmpl b/test/config/multicluster/clusterprofile-status.yaml.tmpl index f4d02f045..d7c00dd92 100644 --- a/test/config/multicluster/clusterprofile-status.yaml.tmpl +++ b/test/config/multicluster/clusterprofile-status.yaml.tmpl @@ -6,6 +6,10 @@ status: cluster: server: "${SPOKE_INTERNAL_ENDPOINT}" certificate-authority-data: "${SPOKE_CA_DATA_B64}" + extensions: + - name: client.authentication.k8s.io/exec + extension: + clusterName: "${SPOKE_CLUSTER_NAME}" conditions: - type: ControlPlaneHealthy status: "True" diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 33360f46d..1d22f6628 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -42,6 +42,12 @@ export KO_FLAGS="${KO_FLAGS:-}" export INGRESS_CLASS=${INGRESS_CLASS:-istio.ingress.networking.knative.dev} export TIMEOUT_CI=30m +: "${MC_PROVIDER_CONFIGMAP:=clusterprofile-provider-file}" +: "${MC_PROVIDER_MOUNT_PATH:=/etc/cluster-inventory}" +: "${MC_PROVIDER_PLUGIN_MOUNT_PATH:=/etc/cluster-inventory/plugin}" +: "${MC_PROVIDER_PLUGIN_IMAGE:=registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3}" +: "${MC_PROVIDER_NAME:=secretreader}" + # Boolean used to indicate whether to generate serving YAML based on the latest code in the branch KNATIVE_SERVING_REPO_BRANCH. GENERATE_SERVING_YAML=0 @@ -464,8 +470,9 @@ function apply_cluster_profile() { SPOKE_INTERNAL_ENDPOINT="${spoke_endpoint}" \ SPOKE_CA_DATA_B64="${spoke_ca_b64}" \ MC_PROVIDER_NAME="${MC_PROVIDER_NAME}" \ + SPOKE_CLUSTER_NAME="${SPOKE_CLUSTER_NAME}" \ TRANSITION="$(date -u +%FT%TZ)" \ - envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${TRANSITION}' \ + envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${SPOKE_CLUSTER_NAME} ${TRANSITION}' \ < test/config/multicluster/clusterprofile-status.yaml.tmpl \ > "${status_file}" || return 1 @@ -477,15 +484,6 @@ function apply_cluster_profile() { } function install_access_provider_config() { - echo ">> Building token-exec-plugin image via ko" - local plugin_image - plugin_image="$(ko build ./test/cmd/token-exec-plugin)" || return 1 - if [[ -z "${plugin_image}" ]]; then - echo "ERROR: ko build did not emit an image reference for token-exec-plugin" >&2 - return 1 - fi - echo ">> token-exec-plugin image: ${plugin_image}" - echo ">> Installing access provider ConfigMap/Secret and patching operator deployment" local tmpdir tmpdir="$(mktemp -d)" || return 1 @@ -493,7 +491,7 @@ function install_access_provider_config() { local token_file="${tmpdir}/token" _spoke_bootstrap_token "${token_file}" || return 1 - local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/ko-app/token-exec-plugin" + local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/bin/secretreader-plugin" cat > "${tmpdir}/provider-config.json" <