Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,26 @@
{{- fail "knative_operator.multicluster.enabled is true but knative_operator.multicluster.accessProvidersConfig is empty; either provide an access-providers config or disable multi-cluster support" }}
{{- end }}
{{- $mountPaths := list }}
{{- range ($mc.plugins | default (list)) }}
{{- $mountPaths = append $mountPaths .mountPath }}
{{- range $index, $plugin := ($mc.plugins | default (list)) }}
{{- $mountPath := $plugin.mountPath | default "" | toString | clean }}
{{- if not (hasPrefix "/" $mountPath) }}
{{- fail (printf "knative_operator.multicluster.plugins[%d].mountPath must be an absolute path, got %q" $index ($plugin.mountPath | default "" | toString)) }}
{{- end }}
{{- $mountPaths = append $mountPaths $mountPath }}
{{- end }}
{{- $cfg := $mc.accessProvidersConfig | default dict }}
{{- range ($cfg.providers | default (list)) }}
{{- $cmd := (.execConfig | default dict).command | default "" }}
{{- if $cmd }}
{{- $cmdDir := dir $cmd }}
{{- if not (has $cmdDir $mountPaths) }}
{{- fail (printf "provider %q: command dir %q does not match any plugins[].mountPath %v" .name $cmdDir $mountPaths) }}
{{- $rawCmd := (.execConfig | default dict).command | default "" | toString }}
{{- $cmd := $rawCmd | clean }}
{{- if $rawCmd }}
{{- $matched := false }}
{{- range $mountPath := $mountPaths }}
{{- if hasPrefix (printf "%s/" $mountPath) $cmd }}
{{- $matched = true }}
{{- end }}
{{- end }}
{{- if not $matched }}
{{- fail (printf "provider %q: command %q is not under any plugins[].mountPath %v" .name $cmd $mountPaths) }}
{{- end }}
{{- end }}
{{- end }}
Expand Down
8 changes: 6 additions & 2 deletions config/charts/knative-operator/templates/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -867,9 +867,13 @@ spec:
- name: access-config
mountPath: /etc/cluster-inventory
readOnly: true
{{- range ($mc.plugins | default list) }}
{{- range $index, $plugin := ($mc.plugins | default list) }}
{{- $mountPath := $plugin.mountPath | default "" | toString | clean }}
{{- if not (hasPrefix "/" $mountPath) }}
{{- fail (printf "knative_operator.multicluster.plugins[%d].mountPath must be an absolute path, got %q" $index ($plugin.mountPath | default "" | toString)) }}
{{- end }}
- name: {{ .name }}
mountPath: {{ .mountPath }}
mountPath: {{ $mountPath }}
readOnly: true
{{- end }}
{{- end }}
Expand Down
13 changes: 8 additions & 5 deletions config/charts/knative-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,24 @@ knative_operator:
# - name: secretreader
# execConfig:
# apiVersion: client.authentication.k8s.io/v1
# command: /access-plugins/secretreader/secretreader-plugin
# command: /access-plugins/secretreader/bin/secretreader-plugin
# interactiveMode: Never
# provideClusterInfo: true
# - name: kubeconfig-secretreader
# execConfig:
# apiVersion: client.authentication.k8s.io/v1
# command: /access-plugins/secretreader/kubeconfig-secretreader-plugin
# command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin
# interactiveMode: Never
# provideClusterInfo: true
# plugins[] uses the Kubernetes "image" volume type
# plugins[] uses the Kubernetes "image" volume type. Each mountPath must be
# absolute, and each execConfig.command must point under one of these paths.
plugins: []
# plugins:
# - name: secretreader
# image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.0
# image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3
# mountPath: /access-plugins/secretreader
# - name: kubeconfig-secretreader
# image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.0
# image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.3
# mountPath: /access-plugins/kubeconfig-secretreader
# Polling cadence for spoke deployment readiness.
remoteDeploymentsPollInterval: 10s
Expand Down
19 changes: 10 additions & 9 deletions docs/development/e2e-multicluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ KUBECONFIG="${SPOKE_HOST_KUBECONFIG}" kubectl get nodes
## 2. Install the ClusterProfile CRD on the hub

```bash
: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.0/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}"
: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.3/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}"
kubectl apply -f "${CLUSTER_INVENTORY_CRD_URL}"
kubectl wait --for=condition=Established --timeout=60s \
crd/clusterprofiles.multicluster.x-k8s.io
Expand All @@ -54,12 +54,13 @@ Apply the operator from source:
ko apply -Rf config/
```

Generate a spoke bootstrap token and mount the access provider plumbing. The
helper script does this end to end:
Generate a spoke bootstrap token, store it as `knative-operator/${SPOKE_CLUSTER_NAME}`,
and mount the official `secretreader` image as the access provider. The helper
script does this end to end:

```bash
source test/e2e-common.sh
install_access_provider_config # builds and installs the token-exec-plugin
install_access_provider_config # installs the secretreader access provider
apply_cluster_profile default # creates the ClusterProfile on the hub
```

Expand All @@ -70,11 +71,11 @@ A minimal provider config (written by the helper to
{
"providers": [
{
"name": "e2e-static-token",
"name": "secretreader",
"execConfig": {
"apiVersion": "client.authentication.k8s.io/v1",
"command": "/etc/cluster-inventory/plugin/ko-app/token-exec-plugin",
"args": ["/etc/cluster-inventory/access/token"],
"command": "/etc/cluster-inventory/plugin/bin/secretreader-plugin",
"provideClusterInfo": true,
"interactiveMode": "Never"
}
}
Expand Down Expand Up @@ -205,8 +206,8 @@ which reuses the same helpers this guide calls manually:
`AccessProviderFailed`, exec into the operator and run the plugin manually:
```bash
kubectl -n knative-operator exec deploy/knative-operator -- \
/etc/cluster-inventory/plugin/ko-app/token-exec-plugin \
/etc/cluster-inventory/access/token
env KUBERNETES_EXEC_INFO="{\"apiVersion\":\"client.authentication.k8s.io/v1\",\"kind\":\"ExecCredential\",\"spec\":{\"cluster\":{\"server\":\"https://debug.invalid\",\"config\":{\"clusterName\":\"${SPOKE_CLUSTER_NAME}\"}}}}" \
/etc/cluster-inventory/plugin/bin/secretreader-plugin
```

## 8. Cleanup
Expand Down
24 changes: 18 additions & 6 deletions docs/multicluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,31 @@ knative_operator:
enabled: true
accessProvidersConfig:
providers:
- name: token-secretreader
- name: secretreader
execConfig:
apiVersion: client.authentication.k8s.io/v1
command: /access-plugins/token-secretreader/kubeconfig-secretreader-plugin
command: /access-plugins/secretreader/bin/secretreader-plugin
interactiveMode: Never
provideClusterInfo: true
- name: kubeconfig-secretreader
execConfig:
apiVersion: client.authentication.k8s.io/v1
command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin
interactiveMode: Never
provideClusterInfo: true
plugins:
- name: token-secretreader
image: ghcr.io/example/plugin:v1.0.0
mountPath: /access-plugins/token-secretreader
- name: secretreader
image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3
mountPath: /access-plugins/secretreader
- name: kubeconfig-secretreader
image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.3
mountPath: /access-plugins/kubeconfig-secretreader
```

The chart creates a `ConfigMap` with the provider config and mounts each
plugin as a Kubernetes image volume inside the operator pod.
plugin as a Kubernetes image volume inside the operator pod. Each
`plugins[].mountPath` must be absolute, and each `execConfig.command` must
point under a plugin mount path, not at the mount directory itself.

## Namespace configuration

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ require (
knative.dev/pkg v0.0.0-20260602142205-ac97e43f6622
knative.dev/reconciler-test v0.0.0-20260602150814-125bf8d48e1c
knative.dev/serving v0.49.1-0.20260604132906-516bc43f3667
sigs.k8s.io/cluster-inventory-api v0.1.0
sigs.k8s.io/cluster-inventory-api v0.1.3
sigs.k8s.io/controller-tools v0.20.1
sigs.k8s.io/yaml v1.6.0
)
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1835,8 +1835,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
sigs.k8s.io/cluster-inventory-api v0.1.0 h1:DG/hLTIJkdkKfuyMMA0ybbtBbFNWr7S4QeQcAmlSnGo=
sigs.k8s.io/cluster-inventory-api v0.1.0/go.mod h1:7J3M6srZ1I4snZR+p5zxgEBdXnia3tlHo5ODMHJpEUk=
sigs.k8s.io/cluster-inventory-api v0.1.3 h1:E7GY85hOIIPdALNdTO9Cbs2PXqjotWq6afe/NTwgCJo=
sigs.k8s.io/cluster-inventory-api v0.1.3/go.mod h1:7J3M6srZ1I4snZR+p5zxgEBdXnia3tlHo5ODMHJpEUk=
sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds=
sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80=
sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0=
Expand Down
49 changes: 0 additions & 49 deletions test/cmd/token-exec-plugin/main.go

This file was deleted.

4 changes: 4 additions & 0 deletions test/config/multicluster/clusterprofile-status.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ status:
cluster:
server: "${SPOKE_INTERNAL_ENDPOINT}"
certificate-authority-data: "${SPOKE_CA_DATA_B64}"
extensions:
- name: client.authentication.k8s.io/exec
extension:
clusterName: "${SPOKE_CLUSTER_NAME}"
conditions:
- type: ControlPlaneHealthy
status: "True"
Expand Down
29 changes: 12 additions & 17 deletions test/e2e-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ export KO_FLAGS="${KO_FLAGS:-}"
export INGRESS_CLASS=${INGRESS_CLASS:-istio.ingress.networking.knative.dev}
export TIMEOUT_CI=30m

: "${MC_PROVIDER_CONFIGMAP:=clusterprofile-provider-file}"
: "${MC_PROVIDER_MOUNT_PATH:=/etc/cluster-inventory}"
: "${MC_PROVIDER_PLUGIN_MOUNT_PATH:=/etc/cluster-inventory/plugin}"
: "${MC_PROVIDER_PLUGIN_IMAGE:=registry.k8s.io/cluster-inventory-api/secretreader:v0.1.3}"
: "${MC_PROVIDER_NAME:=secretreader}"

# Boolean used to indicate whether to generate serving YAML based on the latest code in the branch KNATIVE_SERVING_REPO_BRANCH.
GENERATE_SERVING_YAML=0

Expand Down Expand Up @@ -464,8 +470,9 @@ function apply_cluster_profile() {
SPOKE_INTERNAL_ENDPOINT="${spoke_endpoint}" \
SPOKE_CA_DATA_B64="${spoke_ca_b64}" \
MC_PROVIDER_NAME="${MC_PROVIDER_NAME}" \
SPOKE_CLUSTER_NAME="${SPOKE_CLUSTER_NAME}" \
TRANSITION="$(date -u +%FT%TZ)" \
envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${TRANSITION}' \
envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${SPOKE_CLUSTER_NAME} ${TRANSITION}' \
< test/config/multicluster/clusterprofile-status.yaml.tmpl \
> "${status_file}" || return 1

Expand All @@ -477,23 +484,14 @@ function apply_cluster_profile() {
}

function install_access_provider_config() {
echo ">> Building token-exec-plugin image via ko"
local plugin_image
plugin_image="$(ko build ./test/cmd/token-exec-plugin)" || return 1
if [[ -z "${plugin_image}" ]]; then
echo "ERROR: ko build did not emit an image reference for token-exec-plugin" >&2
return 1
fi
echo ">> token-exec-plugin image: ${plugin_image}"

echo ">> Installing access provider ConfigMap/Secret and patching operator deployment"
local tmpdir
tmpdir="$(mktemp -d)" || return 1
add_trap "rm -rf ${tmpdir}" EXIT
local token_file="${tmpdir}/token"
_spoke_bootstrap_token "${token_file}" || return 1

local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/ko-app/token-exec-plugin"
local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/bin/secretreader-plugin"
cat > "${tmpdir}/provider-config.json" <<EOF
{
"providers": [
Expand All @@ -502,7 +500,7 @@ function install_access_provider_config() {
"execConfig": {
"apiVersion": "client.authentication.k8s.io/v1",
"command": "${plugin_command}",
"args": ["${MC_PROVIDER_TOKEN_MOUNT_PATH}/token"],
"provideClusterInfo": true,
"interactiveMode": "Never"
}
}
Expand All @@ -514,24 +512,21 @@ EOF
--from-file=config.json="${tmpdir}/provider-config.json" \
--dry-run=client -o yaml | kubectl apply -f - || return 1

kubectl -n "${TEST_OPERATOR_NAMESPACE}" create secret generic "${MC_PROVIDER_TOKEN_SECRET}" \
kubectl -n "${TEST_OPERATOR_NAMESPACE}" create secret generic "${SPOKE_CLUSTER_NAME}" \
--from-file=token="${token_file}" \
--dry-run=client -o yaml | kubectl apply -f - || return 1

# 0444
kubectl -n "${TEST_OPERATOR_NAMESPACE}" patch deployment knative-operator \
--type=json \
-p "$(cat <<EOF
[
{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": ["--clusterprofile-provider-file=${MC_PROVIDER_MOUNT_PATH}/config.json"]},
{"op": "add", "path": "/spec/template/spec/volumes", "value": [
{"name": "access-config", "configMap": {"name": "${MC_PROVIDER_CONFIGMAP}"}},
{"name": "provider-token", "secret": {"secretName": "${MC_PROVIDER_TOKEN_SECRET}", "defaultMode": 292}},
{"name": "access-plugin", "image": {"reference": "${plugin_image}", "pullPolicy": "IfNotPresent"}}
{"name": "access-plugin", "image": {"reference": "${MC_PROVIDER_PLUGIN_IMAGE}", "pullPolicy": "IfNotPresent"}}
]},
{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts", "value": [
{"name": "access-config", "mountPath": "${MC_PROVIDER_MOUNT_PATH}", "readOnly": true},
{"name": "provider-token", "mountPath": "${MC_PROVIDER_TOKEN_MOUNT_PATH}", "readOnly": true},
{"name": "access-plugin", "mountPath": "${MC_PROVIDER_PLUGIN_MOUNT_PATH}", "readOnly": true}
]}
]
Expand Down
14 changes: 4 additions & 10 deletions test/e2e-tests-multicluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,12 @@ source "$(dirname "$0")/e2e-common.sh"
: "${SPOKE_CLUSTER_NAME:=spoke}"
: "${SPOKE_KUBECONFIG:=/tmp/spoke.kubeconfig}"
: "${SPOKE_HOST_KUBECONFIG:=/tmp/spoke-host.kubeconfig}"
: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.0/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}"
: "${MC_PROVIDER_CONFIGMAP:=clusterprofile-provider-file}"
: "${MC_PROVIDER_TOKEN_SECRET:=clusterprofile-provider-token}"
: "${MC_PROVIDER_MOUNT_PATH:=/etc/cluster-inventory}"
: "${MC_PROVIDER_TOKEN_MOUNT_PATH:=/etc/cluster-inventory/access}"
: "${MC_PROVIDER_PLUGIN_MOUNT_PATH:=/etc/cluster-inventory/plugin}"
: "${MC_PROVIDER_NAME:=e2e-static-token}"
: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.3/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}"
: "${SPOKE_GATEWAY_API_CRD_URL:=https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/experimental-install.yaml}"
export SPOKE_CLUSTER_NAME SPOKE_KUBECONFIG SPOKE_HOST_KUBECONFIG
export CLUSTER_INVENTORY_CRD_URL MC_PROVIDER_CONFIGMAP MC_PROVIDER_TOKEN_SECRET
export MC_PROVIDER_MOUNT_PATH MC_PROVIDER_TOKEN_MOUNT_PATH
export MC_PROVIDER_PLUGIN_MOUNT_PATH MC_PROVIDER_NAME
export CLUSTER_INVENTORY_CRD_URL MC_PROVIDER_CONFIGMAP
export MC_PROVIDER_MOUNT_PATH MC_PROVIDER_PLUGIN_MOUNT_PATH
export MC_PROVIDER_PLUGIN_IMAGE MC_PROVIDER_NAME
export SPOKE_GATEWAY_API_CRD_URL

function knative_setup() {
Expand Down
2 changes: 1 addition & 1 deletion vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1933,7 +1933,7 @@ knative.dev/serving/test/defaultsystem
knative.dev/serving/test/e2e
knative.dev/serving/test/upgrade
knative.dev/serving/test/v1
# sigs.k8s.io/cluster-inventory-api v0.1.0
# sigs.k8s.io/cluster-inventory-api v0.1.3
## explicit; go 1.25.0
sigs.k8s.io/cluster-inventory-api/apis/v1alpha1
sigs.k8s.io/cluster-inventory-api/client/clientset/versioned
Expand Down
Loading