Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/policies/v1/policyserver_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ type PolicyServerSpec struct {

// Port exposed by the metrics Service for this policy server.
// When unset, defaults to the controller-wide default
// (KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).
// (--policy-server-metrics-port CLI flag, deprecated KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).
// Only relevant when metrics are enabled.
//
// Use this field to customize which port Prometheus scrapes for this
Expand Down
5 changes: 3 additions & 2 deletions charts/kubewarden-controller/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ spec:
{{- if and (eq .Values.telemetry.mode "sidecar") }}
- --enable-otel-sidecar
{{- end }}
{{- if and .Values.telemetry.metrics (eq .Values.telemetry.mode "sidecar") }}
- --policy-server-metrics-port={{ .Values.telemetry.sidecar.metrics.port | default 8080 }}
{{- end }}
{{- if .Values.telemetry.metrics }}
- --enable-metrics
{{- end }}
Expand All @@ -100,8 +103,6 @@ spec:
- /controller
env:
{{- if and .Values.telemetry.metrics (eq .Values.telemetry.mode "sidecar") }}
- name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT
value: "{{ .Values.telemetry.sidecar.metrics.port | default 8080 }}"
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "https://localhost:4317"
- name: OTEL_EXPORTER_OTLP_INSECURE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,9 @@ tests:
content:
--opentelemetry-client-certificate-secret=my-client-cert
- notContains:
path: spec.template.spec.containers[0].env
path: spec.template.spec.containers[0].args
content:
name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT
value: "8080"
--policy-server-metrics-port=8080
- contains:
path: spec.template.spec.containers[0].env
content:
Expand Down Expand Up @@ -138,10 +137,9 @@ tests:
content:
--opentelemetry-client-certificate-secret=
- contains:
path: spec.template.spec.containers[0].env
path: spec.template.spec.containers[0].args
content:
name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT
value: "8080"
--policy-server-metrics-port=8080
- contains:
path: spec.template.spec.containers[0].env
content:
Expand Down Expand Up @@ -202,3 +200,26 @@ tests:
path: tls.crt
- key: tls.key
path: tls.key
- it: "sidecar tracing should not set policy server metrics port when metrics are disabled"
set:
telemetry:
mode: "sidecar"
metrics: false
tracing: true
asserts:
- contains:
path: spec.template.spec.containers[0].args
content:
--enable-otel-sidecar
- notContains:
path: spec.template.spec.containers[0].args
content:
--enable-metrics
- contains:
path: spec.template.spec.containers[0].args
content:
--enable-tracing
- notContains:
path: spec.template.spec.containers[0].args
content:
--policy-server-metrics-port=8080
Original file line number Diff line number Diff line change
Expand Up @@ -1177,7 +1177,7 @@ spec:
description: |-
Port exposed by the metrics Service for this policy server.
When unset, defaults to the controller-wide default
(KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).
(--policy-server-metrics-port CLI flag, deprecated KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).
Only relevant when metrics are enabled.

Use this field to customize which port Prometheus scrapes for this
Expand Down
78 changes: 57 additions & 21 deletions cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,16 @@ func main() {
var openTelemetryClientCertificateSecret string
var openTelemetryCertificateSecret string
var imagePullSecretsFlag string
var policyServerMetricsPort int

flag.StringVar(&mgrOpts.MetricsAddr, "metrics-bind-address", ":8088", "The address the controller-runtime metric endpoint binds to.")
flag.StringVar(&mgrOpts.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.IntVar(&mgrOpts.WebhookServerPort, "webhook-server-port", 9443, "The port the webhook server listens on.")
flag.IntVar(&policyServerMetricsPort,
"policy-server-metrics-port",
constants.PolicyServerMetricsPort,
"The default port exposed by every PolicyServer metrics Service. "+
"Per-PolicyServer overrides (spec.metricsPort) always take priority.")
flag.BoolVar(&mgrOpts.EnableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
Expand Down Expand Up @@ -159,6 +165,12 @@ func main() {
opts := zap.Options{}
opts.BindFlags(flag.CommandLine)
flag.Parse()
policyServerMetricsPortFlagSet := false
flag.Visit(func(f *flag.Flag) {
if f.Name == "policy-server-metrics-port" {
policyServerMetricsPortFlagSet = true
}
})
mgrOpts.EnableMutualTLS = config.ClientCAConfigMapName != ""
config.ImagePullSecrets = parseImagePullSecrets(imagePullSecretsFlag)
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
Expand Down Expand Up @@ -195,30 +207,54 @@ func main() {
"Use only when the Kubernetes API server cannot reach pod-network webhook endpoints.")
}

// Read the global default metrics port for PolicyServer services from the
// environment variable, falling back to the hardcoded constant.
policyServerMetricsPort := int32(constants.PolicyServerMetricsPort)
// Validate --policy-server-metrics-port range.
if int64(policyServerMetricsPort) < minAllowedPort ||
int64(policyServerMetricsPort) > maxAllowedPort {
setupLog.Error(
errors.New("port must be between 1 and 65535"),
"invalid policy server metrics port",
"flag", "--policy-server-metrics-port",
"value", policyServerMetricsPort,
"min", minAllowedPort,
"max", maxAllowedPort,
)
retcode = 1
return
}
if envPort := os.Getenv(constants.PolicyServerMetricsPortEnvVar); envPort != "" {
parsed, err := strconv.ParseInt(envPort, 10, 32)
if err != nil {
setupLog.Error(err, "cannot parse env var as integer port",
"envVar", constants.PolicyServerMetricsPortEnvVar, "value", envPort)
retcode = 1
return
}
if parsed < minAllowedPort || parsed > maxAllowedPort {
setupLog.Error(
errors.New("port must be between 1 and 65535"),
"invalid env var port value",
if policyServerMetricsPortFlagSet {
setupLog.Info(
"WARNING: deprecated environment variable ignored because --policy-server-metrics-port was set",
"envVar", constants.PolicyServerMetricsPortEnvVar,
"value", envPort,
"min", minAllowedPort,
"max", maxAllowedPort,
"flag", "--policy-server-metrics-port",
)
} else {
envPortParsed, err := strconv.ParseInt(envPort, 10, 32)
if err != nil {
setupLog.Error(err, "invalid policy server metrics port environment variable",
"envVar", constants.PolicyServerMetricsPortEnvVar, "value", envPort)
retcode = 1
return
}
policyServerMetricsPort = int(envPortParsed)
setupLog.Info(
"WARNING: deprecated environment variable used as fallback; use --policy-server-metrics-port instead",
"envVar", constants.PolicyServerMetricsPortEnvVar,
"flag", "--policy-server-metrics-port",
)
retcode = 1
return
}
policyServerMetricsPort = int32(parsed)
}
if int64(policyServerMetricsPort) < minAllowedPort ||
int64(policyServerMetricsPort) > maxAllowedPort {
setupLog.Error(
errors.New("port must be between 1 and 65535"),
"invalid policy server metrics port",
"value", policyServerMetricsPort,
"min", minAllowedPort,
"max", maxAllowedPort,
)
retcode = 1
return
}

if enableMetrics {
Expand Down Expand Up @@ -267,7 +303,7 @@ func main() {
mgrOpts.DeploymentsNamespace,
config,
otelConfiguration,
policyServerMetricsPort,
int32(policyServerMetricsPort),
); err != nil {
setupLog.Error(err, "unable to create controllers")
retcode = 1
Expand Down
3 changes: 1 addition & 2 deletions docs/crds/CRD-docs-for-docs-repo.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -959,7 +959,7 @@ Optional: \{} +

| *`metricsPort`* __integer__ | Port exposed by the metrics Service for this policy server. +
When unset, defaults to the controller-wide default +
(KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080). +
(--policy-server-metrics-port CLI flag, deprecated KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080). +
Only relevant when metrics are enabled. +

Use this field to customize which port Prometheus scrapes for this +
Expand Down Expand Up @@ -1588,4 +1588,3 @@ _Underlying type:_ _string_




3 changes: 1 addition & 2 deletions docs/crds/CRD-docs-for-docs-repo.md
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ _Appears in:_
| `namespacedPoliciesCapabilities` _string array_ | NamespacedPoliciesCapabilities lists host capability API calls allowed<br />for namespaced policies running on this PolicyServer. When not set,<br />no host capabilities are granted to namespaced policies.<br />Supported wildcard patterns:<br />- "*": allow all host capabilities<br />- "category/*": allow all capabilities in a category (e.g. "oci/*")<br />- "category/version/*": allow all capabilities of a specific version (e.g. "oci/v1/*")<br />- Specific capability paths (e.g. "oci/v1/verify", "net/v1/dns_lookup_host") | | Optional: \{\} <br /> |
| `webhookPort` _integer_ | Port where the policy server listens for incoming webhook requests.<br />When unset, defaults to 8443. This is the port the Kubernetes API server<br />reaches when evaluating admission requests. | | Maximum: 65535 <br />Minimum: 1 <br />Optional: \{\} <br /> |
| `readinessProbePort` _integer_ | Port used by the policy server to expose the readiness probe endpoint.<br />When unset, defaults to 8081. | | Maximum: 65535 <br />Minimum: 1 <br />Optional: \{\} <br /> |
| `metricsPort` _integer_ | Port exposed by the metrics Service for this policy server.<br />When unset, defaults to the controller-wide default<br />(KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).<br />Only relevant when metrics are enabled.<br />Use this field to customize which port Prometheus scrapes for this<br />PolicyServer's metrics Service (e.g. to match naming conventions or<br />avoid Service-level port collisions).<br />NOTE: this field controls only the Service Port (the externally visible<br />scrape port). The Service TargetPort — the port the pod actually listens<br />on — is always the controller-wide default and is not affected by this<br />field. This is intentional: when the OpenTelemetry sidecar mode is<br />enabled, each pod gets its own injected sidecar, but the pod-side<br />Prometheus listener port is determined by controller-wide/injection<br />configuration, not per PolicyServer. Therefore, changing this field does<br />not change the pod listener port and will not resolve pod-port conflicts<br />such as those caused by hostNetwork. | | Maximum: 65535 <br />Minimum: 1 <br />Optional: \{\} <br /> |
| `metricsPort` _integer_ | Port exposed by the metrics Service for this policy server.<br />When unset, defaults to the controller-wide default<br />(--policy-server-metrics-port CLI flag, deprecated KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).<br />Only relevant when metrics are enabled.<br />Use this field to customize which port Prometheus scrapes for this<br />PolicyServer's metrics Service (e.g. to match naming conventions or<br />avoid Service-level port collisions).<br />NOTE: this field controls only the Service Port (the externally visible<br />scrape port). The Service TargetPort — the port the pod actually listens<br />on — is always the controller-wide default and is not affected by this<br />field. This is intentional: when the OpenTelemetry sidecar mode is<br />enabled, each pod gets its own injected sidecar, but the pod-side<br />Prometheus listener port is determined by controller-wide/injection<br />configuration, not per PolicyServer. Therefore, changing this field does<br />not change the pod listener port and will not resolve pod-port conflicts<br />such as those caused by hostNetwork. | | Maximum: 65535 <br />Minimum: 1 <br />Optional: \{\} <br /> |



Expand Down Expand Up @@ -878,4 +878,3 @@ _Appears in:_




6 changes: 4 additions & 2 deletions internal/controller/policyserver_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,10 @@ type PolicyServerReconciler struct {
// so that in-cluster DNS resolution keeps working.
HostNetwork bool
// PolicyServerMetricsPort is the global default metrics port for PolicyServer
// Service objects. It is populated from the KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT
// environment variable at startup, falling back to constants.PolicyServerMetricsPort.
// Service objects. It is populated from the --policy-server-metrics-port CLI
// flag at startup, falling back to the deprecated
// KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var and then to
// constants.PolicyServerMetricsPort.
// A per-PolicyServer CRD field (spec.metricsPort) always takes priority.
PolicyServerMetricsPort int32
}
Expand Down
10 changes: 5 additions & 5 deletions internal/controller/policyserver_controller_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func findServicePort(ports []corev1.ServicePort, name string) *corev1.ServicePor
// TestUpdateServiceMetricsPortPriorityChain validates the 3-tier priority
// chain for the metrics Service Port:
//
// CRD field (spec.metricsPort) > env var (PolicyServerMetricsPort) > constant (8080)
// CRD field (spec.metricsPort) > --policy-server-metrics-port CLI flag > deprecated env var > constant (8080)
//
// It also verifies that the Service TargetPort is always fixed at the
// controller-wide PolicyServerMetricsPort regardless of any CRD override.
Expand All @@ -87,14 +87,14 @@ func TestUpdateServiceMetricsPortPriorityChain(t *testing.T) {
tests := []struct {
name string
metricsEnabled bool
policyServerMetricPort int32 // simulates env var / constant
policyServerMetricPort int32 // simulates controller-wide CLI flag / env var / constant
crdMetricsPort *int32
expectMetricsPort bool
expectedPort int32 // Service Port
expectedTargetPort int32 // Service TargetPort
}{
{
name: "env var only (no CRD override)",
name: "CLI flag only (no CRD override)",
metricsEnabled: true,
policyServerMetricPort: 9090,
crdMetricsPort: nil,
Expand All @@ -103,7 +103,7 @@ func TestUpdateServiceMetricsPortPriorityChain(t *testing.T) {
expectedTargetPort: 9090,
},
{
name: "CRD overrides env var",
name: "CRD overrides CLI flag",
metricsEnabled: true,
policyServerMetricPort: 9090,
crdMetricsPort: int32Ptr(9091),
Expand Down Expand Up @@ -170,7 +170,7 @@ func TestUpdateServiceMetricsPortPriorityChain(t *testing.T) {

require.NotNil(t, metricsPort, "metrics port must exist when metrics are enabled")
assert.Equal(t, tc.expectedPort, metricsPort.Port,
"Service Port should respect CRD > env var > constant priority")
"Service Port should respect CRD > CLI flag > constant priority")
assert.Equal(t, intstr.FromInt32(tc.expectedTargetPort), metricsPort.TargetPort,
"Service TargetPort must always equal the global PolicyServerMetricsPort (fixed regardless of spec.metricsPort, to avoid breaking OTel sidecar mode)")
assert.Equal(t, corev1.ProtocolTCP, metricsPort.Protocol)
Expand Down