network policies and metrics tweaks

bindata/assets.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 )
 
-//go:embed assets/*
+//go:embed assets
 var f embed.FS
 
 // Asset reads and returns the content of the named file.

bindata/assets/kueue-operator/networkpolicy/10-allow-egress-api.yaml → bindata/assets/kueue-operator/networkpolicy/operand/01-allow-egress-api.yaml

@@ -1,4 +1,4 @@
-# allow outbound traffic to kube-apiserver
+# allow outbound traffic to kube-apiserver from kueue operand pod
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -7,11 +7,11 @@ metadata:
 spec:
   podSelector:
     matchLabels:
-      app.openshift.io/name: kueue # applies to both the operator and kueue pod
+      app.kubernetes.io/name: kueue
+      control-plane: controller-manager
   egress:
   - ports:
     - protocol: TCP
       port: 6443  # we can not use a name because it is not on the pod network
   policyTypes:
   - Egress
-

bindata/assets/kueue-operator/networkpolicy/10-allow-egress-cluster-dns.yaml → bindata/assets/kueue-operator/networkpolicy/operand/02-allow-egress-cluster-dns.yaml

@@ -1,4 +1,4 @@
-# allow traffic to cluster DNS service
+# allow traffic to cluster DNS service from kueue operand pod
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -7,7 +7,8 @@ metadata:
 spec:
   podSelector:
     matchLabels:
-      app.openshift.io/name: kueue # applies to both the operator and kueue pod
+      app.kubernetes.io/name: kueue
+      control-plane: controller-manager
   egress:
   - to:
     - namespaceSelector:

bindata/assets/kueue-operator/networkpolicy/10-allow-ingress-egress-webhook.yaml → bindata/assets/kueue-operator/networkpolicy/operand/03-allow-ingress-webhook.yaml

@@ -7,7 +7,8 @@ metadata:
 spec:
   podSelector:
     matchLabels:
-      app.kubernetes.io/name: kueue # applies to the kueue pod only
+      app.kubernetes.io/name: kueue
+      control-plane: controller-manager
   ingress:
   - from:
     - namespaceSelector:

bindata/assets/kueue-operator/networkpolicy/10-allow-ingress-egress-metrics.yaml → bindata/assets/kueue-operator/networkpolicy/operand/05-allow-ingress-metrics.yaml

@@ -1,4 +1,10 @@
+<<<<<<<< HEAD:bindata/assets/kueue-operator/networkpolicy/10-allow-ingress-egress-metrics.yaml
 # allow ingress and egress traffic to/from metrics endpoint
+|||||||| parent of e772e32c (network policies and metrics tweaks):bindata/assets/kueue-operator/networkpolicy/allow-ingress-metrics.yaml
+# allow ingress traffic to metrics endpoint
+========
+# allow ingress traffic to metrics endpoint on kueue operand pod
+>>>>>>>> e772e32c (network policies and metrics tweaks):bindata/assets/kueue-operator/networkpolicy/operand/05-allow-ingress-metrics.yaml
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -7,7 +13,8 @@ metadata:
 spec:
   podSelector:
     matchLabels:
-      app.openshift.io/name: kueue # applies to both the operator and kueue pod
+      app.kubernetes.io/name: kueue
+      control-plane: controller-manager
   ingress:
   - from:
     - namespaceSelector:

bindata/assets/kueue-operator/networkpolicy/operand/99-deny-all.yaml

@@ -0,0 +1,15 @@
+# default deny-all policy for kueue operand pod
+# Applies to kueue deployment pods only
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kueue-deny-all
+  namespace: openshift-kueue-operator
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: kueue
+      control-plane: controller-manager
+  policyTypes:
+  - Ingress
+  - Egress

bindata/assets/kueue-operator/networkpolicy/operator/01-allow-egress-api.yaml

@@ -0,0 +1,20 @@
+# allow outbound traffic to kube-apiserver from operator pod
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kueue-operator-allow-egress-kube-apiserver
+  namespace: openshift-kueue-operator
+  labels:
+    app.kubernetes.io/managed-by: kueue-operator
+    app.kubernetes.io/component: operator-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app.openshift.io/component: operator
+      app.openshift.io/name: kueue
+  egress:
+  - ports:
+    - protocol: TCP
+      port: 6443  # we can not use a name because it is not on the pod network
+  policyTypes:
+  - Egress

bindata/assets/kueue-operator/networkpolicy/operator/02-allow-egress-cluster-dns.yaml

@@ -0,0 +1,29 @@
+# allow traffic to cluster DNS service from operator pod
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kueue-operator-allow-egress-cluster-dns
+  namespace: openshift-kueue-operator
+  labels:
+    app.kubernetes.io/managed-by: kueue-operator
+    app.kubernetes.io/component: operator-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app.openshift.io/component: operator
+      app.openshift.io/name: kueue
+  egress:
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: openshift-dns
+      podSelector:
+        matchLabels:
+          dns.operator.openshift.io/daemonset-dns: default
+    ports:
+    - protocol: TCP
+      port: dns-tcp
+    - protocol: UDP
+      port: dns
+  policyTypes:
+  - Egress

bindata/assets/kueue-operator/networkpolicy/operator/03-allow-ingress-metrics.yaml

@@ -0,0 +1,30 @@
+# allow ingress traffic to metrics endpoint on operator pod
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kueue-operator-allow-ingress-metrics
+  namespace: openshift-kueue-operator
+  labels:
+    app.kubernetes.io/managed-by: kueue-operator
+    app.kubernetes.io/component: operator-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app.openshift.io/component: operator
+      app.openshift.io/name: kueue
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          openshift.io/cluster-monitoring: "true"
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: openshift-monitoring
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: openshift-user-workload-monitoring
+    ports:
+    - protocol: TCP
+      port: 8443
+  policyTypes:
+  - Ingress

bindata/assets/kueue-operator/networkpolicy/operator/99-deny-all.yaml

@@ -0,0 +1,18 @@
+# default deny-all policy for operator pod
+# Applied LAST to ensure allow policies are in place first
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kueue-operator-deny-all
+  namespace: openshift-kueue-operator
+  labels:
+    app.kubernetes.io/managed-by: kueue-operator
+    app.kubernetes.io/component: operator-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app.openshift.io/component: operator
+      app.openshift.io/name: kueue
+  policyTypes:
+  - Ingress
+  - Egress

bindata/assets/kueue-operator/prometheus-rbac/role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s-metrics
+  namespace: openshift-kueue-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - pods
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch

bindata/assets/kueue-operator/prometheus-rbac/rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s-metrics
+  namespace: openshift-kueue-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s-metrics
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring

bindata/assets/kueue-operator/servicemonitor/operator-metrics.yaml

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: openshift-kueue-operator-metrics
+  namespace: openshift-kueue-operator
+  labels:
+    app.openshift.io/component: operator
+    app.openshift.io/name: kueue
+spec:
+  endpoints:
+    - interval: 30s
+      path: /metrics
+      port: metrics
+      scheme: https
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      tlsConfig:
+        ca:
+          secret:
+            name: openshift-kueue-operator-metrics-tls
+            key: ca.crt
+        serverName: openshift-kueue-operator-metrics.openshift-kueue-operator.svc
+  selector:
+    matchLabels:
+      app.openshift.io/component: operator
+      app.openshift.io/name: kueue

bundle/manifests/kueue-operator.clusterserviceversion.yaml

@@ -236,6 +236,18 @@ spec:
           - get
           - create
           - update
+        - apiGroups:
+          - authorization.k8s.io
+          resources:
+          - subjectaccessreviews
+          verbs:
+          - create
+        - apiGroups:
+          - authentication.k8s.io
+          resources:
+          - tokenreviews
+          verbs:
+          - create
         serviceAccountName: openshift-kueue-operator
       deployments:
       - name: openshift-kueue-operator
@@ -272,7 +284,7 @@ spec:
                 imagePullPolicy: Always
                 name: openshift-kueue-operator
                 ports:
-                - containerPort: 60000
+                - containerPort: 8443
                   name: metrics
                 resources: {}
                 securityContext:
@@ -284,6 +296,9 @@ spec:
                 volumeMounts:
                 - mountPath: /tmp
                   name: tmp
+                - mountPath: /var/run/secrets/serving-cert
+                  name: metrics-tls
+                  readOnly: true
               priorityClassName: system-cluster-critical
               securityContext:
                 runAsNonRoot: true
@@ -293,6 +308,10 @@ spec:
               volumes:
               - emptyDir: {}
                 name: tmp
+              - name: metrics-tls
+                secret:
+                  optional: true
+                  secretName: openshift-kueue-operator-metrics-tls
       permissions:
       - rules:
         - apiGroups:

bundle/manifests/openshift-kueue-operator-metrics_v1_service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    app.openshift.io/component: operator
+    app.openshift.io/name: kueue
+  name: openshift-kueue-operator-metrics
+spec:
+  ports:
+  - name: metrics
+    port: 60000
+    protocol: TCP
+    targetPort: 8443
+  selector:
+    name: openshift-kueue-operator
+  type: ClusterIP
+status:
+  loadBalancer: {}

deploy/02_clusterrole.yaml

@@ -164,3 +164,15 @@ rules:
       - get
       - create
       - update
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create

deploy/07_deployment.yaml

@@ -31,8 +31,11 @@ spec:
           volumeMounts:
             - name: tmp
               mountPath: "/tmp"
+            - name: metrics-tls
+              mountPath: "/var/run/secrets/serving-cert"
+              readOnly: true
           ports:
-            - containerPort: 60000
+            - containerPort: 8443
               name: metrics
           command:
             - kueue-operator
@@ -54,3 +57,7 @@ spec:
       volumes:
         - name: tmp
           emptyDir: {}
+        - name: metrics-tls
+          secret:
+            secretName: openshift-kueue-operator-metrics-tls
+            optional: true

deploy/08_service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: openshift-kueue-operator-metrics
+  namespace: openshift-kueue-operator
+  labels:
+    app.openshift.io/component: operator
+    app.openshift.io/name: kueue
+spec:
+  ports:
+    - name: metrics
+      port: 60000
+      protocol: TCP
+      targetPort: 8443
+  selector:
+    name: openshift-kueue-operator
+  type: ClusterIP