From d418ed29433bd503951e61c05b3d467f3d78cc0f Mon Sep 17 00:00:00 2001 From: Neelesh Karthikeyan Date: Sun, 25 Jan 2026 03:57:08 +0000 Subject: [PATCH] Add Kubernetes manifests for CKN deployment Add K8s deployment manifests for the following services: - broker: Kafka broker in KRaft mode - kafka-connect: Kafka Connect for Neo4j integration - ckn-dashboard: Analytics dashboard - oracle-alert-processor: Camera traps alert processor - oracle-aggr-processor: Camera traps aggregation processor Includes: - Namespace, ConfigMap, and Secrets configuration - Init containers for service dependency ordering - Health checks (readiness/liveness probes) - Resource requests and limits - Kustomization for single-command deployment --- k8s/broker.yaml | 102 +++++++++++++++++++++++++ k8s/ckn-dashboard.yaml | 86 +++++++++++++++++++++ k8s/configmap.yaml | 19 +++++ k8s/kafka-connect.yaml | 129 ++++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 16 ++++ k8s/namespace.yaml | 6 ++ k8s/oracle-aggr-processor.yaml | 50 +++++++++++++ k8s/oracle-alert-processor.yaml | 50 +++++++++++++ 8 files changed, 458 insertions(+) create mode 100644 k8s/broker.yaml create mode 100644 k8s/ckn-dashboard.yaml create mode 100644 k8s/configmap.yaml create mode 100644 k8s/kafka-connect.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/namespace.yaml create mode 100644 k8s/oracle-aggr-processor.yaml create mode 100644 k8s/oracle-alert-processor.yaml diff --git a/k8s/broker.yaml b/k8s/broker.yaml new file mode 100644 index 0000000..6ac9a35 --- /dev/null +++ b/k8s/broker.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Service +metadata: + name: broker + namespace: ckn + labels: + app: broker +spec: + type: ClusterIP + ports: + - name: kafka + port: 29092 + targetPort: 29092 + - name: controller + port: 29093 + targetPort: 29093 + selector: + app: broker +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: broker + namespace: ckn + labels: + app: broker +spec: + replicas: 1 + selector: + matchLabels: + app: broker + template: + metadata: + labels: + app: broker + spec: + containers: + - name: broker + image: confluentinc/cp-kafka:7.4.0 + ports: + - containerPort: 29092 + name: kafka + - containerPort: 29093 + name: controller + env: + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_PROCESS_ROLES + value: "broker,controller" + - name: KAFKA_CONTROLLER_QUORUM_VOTERS + value: "1@broker:29093" + - name: CLUSTER_ID + value: "MkU3OEVBNTcwNTJENDM2Qk" + - name: KAFKA_LISTENERS + value: "PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:29093" + - name: KAFKA_ADVERTISED_LISTENERS + value: "PLAINTEXT://broker:29092" + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - name: KAFKA_CONTROLLER_LISTENER_NAMES + value: "CONTROLLER" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "PLAINTEXT" + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" + - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: "1" + - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR + value: "1" + - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS + value: "0" + - name: KAFKA_NODE_ID + value: "1" + - name: KAFKA_LOG_DIRS + value: "/tmp/kraft-combined-logs" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + readinessProbe: + exec: + command: + - kafka-topics + - --bootstrap-server + - localhost:29092 + - --list + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - kafka-topics + - --bootstrap-server + - localhost:29092 + - --list + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 diff --git a/k8s/ckn-dashboard.yaml b/k8s/ckn-dashboard.yaml new file mode 100644 index 0000000..b1cc35d --- /dev/null +++ b/k8s/ckn-dashboard.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: Service +metadata: + name: ckn-dashboard + namespace: ckn + labels: + app: ckn-dashboard +spec: + type: ClusterIP + ports: + - name: http + port: 8502 + targetPort: 8502 + selector: + app: ckn-dashboard +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ckn-dashboard + namespace: ckn + labels: + app: ckn-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: ckn-dashboard + template: + metadata: + labels: + app: ckn-dashboard + spec: + containers: + - name: ckn-dashboard + image: iud2i/ckn-analytics-dashboard:latest + ports: + - containerPort: 8502 + name: http + env: + - name: NEO4J_URI + valueFrom: + configMapKeyRef: + name: ckn-config + key: neo4j-uri + - name: NEO4J_USER + valueFrom: + secretKeyRef: + name: ckn-secrets + key: neo4j-user + - name: NEO4J_PWD + valueFrom: + secretKeyRef: + name: ckn-secrets + key: neo4j-password + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: ckn-secrets + key: openai-api-key + optional: true + - name: PATRA_SERVER + valueFrom: + configMapKeyRef: + name: ckn-config + key: patra-server + optional: true + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + readinessProbe: + httpGet: + path: / + port: 8502 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: 8502 + initialDelaySeconds: 30 + periodSeconds: 30 diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 0000000..f005938 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ckn-config + namespace: ckn +data: + neo4j-uri: "bolt://neo4j:7687" + patra-server: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ckn-secrets + namespace: ckn +type: Opaque +stringData: + neo4j-user: "neo4j" + neo4j-password: "PWD_HERE" # Change this in production! + openai-api-key: "" diff --git a/k8s/kafka-connect.yaml b/k8s/kafka-connect.yaml new file mode 100644 index 0000000..91a4ae5 --- /dev/null +++ b/k8s/kafka-connect.yaml @@ -0,0 +1,129 @@ +apiVersion: v1 +kind: Service +metadata: + name: kafka-connect + namespace: ckn + labels: + app: kafka-connect +spec: + type: ClusterIP + ports: + - name: rest + port: 8083 + targetPort: 8083 + selector: + app: kafka-connect +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-connect-scripts + namespace: ckn +data: + setup_connector.sh: | + #!/bin/bash + set -e + + echo "Starting Kafka Connect..." + /etc/confluent/docker/run & + + echo "Waiting for Kafka Connect to be ready..." + while ! curl -s http://localhost:8083/connectors > /dev/null; do + echo "Waiting for Kafka Connect REST API..." + sleep 5 + done + + echo "Kafka Connect is ready!" + + # Keep container running + wait +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-connect + namespace: ckn + labels: + app: kafka-connect +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-connect + template: + metadata: + labels: + app: kafka-connect + spec: + initContainers: + - name: wait-for-broker + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z broker 29092; do + echo "Waiting for Kafka broker..." + sleep 5 + done + echo "Kafka broker is ready!" + containers: + - name: kafka-connect + image: confluentinc/cp-kafka-connect:7.4.0 + ports: + - containerPort: 8083 + name: rest + env: + - name: CONNECT_BOOTSTRAP_SERVERS + value: "broker:29092" + - name: CONNECT_REST_PORT + value: "8083" + - name: CONNECT_GROUP_ID + value: "ckn-connect-group" + - name: CONNECT_CONFIG_STORAGE_TOPIC + value: "ckn-connect-configs" + - name: CONNECT_OFFSET_STORAGE_TOPIC + value: "ckn-connect-offsets" + - name: CONNECT_STATUS_STORAGE_TOPIC + value: "ckn-connect-status" + - name: CONNECT_KEY_CONVERTER + value: "org.apache.kafka.connect.storage.StringConverter" + - name: CONNECT_VALUE_CONVERTER + value: "org.apache.kafka.connect.json.JsonConverter" + - name: CONNECT_INTERNAL_KEY_CONVERTER + value: "org.apache.kafka.connect.json.JsonConverter" + - name: CONNECT_INTERNAL_VALUE_CONVERTER + value: "org.apache.kafka.connect.json.JsonConverter" + - name: CONNECT_REST_ADVERTISED_HOST_NAME + value: "kafka-connect" + - name: CONNECT_LOG4J_ROOT_LOGLEVEL + value: "INFO" + - name: CONNECT_LOG4J_LOGGERS + value: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" + - name: CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR + value: "1" + - name: CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR + value: "1" + - name: CONNECT_STATUS_STORAGE_REPLICATION_FACTOR + value: "1" + - name: CONNECT_PLUGIN_PATH + value: "/usr/share/java,/usr/share/confluent-hub-components" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + readinessProbe: + httpGet: + path: /connectors + port: 8083 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /connectors + port: 8083 + initialDelaySeconds: 60 + periodSeconds: 30 diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000..7d1897b --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ckn + +resources: + - namespace.yaml + - configmap.yaml + - broker.yaml + - kafka-connect.yaml + - ckn-dashboard.yaml + - oracle-alert-processor.yaml + - oracle-aggr-processor.yaml + +commonLabels: + app.kubernetes.io/part-of: cyberinfrastructure-knowledge-network diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..8305778 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ckn + labels: + app.kubernetes.io/name: cyberinfrastructure-knowledge-network diff --git a/k8s/oracle-aggr-processor.yaml b/k8s/oracle-aggr-processor.yaml new file mode 100644 index 0000000..322ff32 --- /dev/null +++ b/k8s/oracle-aggr-processor.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oracle-aggr-processor + namespace: ckn + labels: + app: oracle-aggr-processor +spec: + replicas: 1 + selector: + matchLabels: + app: oracle-aggr-processor + template: + metadata: + labels: + app: oracle-aggr-processor + spec: + initContainers: + - name: wait-for-broker + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z broker 29092; do + echo "Waiting for Kafka broker..." + sleep 5 + done + echo "Kafka broker is ready!" + containers: + - name: oracle-aggr-processor + image: iud2i/ckn-cameratraps-agg:latest + env: + - name: CKN_BROKERS + value: "broker:29092" + - name: CKN_ORACLE_WINDOW_TIME + value: "1" + - name: ORACLE_INPUT_TOPIC + value: "oracle-events" + - name: ORACLE_AGG_ALERT_TOPIC + value: "oracle-aggregated" + - name: APP_ID + value: "ckn-camera-traps-oracle-aggreg" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" diff --git a/k8s/oracle-alert-processor.yaml b/k8s/oracle-alert-processor.yaml new file mode 100644 index 0000000..562f122 --- /dev/null +++ b/k8s/oracle-alert-processor.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oracle-alert-processor + namespace: ckn + labels: + app: oracle-alert-processor +spec: + replicas: 1 + selector: + matchLabels: + app: oracle-alert-processor + template: + metadata: + labels: + app: oracle-alert-processor + spec: + initContainers: + - name: wait-for-broker + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z broker 29092; do + echo "Waiting for Kafka broker..." + sleep 5 + done + echo "Kafka broker is ready!" + containers: + - name: oracle-alert-processor + image: iud2i/ckn-cameratraps-acc-alert:latest + env: + - name: CKN_BROKERS + value: "broker:29092" + - name: ORACLE_ACC_CRITICAL_THRESHOLD + value: "0.5" + - name: ORACLE_INPUT_TOPIC + value: "oracle-events" + - name: ORACLE_ACC_ALERT_TOPIC + value: "oracle-alerts" + - name: APP_ID + value: "ckn-camera-traps-oracle-processor" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m"