Add Kubernetes manifests for CKN deployment

k8s/broker.yaml

@@ -0,0 +1,102 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: broker
+  namespace: ckn
+  labels:
+    app: broker
+spec:
+  type: ClusterIP
+  ports:
+    - name: kafka
+      port: 29092
+      targetPort: 29092
+    - name: controller
+      port: 29093
+      targetPort: 29093
+  selector:
+    app: broker
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: broker
+  namespace: ckn
+  labels:
+    app: broker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: broker
+  template:
+    metadata:
+      labels:
+        app: broker
+    spec:
+      containers:
+        - name: broker
+          image: confluentinc/cp-kafka:7.4.0
+          ports:
+            - containerPort: 29092
+              name: kafka
+            - containerPort: 29093
+              name: controller
+          env:
+            - name: KAFKA_BROKER_ID
+              value: "1"
+            - name: KAFKA_PROCESS_ROLES
+              value: "broker,controller"
+            - name: KAFKA_CONTROLLER_QUORUM_VOTERS
+              value: "1@broker:29093"
+            - name: CLUSTER_ID
+              value: "MkU3OEVBNTcwNTJENDM2Qk"
+            - name: KAFKA_LISTENERS
+              value: "PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:29093"
+            - name: KAFKA_ADVERTISED_LISTENERS
+              value: "PLAINTEXT://broker:29092"
+            - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
+              value: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+            - name: KAFKA_CONTROLLER_LISTENER_NAMES
+              value: "CONTROLLER"
+            - name: KAFKA_INTER_BROKER_LISTENER_NAME
+              value: "PLAINTEXT"
+            - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
+              value: "1"
+            - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
+              value: "1"
+            - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
+              value: "1"
+            - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS
+              value: "0"
+            - name: KAFKA_NODE_ID
+              value: "1"
+            - name: KAFKA_LOG_DIRS
+              value: "/tmp/kraft-combined-logs"
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "250m"
+            limits:
+              memory: "2Gi"
+              cpu: "1000m"
+          readinessProbe:
+            exec:
+              command:
+                - kafka-topics
+                - --bootstrap-server
+                - localhost:29092
+                - --list
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command:
+                - kafka-topics
+                - --bootstrap-server
+                - localhost:29092
+                - --list
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 10

k8s/ckn-dashboard.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ckn-dashboard
+  namespace: ckn
+  labels:
+    app: ckn-dashboard
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: 8502
+      targetPort: 8502
+  selector:
+    app: ckn-dashboard
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ckn-dashboard
+  namespace: ckn
+  labels:
+    app: ckn-dashboard
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ckn-dashboard
+  template:
+    metadata:
+      labels:
+        app: ckn-dashboard
+    spec:
+      containers:
+        - name: ckn-dashboard
+          image: iud2i/ckn-analytics-dashboard:latest
+          ports:
+            - containerPort: 8502
+              name: http
+          env:
+            - name: NEO4J_URI
+              valueFrom:
+                configMapKeyRef:
+                  name: ckn-config
+                  key: neo4j-uri
+            - name: NEO4J_USER
+              valueFrom:
+                secretKeyRef:
+                  name: ckn-secrets
+                  key: neo4j-user
+            - name: NEO4J_PWD
+              valueFrom:
+                secretKeyRef:
+                  name: ckn-secrets
+                  key: neo4j-password
+            - name: OPENAI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: ckn-secrets
+                  key: openai-api-key
+                  optional: true
+            - name: PATRA_SERVER
+              valueFrom:
+                configMapKeyRef:
+                  name: ckn-config
+                  key: patra-server
+                  optional: true
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+              cpu: "500m"
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8502
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 8502
+            initialDelaySeconds: 30
+            periodSeconds: 30

k8s/configmap.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ckn-config
+  namespace: ckn
+data:
+  neo4j-uri: "bolt://neo4j:7687"
+  patra-server: ""
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ckn-secrets
+  namespace: ckn
+type: Opaque
+stringData:
+  neo4j-user: "neo4j"
+  neo4j-password: "PWD_HERE"  # Change this in production!
+  openai-api-key: ""

k8s/kafka-connect.yaml

@@ -0,0 +1,129 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-connect
+  namespace: ckn
+  labels:
+    app: kafka-connect
+spec:
+  type: ClusterIP
+  ports:
+    - name: rest
+      port: 8083
+      targetPort: 8083
+  selector:
+    app: kafka-connect
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-connect-scripts
+  namespace: ckn
+data:
+  setup_connector.sh: |
+    #!/bin/bash
+    set -e
+
+    echo "Starting Kafka Connect..."
+    /etc/confluent/docker/run &
+
+    echo "Waiting for Kafka Connect to be ready..."
+    while ! curl -s http://localhost:8083/connectors > /dev/null; do
+      echo "Waiting for Kafka Connect REST API..."
+      sleep 5
+    done
+
+    echo "Kafka Connect is ready!"
+
+    # Keep container running
+    wait
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-connect
+  namespace: ckn
+  labels:
+    app: kafka-connect
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-connect
+  template:
+    metadata:
+      labels:
+        app: kafka-connect
+    spec:
+      initContainers:
+        - name: wait-for-broker
+          image: busybox:1.36
+          command:
+            - sh
+            - -c
+            - |
+              until nc -z broker 29092; do
+                echo "Waiting for Kafka broker..."
+                sleep 5
+              done
+              echo "Kafka broker is ready!"
+      containers:
+        - name: kafka-connect
+          image: confluentinc/cp-kafka-connect:7.4.0
+          ports:
+            - containerPort: 8083
+              name: rest
+          env:
+            - name: CONNECT_BOOTSTRAP_SERVERS
+              value: "broker:29092"
+            - name: CONNECT_REST_PORT
+              value: "8083"
+            - name: CONNECT_GROUP_ID
+              value: "ckn-connect-group"
+            - name: CONNECT_CONFIG_STORAGE_TOPIC
+              value: "ckn-connect-configs"
+            - name: CONNECT_OFFSET_STORAGE_TOPIC
+              value: "ckn-connect-offsets"
+            - name: CONNECT_STATUS_STORAGE_TOPIC
+              value: "ckn-connect-status"
+            - name: CONNECT_KEY_CONVERTER
+              value: "org.apache.kafka.connect.storage.StringConverter"
+            - name: CONNECT_VALUE_CONVERTER
+              value: "org.apache.kafka.connect.json.JsonConverter"
+            - name: CONNECT_INTERNAL_KEY_CONVERTER
+              value: "org.apache.kafka.connect.json.JsonConverter"
+            - name: CONNECT_INTERNAL_VALUE_CONVERTER
+              value: "org.apache.kafka.connect.json.JsonConverter"
+            - name: CONNECT_REST_ADVERTISED_HOST_NAME
+              value: "kafka-connect"
+            - name: CONNECT_LOG4J_ROOT_LOGLEVEL
+              value: "INFO"
+            - name: CONNECT_LOG4J_LOGGERS
+              value: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
+            - name: CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR
+              value: "1"
+            - name: CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR
+              value: "1"
+            - name: CONNECT_STATUS_STORAGE_REPLICATION_FACTOR
+              value: "1"
+            - name: CONNECT_PLUGIN_PATH
+              value: "/usr/share/java,/usr/share/confluent-hub-components"
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "250m"
+            limits:
+              memory: "2Gi"
+              cpu: "1000m"
+          readinessProbe:
+            httpGet:
+              path: /connectors
+              port: 8083
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /connectors
+              port: 8083
+            initialDelaySeconds: 60
+            periodSeconds: 30

k8s/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ckn
+
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - broker.yaml
+  - kafka-connect.yaml
+  - ckn-dashboard.yaml
+  - oracle-alert-processor.yaml
+  - oracle-aggr-processor.yaml
+
+commonLabels:
+  app.kubernetes.io/part-of: cyberinfrastructure-knowledge-network

k8s/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ckn
+  labels:
+    app.kubernetes.io/name: cyberinfrastructure-knowledge-network