Fix CI test failures by properly configuring data pipeline

.github/workflows/ci.yml

@@ -36,32 +36,58 @@ jobs:
           docker cp ckn_kg/constraints.cypher neo4j_server:/constraints.cypher
           docker exec neo4j_server cypher-shell -u neo4j -p PWD_HERE -f /constraints.cypher
 
+      - name: Wait for Kafka Connect and connectors
+        run: |
+          echo "Waiting for Kafka Connect REST API..."
+          while ! curl -s http://localhost:8083/connectors > /dev/null 2>&1; do \
+            echo "Kafka Connect starting..."; \
+            sleep 5; \
+          done
+          echo "Kafka Connect is ready"
+
+          echo "Waiting for Neo4j sink connector..."
+          for i in {1..30}; do
+            if curl -s http://localhost:8083/connectors | grep -q "Neo4jSinkConnectorCameraTraps"; then
+              echo "Neo4j sink connector is ready"
+              break
+            fi
+            echo "Waiting for connector... ($i/30)"
+            sleep 5
+          done
+
       - name: Build CKN Oracle Daemon plugin WITHOUT power_monitoring
         run: |
-          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml up -d --build
+          # Use CI override to connect to local Kafka broker
+          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml up -d --build
           sleep 5
+
+          # Trigger file modification to process events (file already has sample data)
           docker exec ckn-oracle-daemon touch /oracle_logs/image_mapping_final.json
+
+          # Wait for data to flow through pipeline: daemon -> Kafka -> Neo4j
+          echo "Waiting for data ingestion..."
+          sleep 20
 
       - name: Test CKN Oracle Daemon plugin WITHOUT power_monitoring
         run: |
           pip install -r plugins/oracle_ckn_daemon/tests/requirements.txt
           pytest plugins/oracle_ckn_daemon/tests/test_ckn_oracle_daemon.py -v
           pytest plugins/oracle_ckn_daemon/tests/test_power_monitoring_false.py -v
-          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml down
+          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml down
 
       - name: Build CKN Oracle Daemon plugin WITH power_monitoring
         run: |
-          sed -i '20s/ENABLE_POWER_MONITORING=false/ENABLE_POWER_MONITORING=true/' plugins/oracle_ckn_daemon/docker-compose.yml
+          sed -i 's/ENABLE_POWER_MONITORING=false/ENABLE_POWER_MONITORING=true/' plugins/oracle_ckn_daemon/docker-compose.yml
 
-          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml up -d --build
+          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml up -d --build
           sleep 5
           docker exec ckn-oracle-daemon touch /oracle_logs/image_mapping_final.json
-          sleep 5
+          sleep 10
 
       - name: Test CKN Oracle Daemon plugin WITH power_monitoring
         run: |
           pytest plugins/oracle_ckn_daemon/tests/test_power_monitoring_true.py -v
-          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml down
+          docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml down
 
 
       - name: Stop and Remove CKN Docker Network

plugins/oracle_ckn_daemon/docker-compose.ci.yml

@@ -0,0 +1,11 @@
+services:
+  ckn-daemon:
+    environment:
+      - CKN_KAFKA_BROKER=broker:29092
+      - CKN_KAFKA_SECURITY_PROTOCOL=PLAINTEXT
+    networks:
+      - ckn-network
+
+networks:
+  ckn-network:
+    external: true

plugins/oracle_ckn_daemon/oracle_daemon.py

@@ -23,6 +23,7 @@
 POWER_SUMMARY_TIMOUT = os.getenv('POWER_SUMMARY_TIMOUT', 10)
 POWER_SUMMARY_MAX_TRIES = os.getenv('POWER_SUMMARY_TIMOUT', 5)
 ENABLE_POWER_MONITORING = os.getenv('ENABLE_POWER_MONITORING', "false")
+KAFKA_SECURITY_PROTOCOL = os.getenv('CKN_KAFKA_SECURITY_PROTOCOL', 'SSL')
 
 
 class OracleEventHandler(FileSystemEventHandler):
@@ -437,7 +438,7 @@ def test_ckn_broker_connection(configuration, timeout=10, num_tries=5):
         time.sleep(1)
 
     # Configure Kafka producer.
-    kafka_conf = {'bootstrap.servers': KAFKA_BROKER, 'log_level': 0, 'security.protocol': 'SSL'}
+    kafka_conf = {'bootstrap.servers': KAFKA_BROKER, 'log_level': 0, 'security.protocol': KAFKA_SECURITY_PROTOCOL}
 
     logging.info("Connecting to the CKN broker at %s", KAFKA_BROKER)