diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a94620..cfa1ea1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,32 +36,58 @@ jobs: docker cp ckn_kg/constraints.cypher neo4j_server:/constraints.cypher docker exec neo4j_server cypher-shell -u neo4j -p PWD_HERE -f /constraints.cypher + - name: Wait for Kafka Connect and connectors + run: | + echo "Waiting for Kafka Connect REST API..." + while ! curl -s http://localhost:8083/connectors > /dev/null 2>&1; do \ + echo "Kafka Connect starting..."; \ + sleep 5; \ + done + echo "Kafka Connect is ready" + + echo "Waiting for Neo4j sink connector..." + for i in {1..30}; do + if curl -s http://localhost:8083/connectors | grep -q "Neo4jSinkConnectorCameraTraps"; then + echo "Neo4j sink connector is ready" + break + fi + echo "Waiting for connector... ($i/30)" + sleep 5 + done + - name: Build CKN Oracle Daemon plugin WITHOUT power_monitoring run: | - docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml up -d --build + # Use CI override to connect to local Kafka broker + docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml up -d --build sleep 5 + + # Trigger file modification to process events (file already has sample data) docker exec ckn-oracle-daemon touch /oracle_logs/image_mapping_final.json + + # Wait for data to flow through pipeline: daemon -> Kafka -> Neo4j + echo "Waiting for data ingestion..." + sleep 20 - name: Test CKN Oracle Daemon plugin WITHOUT power_monitoring run: | pip install -r plugins/oracle_ckn_daemon/tests/requirements.txt pytest plugins/oracle_ckn_daemon/tests/test_ckn_oracle_daemon.py -v pytest plugins/oracle_ckn_daemon/tests/test_power_monitoring_false.py -v - docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml down + docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml down - name: Build CKN Oracle Daemon plugin WITH power_monitoring run: | - sed -i '20s/ENABLE_POWER_MONITORING=false/ENABLE_POWER_MONITORING=true/' plugins/oracle_ckn_daemon/docker-compose.yml + sed -i 's/ENABLE_POWER_MONITORING=false/ENABLE_POWER_MONITORING=true/' plugins/oracle_ckn_daemon/docker-compose.yml - docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml up -d --build + docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml up -d --build sleep 5 docker exec ckn-oracle-daemon touch /oracle_logs/image_mapping_final.json - sleep 5 + sleep 10 - name: Test CKN Oracle Daemon plugin WITH power_monitoring run: | pytest plugins/oracle_ckn_daemon/tests/test_power_monitoring_true.py -v - docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml down + docker compose -f plugins/oracle_ckn_daemon/docker-compose.yml -f plugins/oracle_ckn_daemon/docker-compose.ci.yml down - name: Stop and Remove CKN Docker Network diff --git a/plugins/oracle_ckn_daemon/docker-compose.ci.yml b/plugins/oracle_ckn_daemon/docker-compose.ci.yml new file mode 100644 index 0000000..701c7ab --- /dev/null +++ b/plugins/oracle_ckn_daemon/docker-compose.ci.yml @@ -0,0 +1,11 @@ +services: + ckn-daemon: + environment: + - CKN_KAFKA_BROKER=broker:29092 + - CKN_KAFKA_SECURITY_PROTOCOL=PLAINTEXT + networks: + - ckn-network + +networks: + ckn-network: + external: true diff --git a/plugins/oracle_ckn_daemon/oracle_daemon.py b/plugins/oracle_ckn_daemon/oracle_daemon.py index 4ce956a..cbddc73 100644 --- a/plugins/oracle_ckn_daemon/oracle_daemon.py +++ b/plugins/oracle_ckn_daemon/oracle_daemon.py @@ -23,6 +23,7 @@ POWER_SUMMARY_TIMOUT = os.getenv('POWER_SUMMARY_TIMOUT', 10) POWER_SUMMARY_MAX_TRIES = os.getenv('POWER_SUMMARY_TIMOUT', 5) ENABLE_POWER_MONITORING = os.getenv('ENABLE_POWER_MONITORING', "false") +KAFKA_SECURITY_PROTOCOL = os.getenv('CKN_KAFKA_SECURITY_PROTOCOL', 'SSL') class OracleEventHandler(FileSystemEventHandler): @@ -437,7 +438,7 @@ def test_ckn_broker_connection(configuration, timeout=10, num_tries=5): time.sleep(1) # Configure Kafka producer. - kafka_conf = {'bootstrap.servers': KAFKA_BROKER, 'log_level': 0, 'security.protocol': 'SSL'} + kafka_conf = {'bootstrap.servers': KAFKA_BROKER, 'log_level': 0, 'security.protocol': KAFKA_SECURITY_PROTOCOL} logging.info("Connecting to the CKN broker at %s", KAFKA_BROKER)