From 37020866d06b8a6906f4e5758b3d5a3f3a26f643 Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:38:28 -0700 Subject: [PATCH 1/9] feat: optimize dashboard to use stored metrics from Experiment node - Add get_experiment_metrics() method to retrieve stored metrics from Neo4j - Replace computed metrics with database-stored metrics in Camera Traps dashboard - Remove complex metric calculation functions (calculate_object_detection_metrics, etc.) - Add support for mAP, IoU, and other advanced metrics from oracle daemon - Update docker-compose.yml to use iud2i/ckn-analytics-dashboard:2.0.0 - Improve performance by eliminating real-time metric calculations - Ensure consistency between oracle daemon calculations and dashboard display --- ckn_dashboard/ckn_kg.py | 42 +++ ckn_dashboard/pages/1_Camera_Traps.py | 486 ++++++-------------------- docker-compose.yml | 4 +- 3 files changed, 148 insertions(+), 384 deletions(-) diff --git a/ckn_dashboard/ckn_kg.py b/ckn_dashboard/ckn_kg.py index 593dca1..6ccf4a9 100644 --- a/ckn_dashboard/ckn_kg.py +++ b/ckn_dashboard/ckn_kg.py @@ -575,6 +575,48 @@ def convert_to_datetime(self, neo4j_datetime): int(neo4j_datetime.nanosecond / 1000), tzinfo=neo4j_datetime.tzinfo) + def get_experiment_metrics(self, experiment_id): + """ + Get experiment metrics directly from the Experiment node. + Returns the stored metrics like f1_score, precision, recall, etc. + """ + query = """ + MATCH (e:Experiment {experiment_id: '""" + experiment_id + """'}) + RETURN e.f1_score as f1_score, + e.precision as precision, + e.recall as recall, + e.false_positives as false_positives, + e.false_negatives as false_negatives, + e.true_positives as true_positives, + e.total_ground_truth_objects as total_ground_truth_objects, + e.total_predictions as total_predictions, + e.total_images as total_images, + e.mean_iou as mean_iou, + e.map_50 as map_50, + e.map_50_95 as map_50_95 + """ + + result = self.session.run(query) + record = result.single() + + if record: + return { + "f1_score": record["f1_score"], + "precision": record["precision"], + "recall": record["recall"], + "false_positives": record["false_positives"], + "false_negatives": record["false_negatives"], + "true_positives": record["true_positives"], + "total_ground_truth_objects": record["total_ground_truth_objects"], + "total_predictions": record["total_predictions"], + "total_images": record["total_images"], + "mean_iou": record["mean_iou"], + "map_50": record["map_50"], + "map_50_95": record["map_50_95"] + } + else: + return None + def convert_to_native(self, dt): """Convert Neo4j DateTime to Python native datetime""" if isinstance(dt, neo4j.time.DateTime): diff --git a/ckn_dashboard/pages/1_Camera_Traps.py b/ckn_dashboard/pages/1_Camera_Traps.py index 03bb495..d0a4c3f 100644 --- a/ckn_dashboard/pages/1_Camera_Traps.py +++ b/ckn_dashboard/pages/1_Camera_Traps.py @@ -71,286 +71,6 @@ def get_experiment_indicators(experiment_id, experiment_df, model_id): return date_str, time_str, model_name, device_info, average_accuracy -def calculate_accuracy_from_experiment(experiment_details): - """ - Calculate the accuracy of model predictions against ground truth labels. - """ - - total = 0 - correct = 0 - missing_ground_truth = 0 - missing_or_invalid_scores = 0 - - for index, row in experiment_details.iterrows(): - ground_truth = row.get("Ground Truth") - scores_str = row.get("Scores", "") - - # Check for missing Ground Truth - if pd.isna(ground_truth): - missing_ground_truth += 1 - continue - - # Check for missing or empty Scores - if pd.isna(scores_str) or not isinstance(scores_str, str) or not scores_str.strip(): - missing_or_invalid_scores += 1 - continue - - try: - # Parse the JSON string in Scores - scores = json.loads(scores_str) - except json.JSONDecodeError: - missing_or_invalid_scores += 1 - continue - - # Aggregate probabilities per label - label_prob = {} - for entry in scores: - label = entry.get("label") - probability = entry.get("probability", 0) - if label: - label_prob[label] = label_prob.get(label, 0) + probability - - if not label_prob: - missing_or_invalid_scores += 1 - continue - - # Determine the predicted label with the highest aggregated probability - predicted_label = max(label_prob, key=label_prob.get) - - # Update total and correct counts - total += 1 - if predicted_label.lower() == ground_truth.lower(): - correct += 1 - - # Calculate accuracy - accuracy = (correct / total) * 100 if total > 0 else 0 - - return accuracy - - -def calculate_iou(box1, box2): - """ - Calculate Intersection over Union (IoU) between two bounding boxes. - Box format: [x1, y1, x2, y2] where (x1,y1) is top-left and (x2,y2) is bottom-right. - """ - # Calculate intersection coordinates - x1 = max(box1[0], box2[0]) - y1 = max(box1[1], box2[1]) - x2 = min(box1[2], box2[2]) - y2 = min(box1[3], box2[3]) - - # Calculate intersection area - intersection = max(0, x2 - x1) * max(0, y2 - y1) - - # Calculate union area - area1 = (box1[2] - box1[0]) * (box1[3] - box1[1]) - area2 = (box2[2] - box2[0]) * (box2[3] - box2[1]) - union = area1 + area2 - intersection - - return intersection / union if union > 0 else 0 - - -def calculate_object_detection_metrics(experiment_details): - """ - Calculate comprehensive object detection metrics for camera traps experiments. - Returns both dataset-level and image-level classification metrics. - """ - - # Initialize counters for dataset metrics - total_images = 0 - total_ground_truth_objects = 0 - total_predictions = 0 - true_positives = 0 - false_positives = 0 - false_negatives = 0 - iou_scores = [] - - # Initialize counters for image classification metrics - correct_predictions = 0 - incorrect_predictions = 0 - - # Debug: Print first few rows to understand data structure (only in debug mode) - debug_mode = False # Set to True to see debug output - if debug_mode: - print("DEBUG: First few rows of experiment_details:") - for idx, row in experiment_details.head(3).iterrows(): - print(f"Row {idx}:") - print(f" Ground Truth: {row.get('Ground Truth')}") - print(f" Scores: {row.get('Scores')[:200]}...") # First 200 chars - print(f" Ground Truth BBox: {row.get('Ground Truth BBox', 'Not available')}") - print() - - for index, row in experiment_details.iterrows(): - ground_truth = row.get("Ground Truth") - scores_str = row.get("Scores", "") - ground_truth_bbox = row.get("Ground Truth BBox", None) - - # Skip if missing ground truth or scores - if pd.isna(ground_truth) or pd.isna(scores_str) or not isinstance(scores_str, str): - continue - - try: - scores = json.loads(scores_str) - except json.JSONDecodeError: - continue - - total_images += 1 - - # Parse predictions from scores - predictions = [] - if debug_mode: - print(f"DEBUG: Processing {len(scores)} scores for image {total_images + 1}") - - for i, entry in enumerate(scores): - label = entry.get("label") - probability = entry.get("probability", 0) - bbox = entry.get("bbox", []) # [x1, y1, x2, y2] format - - if debug_mode: - print(f" Score {i}: label='{label}', probability={probability}, bbox={bbox}") - - # Accept predictions with valid labels (bbox is optional) - if label: - # If bbox is missing or invalid, use placeholder - if not isinstance(bbox, list) or len(bbox) != 4: - bbox = [0, 0, 100, 100] # Placeholder bbox - - predictions.append({ - "label": label, - "probability": probability, - "bbox": bbox - }) - if debug_mode: - print(f" -> Added to predictions") - else: - if debug_mode: - print(f" -> Skipped (no valid label)") - - if debug_mode: - print(f"DEBUG: Found {len(predictions)} valid predictions") - - # Sort predictions by probability (highest first) - predictions.sort(key=lambda x: x["probability"], reverse=True) - - # For image classification, use the highest confidence prediction - if predictions: - best_prediction = predictions[0] - predicted_label = best_prediction["label"] - - # Compare with ground truth for image classification - if predicted_label.lower() == ground_truth.lower(): - correct_predictions += 1 - else: - incorrect_predictions += 1 - - # For object detection metrics, parse ground truth bounding boxes - gt_objects = [] - if ground_truth and ground_truth.lower() != "empty": - if ground_truth_bbox and not pd.isna(ground_truth_bbox): - try: - # Try to parse ground truth bounding box if available - if isinstance(ground_truth_bbox, str): - gt_bbox = json.loads(ground_truth_bbox) - else: - gt_bbox = ground_truth_bbox - - if isinstance(gt_bbox, list) and len(gt_bbox) == 4: - gt_objects.append({ - "label": ground_truth, - "bbox": gt_bbox - }) - else: - # Fallback to placeholder bbox - gt_objects.append({ - "label": ground_truth, - "bbox": [0, 0, 100, 100] - }) - except (json.JSONDecodeError, TypeError): - # Fallback to placeholder bbox - gt_objects.append({ - "label": ground_truth, - "bbox": [0, 0, 100, 100] - }) - else: - # No bounding box data available, use placeholder - gt_objects.append({ - "label": ground_truth, - "bbox": [0, 0, 100, 100] - }) - - total_ground_truth_objects += len(gt_objects) - - # Match predictions to ground truth objects using IoU - matched_gt = set() - matched_pred = set() - - for pred_idx, prediction in enumerate(predictions): - best_iou = 0 - best_gt_idx = -1 - - for gt_idx, gt_object in enumerate(gt_objects): - if gt_idx in matched_gt: - continue - - if prediction["label"].lower() == gt_object["label"].lower(): - iou = calculate_iou(prediction["bbox"], gt_object["bbox"]) - if iou > best_iou and iou > 0.5: # IoU threshold - best_iou = iou - best_gt_idx = gt_idx - - if best_gt_idx >= 0: - true_positives += 1 - matched_gt.add(best_gt_idx) - matched_pred.add(pred_idx) - iou_scores.append(best_iou) - else: - false_positives += 1 - - # Count unmatched ground truth objects as false negatives - false_negatives += len(gt_objects) - len(matched_gt) - - # Total predictions is the sum of predictions with valid bounding boxes - total_predictions += len(predictions) - - # Calculate dataset metrics - precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0 - recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0 - f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 - mean_iou = sum(iou_scores) / len(iou_scores) if iou_scores else 0 - - # Calculate mAP (simplified - in practice you'd need per-class AP calculations) - map_50 = precision # Simplified mAP@0.5 - map_75 = precision * 0.9 # Simplified mAP@0.75 (assume 90% of mAP@0.5) - map_50_95 = (map_50 + map_75) / 2 # Simplified mAP@[0.5:0.95] - - # Calculate image classification accuracy - image_accuracy = correct_predictions / total_images if total_images > 0 else 0 - - dataset_metrics = { - "total_images": total_images, - "total_ground_truth_objects": total_ground_truth_objects, - "total_predictions": total_predictions, - "true_positives": true_positives, - "false_positives": false_positives, - "false_negatives": false_negatives, - "mean_iou": round(mean_iou, 3), - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1_score": round(f1_score, 4), - "average_precision": round(map_50, 3), - "map_50": round(map_50, 3), - "map_75": round(map_75, 3), - "map_50_95": round(map_50_95, 3) - } - - image_classification_metrics = { - "total_images": total_images, - "correct_predictions": correct_predictions, - "incorrect_predictions": incorrect_predictions, - "accuracy": round(image_accuracy, 2) - } - - return dataset_metrics, image_classification_metrics def get_power_info(deployment_info): @@ -422,8 +142,8 @@ def get_power_info(deployment_info): except: experiment_info = kg.get_exp_info_raw(selected_experiment) - # Calculate object detection metrics - dataset_metrics, image_classification_metrics = calculate_object_detection_metrics(experiment_info) + # Get stored metrics from Experiment node instead of computing them + stored_metrics = kg.get_experiment_metrics(selected_experiment) # extracting model id and drop model_id = experiment_info['Model'].iloc[0] @@ -450,152 +170,156 @@ def get_power_info(deployment_info): # Object Detection Metrics Section # Performance Summary with Progress Bars - if dataset_metrics["total_predictions"] > 0: - overall_accuracy = (dataset_metrics["true_positives"] / dataset_metrics["total_predictions"]) * 100 + if stored_metrics and stored_metrics["total_predictions"] > 0: + overall_accuracy = (stored_metrics["true_positives"] / stored_metrics["total_predictions"]) * 100 col1, col2, col3 = st.columns(3) with col1: st.metric( label="Overall Detection Accuracy", value=f"{overall_accuracy:.1f}%", - delta=f"{dataset_metrics['true_positives']}/{dataset_metrics['total_predictions']} correct", + delta=f"{stored_metrics['true_positives']}/{stored_metrics['total_predictions']} correct", help="Percentage of correct predictions out of all predictions made by the model (True Positives / Total Predictions)" ) st.progress(overall_accuracy / 100) with col2: - if dataset_metrics["total_ground_truth_objects"] > 0: - detection_rate = (dataset_metrics["true_positives"] / dataset_metrics["total_ground_truth_objects"]) * 100 + if stored_metrics["total_ground_truth_objects"] > 0: + detection_rate = (stored_metrics["true_positives"] / stored_metrics["total_ground_truth_objects"]) * 100 st.metric( label="Detection Rate", value=f"{detection_rate:.1f}%", - delta=f"{dataset_metrics['true_positives']}/{dataset_metrics['total_ground_truth_objects']} detected", + delta=f"{stored_metrics['true_positives']}/{stored_metrics['total_ground_truth_objects']} detected", help="Percentage of ground truth objects that were successfully detected by the model (True Positives / Ground Truth Objects)" ) st.progress(detection_rate / 100) with col3: - if image_classification_metrics["total_images"] > 0: - classification_accuracy = image_classification_metrics["accuracy"] * 100 + if stored_metrics["total_images"] > 0: + # Calculate classification accuracy from stored metrics + correct_predictions = stored_metrics["true_positives"] # Assuming this represents correct classifications + classification_accuracy = (correct_predictions / stored_metrics["total_images"]) * 100 - # Single accuracy metric with progress bar st.metric( label="Classification Accuracy", value=f"{classification_accuracy:.1f}%", - delta=f"{image_classification_metrics['correct_predictions']}/{image_classification_metrics['total_images']} correct", - help="Percentage of images correctly classified by selecting the highest-confidence prediction per image" + delta=f"{correct_predictions}/{stored_metrics['total_images']} correct", + help="Percentage of images correctly classified based on stored metrics" ) st.progress(classification_accuracy / 100) else: - st.warning("No predictions found in the data") + st.warning("No stored metrics found in the database for this experiment") # Detection Metrics (Collapsible) - with st.expander("Detection Metrics", expanded=True): - # First row - Basic counts - col1, col2, col3 = st.columns(3) + if stored_metrics: + with st.expander("Detection Metrics", expanded=True): + # First row - Basic counts + col1, col2, col3 = st.columns(3) - with col1: - st.metric( - label="Total Images", - value=dataset_metrics["total_images"], - help="Number of images processed in this experiment" - ) + with col1: + st.metric( + label="Total Images", + value=stored_metrics["total_images"], + help="Number of images processed in this experiment" + ) - with col2: - st.metric( - label="Ground Truth Objects", - value=dataset_metrics["total_ground_truth_objects"], - help="Number of objects in ground truth annotations" - ) + with col2: + st.metric( + label="Ground Truth Objects", + value=stored_metrics["total_ground_truth_objects"], + help="Number of objects in ground truth annotations" + ) - with col3: - st.metric( - label="Total Predictions", - value=dataset_metrics["total_predictions"], - help="Total number of predictions made by the model" - ) + with col3: + st.metric( + label="Total Predictions", + value=stored_metrics["total_predictions"], + help="Total number of predictions made by the model" + ) - # Second row - Performance metrics - col1, col2, col3 = st.columns(3) + # Second row - Performance metrics + col1, col2, col3 = st.columns(3) - with col1: - st.metric( - label="True Positives", - value=dataset_metrics["true_positives"], - help="Correctly detected objects (IoU > 0.5)" - ) + with col1: + st.metric( + label="True Positives", + value=stored_metrics["true_positives"], + help="Correctly detected objects" + ) - with col2: - st.metric( - label="False Positives", - value=dataset_metrics["false_positives"], - help="Incorrect detections" - ) + with col2: + st.metric( + label="False Positives", + value=stored_metrics["false_positives"], + help="Incorrect detections" + ) - with col3: - st.metric( - label="False Negatives", - value=dataset_metrics["false_negatives"], - help="Missed ground truth objects" - ) + with col3: + st.metric( + label="False Negatives", + value=stored_metrics["false_negatives"], + help="Missed ground truth objects" + ) # Precision & Recall (Collapsible) - with st.expander("Precision & Recall", expanded=False): - col1, col2, col3 = st.columns(3) + if stored_metrics: + with st.expander("Precision & Recall", expanded=False): + col1, col2, col3 = st.columns(3) - with col1: - st.metric( - label="Precision", - value=f"{dataset_metrics['precision']:.4f}", - help="TP / (TP + FP) - Accuracy of positive predictions" - ) - st.progress(dataset_metrics['precision']) + with col1: + st.metric( + label="Precision", + value=f"{stored_metrics['precision']:.4f}", + help="TP / (TP + FP) - Accuracy of positive predictions" + ) + st.progress(stored_metrics['precision']) - with col2: - st.metric( - label="Recall", - value=f"{dataset_metrics['recall']:.4f}", - help="TP / (TP + FN) - Ability to find all positive instances" - ) - st.progress(dataset_metrics['recall']) + with col2: + st.metric( + label="Recall", + value=f"{stored_metrics['recall']:.4f}", + help="TP / (TP + FN) - Ability to find all positive instances" + ) + st.progress(stored_metrics['recall']) - with col3: - st.metric( - label="F1 Score", - value=f"{dataset_metrics['f1_score']:.4f}", - help="Harmonic mean of precision and recall" - ) - st.progress(dataset_metrics['f1_score']) + with col3: + st.metric( + label="F1 Score", + value=f"{stored_metrics['f1_score']:.4f}", + help="Harmonic mean of precision and recall" + ) + st.progress(stored_metrics['f1_score']) # mAP Metrics (Collapsible) - with st.expander("mAP Metrics", expanded=False): - col1, col2, col3 = st.columns(3) + if stored_metrics and stored_metrics.get('map_50') is not None: + with st.expander("mAP Metrics", expanded=False): + col1, col2, col3 = st.columns(3) - with col1: - st.metric( - label="mAP@0.5", - value=f"{dataset_metrics['map_50']:.3f}", - help="Mean Average Precision at IoU threshold 0.5" - ) - st.progress(dataset_metrics['map_50']) + with col1: + st.metric( + label="mAP@0.5", + value=f"{stored_metrics['map_50']:.3f}", + help="Mean Average Precision at IoU threshold 0.5" + ) + st.progress(stored_metrics['map_50']) - with col2: - st.metric( - label="mAP@0.75", - value=f"{dataset_metrics['map_75']:.3f}", - help="Mean Average Precision at IoU threshold 0.75" - ) - st.progress(dataset_metrics['map_75']) + with col2: + st.metric( + label="mAP@[0.5:0.95]", + value=f"{stored_metrics['map_50_95']:.3f}", + help="Mean Average Precision across IoU thresholds 0.5 to 0.95" + ) + st.progress(stored_metrics['map_50_95']) - with col3: - st.metric( - label="mAP@[0.5:0.95]", - value=f"{dataset_metrics['map_50_95']:.3f}", - help="Mean Average Precision across IoU thresholds 0.5 to 0.95" - ) - st.progress(dataset_metrics['map_50_95']) + with col3: + st.metric( + label="Mean IoU", + value=f"{stored_metrics['mean_iou']:.3f}", + help="Mean Intersection over Union" + ) + st.progress(stored_metrics['mean_iou']) # Image Classification Metrics st.markdown("---") diff --git a/docker-compose.yml b/docker-compose.yml index efcd3a1..6c7e579 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: neo4j: image: neo4j:5.21.0-community @@ -91,7 +89,7 @@ services: - ckn-network ckn-dashboard: - image: iud2i/ckn-analytics-dashboard:latest + image: iud2i/ckn-analytics-dashboard:2.0.0 container_name: dashboard ports: - "8502:8502" From 551f9eff93f0c68bfb71f34f1209d8118fa5b1af Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:44:46 -0700 Subject: [PATCH 2/9] chore: update docker-compose.yml to version 3.8 --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6c7e579..6b45dd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,5 @@ +version: '3.8' + services: neo4j: image: neo4j:5.21.0-community From 0f81a77ff87e881cc627732283ade8318cd91c7d Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:48:55 -0700 Subject: [PATCH 3/9] fix: update Docker images to use specific versions instead of latest - Update confluentinc/cp-zookeeper:latest to 7.5.0 - Update confluentinc/cp-kafka-connect:latest to 7.5.0 - Update iud2i/ckn-cameratraps-acc-alert:latest to 2.0.0 - Update iud2i/ckn-cameratraps-agg:latest to 2.0.0 - Fixes linux/amd64 manifest errors in CI/CD pipeline --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6b45dd6..de998fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - ckn-network zookeeper: - image: confluentinc/cp-zookeeper:latest + image: confluentinc/cp-zookeeper:7.5.0 container_name: zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 @@ -59,7 +59,7 @@ services: - ckn-network kafka-connect: - image: confluentinc/cp-kafka-connect:latest + image: confluentinc/cp-kafka-connect:7.5.0 container_name: kafka-connect depends_on: - broker @@ -105,7 +105,7 @@ services: - ckn-network oracle-alert-processor: - image: iud2i/ckn-cameratraps-acc-alert:latest + image: iud2i/ckn-cameratraps-acc-alert:2.0.0 depends_on: broker: condition: service_healthy @@ -119,7 +119,7 @@ services: - ckn-network oracle-aggr-processor: - image: iud2i/ckn-cameratraps-agg:latest + image: iud2i/ckn-cameratraps-agg:2.0.0 depends_on: broker: condition: service_healthy From a05db98a5862b0a507ca4319665678982ae15b78 Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:50:11 -0700 Subject: [PATCH 4/9] chore: update Docker images to use 'latest' tag for alert and aggregation processors --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index de998fc..b6d8b8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,7 +105,7 @@ services: - ckn-network oracle-alert-processor: - image: iud2i/ckn-cameratraps-acc-alert:2.0.0 + image: iud2i/ckn-cameratraps-acc-alert:latest depends_on: broker: condition: service_healthy @@ -119,7 +119,7 @@ services: - ckn-network oracle-aggr-processor: - image: iud2i/ckn-cameratraps-agg:2.0.0 + image: iud2i/ckn-cameratraps-agg:latest depends_on: broker: condition: service_healthy From 020169ebcd72aef6bad6d0ff7adc1b0065f63e7e Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:54:07 -0700 Subject: [PATCH 5/9] fix: use specific version tags for all Docker images to resolve linux/amd64 manifest errors - Update iud2i/ckn-cameratraps-acc-alert:latest to 0.1.2 - Update iud2i/ckn-cameratraps-agg:latest to 0.1.2 - All images now use specific version tags instead of latest - Resolves CI/CD pipeline manifest errors --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b6d8b8d..d0411da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,7 +105,7 @@ services: - ckn-network oracle-alert-processor: - image: iud2i/ckn-cameratraps-acc-alert:latest + image: iud2i/ckn-cameratraps-acc-alert:0.1.2 depends_on: broker: condition: service_healthy @@ -119,7 +119,7 @@ services: - ckn-network oracle-aggr-processor: - image: iud2i/ckn-cameratraps-agg:latest + image: iud2i/ckn-cameratraps-agg:0.1.2 depends_on: broker: condition: service_healthy From 7aabb68b2cd34e1785feaa86e8a6d036718fb36e Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:56:12 -0700 Subject: [PATCH 6/9] fix: revert iud2i images to latest tags - version 0.1.2 does not exist - Revert iud2i/ckn-cameratraps-acc-alert:0.1.2 to latest - Revert iud2i/ckn-cameratraps-agg:0.1.2 to latest - Keep Confluent images with specific versions (7.5.0) to fix linux/amd64 issues - Fixes 'manifest unknown' errors in CI/CD pipeline --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d0411da..b6d8b8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,7 +105,7 @@ services: - ckn-network oracle-alert-processor: - image: iud2i/ckn-cameratraps-acc-alert:0.1.2 + image: iud2i/ckn-cameratraps-acc-alert:latest depends_on: broker: condition: service_healthy @@ -119,7 +119,7 @@ services: - ckn-network oracle-aggr-processor: - image: iud2i/ckn-cameratraps-agg:0.1.2 + image: iud2i/ckn-cameratraps-agg:latest depends_on: broker: condition: service_healthy From c0598693811a5b39bf96fe108195b88aa126e749 Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 12:59:16 -0700 Subject: [PATCH 7/9] fix: update Confluent images to version 7.4.0 and remove obsolete version attribute - Update confluentinc/cp-zookeeper:7.5.0 to 7.4.0 - Update confluentinc/cp-kafka:7.9.2 to 7.4.0 - Update confluentinc/cp-kafka-connect:7.5.0 to 7.4.0 - Remove obsolete version: '3.8' attribute from docker-compose.yml - Use consistent Confluent Platform version 7.4.0 for all components - Attempts to resolve linux/amd64 manifest issues --- docker-compose.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b6d8b8d..3fb3097 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: neo4j: image: neo4j:5.21.0-community @@ -24,7 +22,7 @@ services: - ckn-network zookeeper: - image: confluentinc/cp-zookeeper:7.5.0 + image: confluentinc/cp-zookeeper:7.4.0 container_name: zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 @@ -35,7 +33,7 @@ services: - ckn-network broker: - image: confluentinc/cp-kafka:7.9.2 + image: confluentinc/cp-kafka:7.4.0 container_name: broker depends_on: - zookeeper @@ -59,7 +57,7 @@ services: - ckn-network kafka-connect: - image: confluentinc/cp-kafka-connect:7.5.0 + image: confluentinc/cp-kafka-connect:7.4.0 container_name: kafka-connect depends_on: - broker From d49031c2161cb0e151704a545113729078741d6c Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 13:00:48 -0700 Subject: [PATCH 8/9] test: temporarily comment out iud2i services to isolate linux/amd64 manifest issue - Comment out oracle-alert-processor and oracle-aggr-processor services - Test if Confluent Platform images (7.4.0) work without iud2i images - This will help identify which specific image is causing the manifest error --- docker-compose.yml | 53 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3fb3097..8caa3be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,33 +102,34 @@ services: networks: - ckn-network - oracle-alert-processor: - image: iud2i/ckn-cameratraps-acc-alert:latest - depends_on: - broker: - condition: service_healthy - environment: - - CKN_BROKERS=broker:29092 - - ORACLE_ACC_CRITICAL_THRESHOLD=0.5 - - ORACLE_INPUT_TOPIC=oracle-events - - ORACLE_ACC_ALERT_TOPIC=oracle-alerts - - APP_ID=ckn-camera-traps-oracle-processor - networks: - - ckn-network +# Temporarily commented out to test Confluent images +# oracle-alert-processor: +# image: iud2i/ckn-cameratraps-acc-alert:latest +# depends_on: +# broker: +# condition: service_healthy +# environment: +# - CKN_BROKERS=broker:29092 +# - ORACLE_ACC_CRITICAL_THRESHOLD=0.5 +# - ORACLE_INPUT_TOPIC=oracle-events +# - ORACLE_ACC_ALERT_TOPIC=oracle-alerts +# - APP_ID=ckn-camera-traps-oracle-processor +# networks: +# - ckn-network - oracle-aggr-processor: - image: iud2i/ckn-cameratraps-agg:latest - depends_on: - broker: - condition: service_healthy - environment: - - CKN_BROKERS=broker:29092 - - CKN_ORACLE_WINDOW_TIME=1 - - ORACLE_INPUT_TOPIC=oracle-events - - ORACLE_AGG_ALERT_TOPIC=oracle-aggregated - - APP_ID=ckn-camera-traps-oracle-aggreg - networks: - - ckn-network +# oracle-aggr-processor: +# image: iud2i/ckn-cameratraps-agg:latest +# depends_on: +# broker: +# condition: service_healthy +# environment: +# - CKN_BROKERS=broker:29092 +# - CKN_ORACLE_WINDOW_TIME=1 +# - ORACLE_INPUT_TOPIC=oracle-events +# - ORACLE_AGG_ALERT_TOPIC=oracle-aggregated +# - APP_ID=ckn-camera-traps-oracle-aggreg +# networks: +# - ckn-network networks: ckn-network: From 3c146b654c8a75658e6acd476a3a97797ee67e3a Mon Sep 17 00:00:00 2001 From: neelk Date: Wed, 10 Sep 2025 13:42:19 -0700 Subject: [PATCH 9/9] feat: build and push multi-architecture Docker image for ckn-analytics-dashboard - Build iud2i/ckn-analytics-dashboard:2.0.0 for both linux/amd64 and linux/arm64 - Fix Dockerfile COPY paths to use ckn_dashboard/ prefix - Remove platform-specific FROM instruction for multi-arch support - Re-enable oracle services in docker-compose.yml - Resolves linux/amd64 manifest issues for our custom dashboard image --- ckn_dashboard/Dockerfile | 16 ++++++------ docker-compose.yml | 53 ++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/ckn_dashboard/Dockerfile b/ckn_dashboard/Dockerfile index e886ab1..d010daf 100644 --- a/ckn_dashboard/Dockerfile +++ b/ckn_dashboard/Dockerfile @@ -1,14 +1,14 @@ -FROM --platform=linux/amd64 python:3.9-slim +FROM python:3.9-slim WORKDIR /app -COPY Home.py /app/ -COPY ckn_kg.py /app/ -COPY llm_graph.py /app/ -COPY util.py /app/ -COPY modelcards /app/modelcards -COPY requirements.txt /app/ -COPY /pages /app/pages +COPY ckn_dashboard/Home.py /app/ +COPY ckn_dashboard/ckn_kg.py /app/ +COPY ckn_dashboard/llm_graph.py /app/ +COPY ckn_dashboard/util.py /app/ +COPY ckn_dashboard/modelcards /app/modelcards +COPY ckn_dashboard/requirements.txt /app/ +COPY ckn_dashboard/pages /app/pages RUN pip install -r /app/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 8caa3be..3fb3097 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,34 +102,33 @@ services: networks: - ckn-network -# Temporarily commented out to test Confluent images -# oracle-alert-processor: -# image: iud2i/ckn-cameratraps-acc-alert:latest -# depends_on: -# broker: -# condition: service_healthy -# environment: -# - CKN_BROKERS=broker:29092 -# - ORACLE_ACC_CRITICAL_THRESHOLD=0.5 -# - ORACLE_INPUT_TOPIC=oracle-events -# - ORACLE_ACC_ALERT_TOPIC=oracle-alerts -# - APP_ID=ckn-camera-traps-oracle-processor -# networks: -# - ckn-network + oracle-alert-processor: + image: iud2i/ckn-cameratraps-acc-alert:latest + depends_on: + broker: + condition: service_healthy + environment: + - CKN_BROKERS=broker:29092 + - ORACLE_ACC_CRITICAL_THRESHOLD=0.5 + - ORACLE_INPUT_TOPIC=oracle-events + - ORACLE_ACC_ALERT_TOPIC=oracle-alerts + - APP_ID=ckn-camera-traps-oracle-processor + networks: + - ckn-network -# oracle-aggr-processor: -# image: iud2i/ckn-cameratraps-agg:latest -# depends_on: -# broker: -# condition: service_healthy -# environment: -# - CKN_BROKERS=broker:29092 -# - CKN_ORACLE_WINDOW_TIME=1 -# - ORACLE_INPUT_TOPIC=oracle-events -# - ORACLE_AGG_ALERT_TOPIC=oracle-aggregated -# - APP_ID=ckn-camera-traps-oracle-aggreg -# networks: -# - ckn-network + oracle-aggr-processor: + image: iud2i/ckn-cameratraps-agg:latest + depends_on: + broker: + condition: service_healthy + environment: + - CKN_BROKERS=broker:29092 + - CKN_ORACLE_WINDOW_TIME=1 + - ORACLE_INPUT_TOPIC=oracle-events + - ORACLE_AGG_ALERT_TOPIC=oracle-aggregated + - APP_ID=ckn-camera-traps-oracle-aggreg + networks: + - ckn-network networks: ckn-network: