Adding helm tests for IAG5 deployment verification

README.md

@@ -262,6 +262,33 @@ This project uses the helm unittest plugin.
 ```bash
 helm unittest .
 ```
+### Run helm tests
+
+This project uses helm tests that should be run after deployment. These are integration tests that run actual pods in your cluster after deployment to verify the application is working correctly.
+
+Helm tests are present in `iag5-helm/charts/iag5/templates/tests/`
+
+Helm looks through ALL rendered Kubernetes manifests in chart for resources that have this specific annotation:
+
+```bash
+annotations:
+  "helm.sh/hook": test
+```
+Run all test using below command
+
+```bash
+helm test iag5 -n <NAMESPACE> --logs
+```
+Run individual test using below command
+
+```bash
+helm test iag5 -n <NAMESPACE> --filter name=iag5-test-connection --logs
+```
+Run multiple tests using below command
+
+```bash
+helm test iag5 -n <NAMESPACE> --filter 'name=iag5-test-version,name=iag5-test-processes' --logs
+```
 
 #### Values
 

charts/iag5/Chart.yaml

@@ -10,7 +10,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.3
+version: 1.0.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/iag5/templates/tests/test-connection.yaml

@@ -0,0 +1,29 @@
+# Test to check the connection to 50051 to ensure IAG5 service is running
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "iag5.fullname" . }}-test-connection"
+  labels:
+    {{- include "iag5.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  containers:
+  - name: connection-test
+    image: busybox:latest
+    command: ['sh', '-c']
+# Testing connection to iag5-service.am.svc.cluster.local:50051
+    args:
+    - |
+      echo "Testing connection to {{ .Values.service.name }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.port }}"
+      nc -zv -w 10 {{ .Values.service.name }}.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.port }}
+      if [ $? -eq 0 ]; then
+        echo "SUCCESS: Port {{ .Values.port }} is open and accepting connections"
+        exit 0
+      else
+        echo "FAILED: Cannot connect to port {{ .Values.port }}"
+        exit 1
+      fi
+  restartPolicy: Never

charts/iag5/templates/tests/test-processes.yaml

@@ -0,0 +1,160 @@
+# Test to check the number of processes running
+# Test will check the number of processes running in sever & running with 
+# minimum number of processes mentioned in values.yaml
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "iag5.fullname" . }}-test-processes"
+  labels:
+    {{- include "iag5.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  serviceAccountName: {{ include "iag5.fullname" . }}-test
+  containers:
+  - name: process-checker
+    image: bitnami/kubectl:latest
+    command: ['sh', '-c']
+    args:
+    - |
+      set -e
+
+      # Get minimum process counts from values (with defaults)
+      MIN_PROCESSES_SERVER={{ .Values.tests.processCount.server.min | default 1 }}
+      MIN_PROCESSES_RUNNER={{ .Values.tests.processCount.runner.min | default 1 }}
+
+      echo "========================================="
+      echo "IAG5 Process Count Verification"
+      echo "Namespace: {{ .Release.Namespace }}"
+      echo "Minimum expected 'iagctl server' processes: $MIN_PROCESSES_SERVER"
+      echo "Minimum expected 'iagctl runner' processes: $MIN_PROCESSES_RUNNER"
+      echo "========================================="
+      echo ""
+
+      FAILED=0
+      TOTAL_PODS=0
+      PASSED_PODS=0
+
+      # Wait for pods to be ready
+      echo "Waiting for IAG5 pods to be ready..."
+      kubectl wait --for=condition=ready pod \
+        -l app.kubernetes.io/name={{ include "iag5.name" . }} \
+        -n {{ .Release.Namespace }} \
+        --timeout=60s || true
+      echo ""
+
+      # Check server pods
+      echo "========================================="
+      echo "Checking SERVER pods..."
+      echo "========================================="
+      SERVER_PODS=$(kubectl get pods \
+        -l app.kubernetes.io/name={{ include "iag5.name" . }},app.kubernetes.io/component=server \
+        -n {{ .Release.Namespace }} \
+        -o jsonpath='{.items[*].metadata.name}')
+
+      if [ -z "$SERVER_PODS" ]; then
+        echo "WARNING: No server pods found"
+      else
+        for POD in $SERVER_PODS; do
+          TOTAL_PODS=$((TOTAL_PODS + 1))
+          echo ""
+          echo "Pod: $POD"
+
+          # Get pod status
+          POD_STATUS=$(kubectl get pod $POD -n {{ .Release.Namespace }} -o jsonpath='{.status.phase}')
+          echo "  Status: $POD_STATUS"
+
+          if [ "$POD_STATUS" != "Running" ]; then
+            echo "  Result:  SKIPPED (Pod not running)"
+            continue
+          fi
+
+          # Count 'iagctl server' processes
+          PROCESS_COUNT=$(kubectl exec -n {{ .Release.Namespace }} $POD -- sh -c "ps aux | grep 'iagctl server' | grep -v grep | wc -l" 2>/dev/null || echo "0")
+          echo "  'iagctl server' process count: $PROCESS_COUNT"
+
+          if [ "$PROCESS_COUNT" -ge "$MIN_PROCESSES_SERVER" ]; then
+            echo "  Result: ✓ PASS ($PROCESS_COUNT >= $MIN_PROCESSES_SERVER)"
+            PASSED_PODS=$((PASSED_PODS + 1))
+          else
+            echo "  Result: ✗ FAIL ($PROCESS_COUNT < $MIN_PROCESSES_SERVER)"
+            FAILED=1
+
+            # Only show processes on failure for debugging
+            echo "  All processes in pod (for debugging):"
+            kubectl exec -n {{ .Release.Namespace }} $POD -- ps aux 2>/dev/null | sed 's/^/    /' || echo "    Could not retrieve process list"
+          fi
+        done
+      fi
+
+      # Check runner pods
+      echo ""
+      echo "========================================="
+      echo "Checking RUNNER pods..."
+      echo "========================================="
+      RUNNER_PODS=$(kubectl get pods \
+        -l app.kubernetes.io/name={{ include "iag5.name" . }},app.kubernetes.io/component=runner \
+        -n {{ .Release.Namespace }} \
+        -o jsonpath='{.items[*].metadata.name}')
+
+      if [ -z "$RUNNER_PODS" ]; then
+        echo "INFO: No runner pods found (runnerSettings.replicaCount may be 0)"
+      else
+        for POD in $RUNNER_PODS; do
+          TOTAL_PODS=$((TOTAL_PODS + 1))
+          echo ""
+          echo "Pod: $POD"
+
+          # Get pod status
+          POD_STATUS=$(kubectl get pod $POD -n {{ .Release.Namespace }} -o jsonpath='{.status.phase}')
+          echo "  Status: $POD_STATUS"
+
+          if [ "$POD_STATUS" != "Running" ]; then
+            echo "  Result:  SKIPPED (Pod not running)"
+            continue
+          fi
+
+          # Count 'iagctl runner' processes
+          PROCESS_COUNT=$(kubectl exec -n {{ .Release.Namespace }} $POD -- sh -c "ps aux | grep 'iagctl runner' | grep -v grep | wc -l" 2>/dev/null || echo "0")
+          echo "  'iagctl runner' process count: $PROCESS_COUNT"
+
+          if [ "$PROCESS_COUNT" -ge "$MIN_PROCESSES_RUNNER" ]; then
+            echo "  Result: ✓ PASS ($PROCESS_COUNT >= $MIN_PROCESSES_RUNNER)"
+            PASSED_PODS=$((PASSED_PODS + 1))
+          else
+            echo "  Result: ✗ FAIL ($PROCESS_COUNT < $MIN_PROCESSES_RUNNER)"
+            FAILED=1
+
+            # Only show processes on failure for debugging
+            echo "  All processes in pod (for debugging):"
+            kubectl exec -n {{ .Release.Namespace }} $POD -- ps aux 2>/dev/null | sed 's/^/    /' || echo "    Could not retrieve process list"
+          fi
+        done
+      fi
+
+      # Print summary
+      echo ""
+      echo "========================================="
+      echo "SUMMARY"
+      echo "========================================="
+      echo "Total pods checked: $TOTAL_PODS"
+      echo "Passed: $PASSED_PODS"
+      echo "Failed: $((TOTAL_PODS - PASSED_PODS))"
+      echo ""
+
+      if [ $FAILED -eq 0 ] && [ $TOTAL_PODS -gt 0 ]; then
+        echo "Result: ✓ SUCCESS"
+        echo "All pods have adequate 'iagctl' process counts"
+        exit 0
+      elif [ $TOTAL_PODS -eq 0 ]; then
+        echo "Result: ✗ FAILED"
+        echo "No IAG5 pods found to test"
+        exit 1
+      else
+        echo "Result: ✗ FAILED"
+        echo "One or more pods do not have adequate 'iagctl' process counts"
+        exit 1
+      fi
+  restartPolicy: Never