Adding IAPI email generation to addons

docker-compose-addons.yml

@@ -12,10 +12,11 @@ services:
       ENABLE_EMAILER: ${ENABLE_EMAILER:-}
       DEPLOYMENT_TITLE: ${DEPLOYMENT_TITLE:-}
 
-      SMTP_SERVER_IP: ${SMTP_SERVER_IP}
-      SMTP_USERNAME: ${SMTP_USERNAME}
-      SMTP_PASSWORD: ${SMTP_PASSWORD}
-      SMTP_EMAIL: ${SMTP_EMAIL}
+      IAPI_ENDPOINT: ${IAPI_ENDPOINT:-http://${DOCKER_HOST_IP}:8089}
+      KC_ENDPOINT: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084}
+      KC_REALM: ${KEYCLOAK_REALM:-cvmanager}
+      KC_SA_CLIENT_ID: ${KEYCLOAK_SA_COUNT_METRIC_CLIENT_ID:-sa_count_metric}
+      KC_SA_CLIENT_SECRET: ${KEYCLOAK_SA_COUNT_METRIC_CLIENT_SECRET_KEY:-sa-count-metric-secret-key}
 
       PROJECT_ID: ${GCP_PROJECT_ID:-}
 
@@ -122,7 +123,7 @@ services:
     restart: on-failure:3
 
     ports:
-      - '8095:8080'
+      - '8091:8080'
     environment:
       PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432}
       PG_DB_NAME: ${PG_DB_NAME:-postgres}
@@ -160,10 +161,11 @@ services:
       GCP_PROJECT: ${GCP_PROJECT_ID:-}
       GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json
 
-      SMTP_SERVER_IP: ${SMTP_SERVER_IP}
-      SMTP_EMAIL: ${SMTP_EMAIL}
-      SMTP_USERNAME: ${SMTP_USERNAME}
-      SMTP_PASSWORD: ${SMTP_PASSWORD}
+      IAPI_ENDPOINT: ${IAPI_ENDPOINT:-http://${DOCKER_HOST_IP}:8089}
+      KC_ENDPOINT: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084}
+      KC_REALM: ${KEYCLOAK_REALM:-cvmanager}
+      KC_SA_CLIENT_ID: ${KEYCLOAK_SA_FIRMWARE_UPGRADE_RUNNER_CLIENT_ID:-sa_firmware_upgrade_runner}
+      KC_SA_CLIENT_SECRET: ${KEYCLOAK_SA_FIRMWARE_UPGRADE_RUNNER_CLIENT_SECRET_KEY:-sa-firmware-upgrade-runner-secret-key}
 
       UPGRADE_SCHEDULER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT:-http://firmware_manager_upgrade_scheduler:8090}
 

docker-compose-intersection.yml

@@ -59,7 +59,7 @@ services:
       UNSUBSCRIBE_SECRET_KEY: ${UNSUBSCRIBE_SECRET_KEY:-b5f9070ca58d4576ea6a2d8d4c1b97caf8abc2554198f8f9acf86d0d31a52a52}
       CV_MANAGER_FRONT_END_URI: ${CV_MANAGER_FRONT_END_URI:-http://localhost:${WEBAPP_PORT:-3000}}
       INTERSECTION_SMTP_SERVER_HOST: ${INTERSECTION_SMTP_SERVER_HOST:-smtp4dev}
-      INTERSECTION_SMTP_SERVER_PORT: ${INTERSECTION_SMTP_SERVER_PORT:-1025}
+      INTERSECTION_SMTP_SERVER_PORT: ${INTERSECTION_SMTP_SERVER_PORT:-25}
       INTERSECTION_SMTP_USERNAME: ${INTERSECTION_SMTP_USERNAME:-admin}
       INTERSECTION_SMTP_PASSWORD: ${INTERSECTION_SMTP_PASSWORD:-password}
       INTERSECTION_SMTP_AUTH: ${INTERSECTION_SMTP_AUTH_ENABLED:-true}

services/addons/images/count_metric/count_metric_environment.py

@@ -1,10 +1,10 @@
 from common.common_environment import get_env_var
 
-SMTP_SERVER_IP=get_env_var("SMTP_SERVER_IP", error=True)
-SMTP_SERVER_PORT=int(get_env_var("SMTP_SERVER_PORT", "587", warn=False))
-SMTP_EMAIL=get_env_var("SMTP_EMAIL", error=True)
-DEPLOYMENT_TITLE=get_env_var("DEPLOYMENT_TITLE", "Example Deployment", warn=True)
-SMTP_USERNAME=get_env_var("SMTP_USERNAME", error=True)
-SMTP_PASSWORD=get_env_var("SMTP_PASSWORD", error=True)
-MONGO_DB_URI=get_env_var("MONGO_DB_URI", "mongodb://localhost:27017")
-MONGO_DB_NAME=get_env_var("MONGO_DB_NAME", "CV")
+DEPLOYMENT_TITLE = get_env_var("DEPLOYMENT_TITLE", "Example Deployment", warn=True)
+MONGO_DB_URI = get_env_var("MONGO_DB_URI", "mongodb://localhost:27017")
+MONGO_DB_NAME = get_env_var("MONGO_DB_NAME", "CV")
+IAPI_ENDPOINT = get_env_var("IAPI_ENDPOINT", error=True)
+KC_ENDPOINT = get_env_var("KC_ENDPOINT", error=True)
+KC_REALM = get_env_var("KC_REALM", error=True)
+KC_SA_CLIENT_ID = get_env_var("KC_SA_CLIENT_ID", "sa_count_metric")
+KC_SA_CLIENT_SECRET = get_env_var("KC_SA_CLIENT_SECRET", error=True, secret=True)

services/addons/images/count_metric/daily_emailer.py

@@ -1,11 +1,10 @@
 import logging
-import gen_email
-from common.emailSender import EmailSender
+from common.email_api import EmailApi
 import common.pgquery as pgquery
-from common.email_util import get_email_list
 from datetime import datetime, timedelta
 from pymongo import MongoClient
 import count_metric_environment
+from common.keycloak_api import KeycloakServiceAccountApi
 
 message_types = ["BSM", "TIM", "Map", "SPaT", "SRM", "SSM"]
 
@@ -122,26 +121,34 @@ def prepare_org_rsu_dict():
     return rsu_dict
 
 
-def email_daily_counts(org_name, email_body):
+def email_daily_counts(
+    org_name: str,
+    deployment_title: str,
+    start_date: datetime,
+    end_date: datetime,
+    message_type_list: list[str],
+    counts: list[dict],
+):
     logging.info("Attempting to send the count emails...")
     try:
-        email_addresses = get_email_list("Daily Message Counts", org_name)
-
-        for email_address in email_addresses:
-            emailSender = EmailSender(
-                count_metric_environment.SMTP_SERVER_IP,
-                count_metric_environment.SMTP_SERVER_PORT,
-            )
-            emailSender.send(
-                sender=count_metric_environment.SMTP_EMAIL,
-                recipient=email_address,
-                subject=f"{org_name} {count_metric_environment.DEPLOYMENT_TITLE} Counts",
-                message=email_body,
-                replyEmail="",
-                username=count_metric_environment.SMTP_USERNAME,
-                password=count_metric_environment.SMTP_PASSWORD,
-                pretty=True,
-            )
+        kc_api = KeycloakServiceAccountApi(
+            endpoint=count_metric_environment.KC_ENDPOINT,
+            realm=count_metric_environment.KC_REALM,
+            client_id=count_metric_environment.KC_SA_CLIENT_ID,
+            client_secret=count_metric_environment.KC_SA_CLIENT_SECRET,
+        )
+        email_api = EmailApi(
+            iapi_base_url=count_metric_environment.IAPI_ENDPOINT, kc_api=kc_api
+        )
+
+        email_api.send_message_counts(
+            org_name,
+            deployment_title,
+            start_date,
+            end_date,
+            message_type_list,
+            counts,
+        )
     except Exception as e:
         logging.error(e)
 
@@ -164,11 +171,24 @@ def run_daily_emailer():
         query_mongo_in_counts(rsu_dict, start_dt, end_dt, mongo_db)
         query_mongo_out_counts(rsu_dict, start_dt, end_dt, mongo_db)
 
-        # Generate the email content with the populated rsu_dict
-        email_body = gen_email.generate_email_body(
-            org_name, rsu_dict, start_dt, end_dt, message_types
+        rsu_counts = [
+            {
+                "rsu_ip": rsu_ip,
+                "counts": data["counts"],
+                "primary_route": data["primary_route"],
+            }
+            for rsu_ip, data in rsu_dict.items()
+        ]
+
+        # Send emails through the Intersection API
+        email_daily_counts(
+            org_name=org_name,
+            deployment_title=count_metric_environment.DEPLOYMENT_TITLE,
+            start_date=start_dt,
+            end_date=end_dt,
+            message_type_list=message_types,
+            counts=rsu_counts,
         )
-        email_daily_counts(org_name, email_body)
 
 
 if __name__ == "__main__":

services/addons/images/count_metric/requirements.txt

@@ -4,3 +4,4 @@ pymongo==4.5.0
 sqlalchemy==2.0.21
 pg8000==1.30.2
 python-dateutil==2.8.2
+python-keycloak==5.8.1

services/addons/images/count_metric/sample.env

@@ -16,11 +16,10 @@ PG_DB_NAME =
 MONGO_DB_URI = 'mongodb://:27017/'
 MONGO_DB_NAME = ''
 
-# SMTP REQUIRED VARIABLES
-SMTP_SERVER_IP = ''
-SMTP_USERNAME = ''
-SMTP_PASSWORD = ''
-SMTP_EMAIL = ''
+# Error Email Configuration (proxied email sending through the intersection api)
+IAPI_ENDPOINT=http://${DOCKER_HOST_IP}:8089
+KC_SA_CLIENT_ID=sa_count_metric
+KC_SA_CLIENT_SECRET=sa-count-metric-secret-key
 
 # ---------------------------------------------------------------------
 # If ENABLE_EMAILER is 'False', set the following environment variables

services/addons/images/firmware_manager/requirements.txt

@@ -10,3 +10,4 @@ sqlalchemy==2.0.21
 waitress==2.1.2
 python-dateutil==2.8.2
 pytz==2023.3.post1
+python-keycloak==5.8.1

services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py

@@ -1,4 +1,5 @@
 import time
+import traceback
 from paramiko import SSHClient, WarningPolicy
 from scp import SCPClient
 import upgrader
@@ -74,7 +75,10 @@ def upgrade(self):
             self.cleanup()
             self.notify_firmware_manager(success=False)
             # send email to support team with the rsu and error
-            self.send_error_email("Firmware Upgrader", err)
+            stack_trace = traceback.format_exc()
+            self.send_error_email(
+                str(err), stack_trace, "Commsignia Firmware Upgrade Error"
+            )
 
     def post_upgrade(self):
         if self.wait_until_online() == -1:
@@ -122,7 +126,10 @@ def post_upgrade(self):
                 f"Failed to execute post upgrade script for rsu {self.rsu_ip}: {err}"
             )
             # send email to support team with the rsu and error
-            self.send_error_email("Post-Upgrade Script", err)
+            stack_trace = traceback.format_exc()
+            self.send_error_email(
+                str(err), stack_trace, "Commsignia Post-Upgrade Script Error"
+            )
 
 
 # sys.argv[1] - JSON string with the following key-values:

services/addons/images/firmware_manager/upgrade_runner/sample.env

@@ -13,11 +13,10 @@ FW_UPGRADE_MAX_RETRY_LIMIT=3
 GCP_PROJECT=""
 GOOGLE_APPLICATION_CREDENTIALS=""
 
-# For sending failure emails
-SMTP_SERVER_IP=
-SMTP_EMAIL=
-SMTP_USERNAME=
-SMTP_PASSWORD=
+# Error Email Configuration (proxied email sending through the intersection api)
+IAPI_ENDPOINT=http://${DOCKER_HOST_IP}:8089
+KC_SA_CLIENT_ID=sa_firmware_upgrade_runner
+KC_SA_CLIENT_SECRET=sa-firmware-upgrade-runner-secret-key
 
 # Must specify this endpoint to wherever the Upgrade Scheduler is hosted
 # Must include 'http://' or 'https://' along with specified port if non-standard

services/addons/images/firmware_manager/upgrade_runner/upgrade_runner_environment.py

@@ -1,11 +1,10 @@
 from common.common_environment import get_env_var
 
-BLOB_STORAGE_PROVIDER=get_env_var("BLOB_STORAGE_PROVIDER", "DOCKER", warn=False)
-UPGRADE_SCHEDULER_ENDPOINT=get_env_var("UPGRADE_SCHEDULER_ENDPOINT", "127.0.0.1")
+BLOB_STORAGE_PROVIDER = get_env_var("BLOB_STORAGE_PROVIDER", "DOCKER", warn=False)
+UPGRADE_SCHEDULER_ENDPOINT = get_env_var("UPGRADE_SCHEDULER_ENDPOINT", "127.0.0.1")
 
-SMTP_SERVER_IP=get_env_var("SMTP_SERVER_IP", error=True)
-SMTP_SERVER_PORT=int(get_env_var("SMTP_SERVER_PORT", "587", warn=False))
-SMTP_EMAIL=get_env_var("SMTP_EMAIL", error=True)
-DEPLOYMENT_TITLE=get_env_var("DEPLOYMENT_TITLE", "Example Deployment", warn=True)
-SMTP_USERNAME=get_env_var("SMTP_USERNAME", error=True)
-SMTP_PASSWORD=get_env_var("SMTP_PASSWORD", error=True)
+IAPI_ENDPOINT = get_env_var("IAPI_ENDPOINT", error=True)
+KC_ENDPOINT = get_env_var("KC_ENDPOINT", error=True)
+KC_REALM = get_env_var("KC_REALM", error=True)
+KC_SA_CLIENT_ID = get_env_var("KC_SA_CLIENT_ID", "sa_firmware_upgrade_runner")
+KC_SA_CLIENT_SECRET = get_env_var("KC_SA_CLIENT_SECRET", error=True, secret=True)

services/addons/images/firmware_manager/upgrade_runner/upgrader.py

@@ -6,8 +6,8 @@
 import logging
 import requests
 import shutil
-from common.emailSender import EmailSender
-from common.email_util import get_email_list_from_rsu
+from common.email_api import EmailApi
+from common.keycloak_api import KeycloakServiceAccountApi
 from addons.images.firmware_manager.upgrade_runner import download_blob
 import upgrade_runner_environment
 
@@ -116,33 +116,24 @@ def check_online(self):
         # 5 seconds pass with no response
         return False
 
-    def send_error_email(self, type="Firmware Upgrader", err=""):
+    def send_error_email(self, err: Exception, stack_trace: str, type: str):
         try:
-            email_addresses = get_email_list_from_rsu(
-                "Firmware Upgrade Failures", self.rsu_ip
+            kc_api = KeycloakServiceAccountApi(
+                endpoint=upgrade_runner_environment.KC_ENDPOINT,
+                realm=upgrade_runner_environment.KC_REALM,
+                client_id=upgrade_runner_environment.KC_SA_CLIENT_ID,
+                client_secret=upgrade_runner_environment.KC_SA_CLIENT_SECRET,
             )
-
-            subject = (
-                f"{self.rsu_ip} Firmware Upgrader Failure"
-                if type == "Firmware Upgrader"
-                else f"{self.rsu_ip} Firmware Upgrader Post Upgrade Script Failure"
+            email_api = EmailApi(
+                iapi_base_url=upgrade_runner_environment.IAPI_ENDPOINT, kc_api=kc_api
             )
 
-            for email_address in email_addresses:
-                emailSender = EmailSender(
-                    upgrade_runner_environment.SMTP_SERVER_IP,
-                    upgrade_runner_environment.SMTP_SERVER_PORT,
-                )
-                emailSender.send(
-                    sender=upgrade_runner_environment.SMTP_EMAIL,
-                    recipient=email_address,
-                    subject=subject,
-                    message=f"{type}: Failed to perform update on RSU {self.rsu_ip} due to the following error: {err}",
-                    replyEmail="",
-                    username=upgrade_runner_environment.SMTP_USERNAME,
-                    password=upgrade_runner_environment.SMTP_PASSWORD,
-                    pretty=True,
-                )
+            email_api.send_firmware_upgrade_failure(
+                rsu_ip=self.rsu_ip,
+                error_message=f"{type}: Failed to perform update on RSU {self.rsu_ip} due to the following error: {err}",
+                failure_type=type,
+                stack_trace=stack_trace,
+            )
         except Exception as e:
             logging.error(e)
 

services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py

@@ -5,6 +5,7 @@
 import sys
 import tarfile
 import time
+import traceback
 from common import common_environment
 
 
@@ -111,7 +112,8 @@ def upgrade(self):
             self.cleanup()
             self.notify_firmware_manager(success=False)
             # send email to support team with the rsu and error
-            self.send_error_email("Firmware Upgrader", err)
+            stack_trace = traceback.format_exc()
+            self.send_error_email(str(err), stack_trace, "Yunex Firmware Upgrade Error")
 
 
 # sys.argv[1] - JSON string with the following key-values:

services/addons/tests/count_metric/conftest.py

@@ -1,6 +1,7 @@
 import os
 
-os.environ['SMTP_SERVER_IP'] = 'smtp-server-ip'
-os.environ['SMTP_EMAIL'] = 'smtp-email'
-os.environ['SMTP_USERNAME'] = 'smtp-username'
-os.environ['SMTP_PASSWORD'] = 'smtp-password'
+os.environ["IAPI_ENDPOINT"] = "localhost:8089"
+os.environ["KC_ENDPOINT"] = "http://localhost:8084"
+os.environ["KC_REALM"] = "cvmanager"
+os.environ["KC_SA_CLIENT_ID"] = "sa_count_metric"
+os.environ["KC_SA_CLIENT_SECRET"] = "sa_count_metric_secret"