Merge pull request #29 from TechnologyEnhancedLearning/poc-dab-workflow

Poc dab workflow

Author: Phil-NHS

.github/workflows/dev-cd.yml

@@ -58,4 +58,4 @@ jobs:
       - name: Check Dev Integration Tests After Deploy
         run: |
           echo "running the integration yml test job integration_test_job.yml"
-          databricks bundle run -t dev run_integration_tests 
+          databricks bundle run -t dev integration_test_job 

.github/workflows/prod-cd.yml

@@ -10,26 +10,26 @@ on:
 jobs:
 
   deploy_prod:
-      name: "Deploy Bundle to Production Environment"
-      runs-on: ubuntu-latest
-      environment: prod
-      env:
-        DATABRICKS_HOST: ${{ vars.DBX_HOST }}
-        DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
-        DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
-
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
-
-        - name: Set up Databricks CLI
-          uses: databricks/setup-cli@main
+    name: "Deploy Bundle to Production Environment"
+    runs-on: ubuntu-latest
+    environment: prod
+    env:
+      DATABRICKS_HOST: ${{ vars.DBX_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Databricks CLI
+        uses: databricks/setup-cli@main
 
-        - name: Deploy Prod Bundle
+      - name: Deploy Prod Bundle
           # trigger target the prod deploy in databricks.yml
-          run: databricks bundle deploy -t prod --auto-approve
+        run: databricks bundle deploy -t prod --auto-approve
+
 
-          
         # qqqq add version and changelog creation step, and give a dab version matching repo version
         # - uses: actions/checkout@v4
         # - uses: databricks/setup-cli@main

.github/workflows/prod-pr.yml

@@ -24,24 +24,24 @@ jobs:
       - name: Validate Prod Target
         run: databricks bundle validate -t prod
 
-    check-staging-is-stable:
-      name: Check staging still stable before deploy to prod
-      runs-on: ubuntu-latest
-      environment: staging
-      env:
-        DATABRICKS_HOST: ${{ vars.DBX_HOST }}
-        DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
-        DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
-
-        - name: Set up Databricks CLI
-          uses: databricks/setup-cli@main
+  check-staging-is-stable:
+    name: Check staging still stable before deploy to prod
+    runs-on: ubuntu-latest
+    environment: staging
+    env:
+      DATABRICKS_HOST: ${{ vars.DBX_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Databricks CLI
+        uses: databricks/setup-cli@main
 
         # The notebook will actually run both as set up atm but maybe its ok for now
-        - name: Check Staging Integration Tests Before Allowing Prod Deploy
-          run: |
-            echo "running the integration yml test job integration_test_job.yml"
-            databricks bundle run -t staging run_integration_tests
+      - name: Check Staging Integration Tests Before Allowing Prod Deploy
+        run: |
+          echo "running the integration yml test job integration_test_job.yml"
+          databricks bundle run -t staging integration_test_job
 

.github/workflows/staging-cd.yml

@@ -56,4 +56,4 @@ jobs:
       - name: Check Staging Integration Tests After Deploy
         run: |
           echo "running the integration yml test job integration_test_job.yml"
-          databricks bundle run -t staging run_integration_tests
+          databricks bundle run -t staging integration_test_job

.github/workflows/staging-pr.yml

@@ -24,23 +24,23 @@ jobs:
       - name: Validate Staging Target
         run: databricks bundle validate -t staging
 
-    check-dev-is-stable:
-      name: Check dev still stable before deploy to staging
-      runs-on: ubuntu-latest
-      environment: dev
-      env:
-        DATABRICKS_HOST: ${{ vars.DBX_HOST }}
-        DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
-        DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
+  check-dev-is-stable:
+    name: Check dev still stable before deploy to staging
+    runs-on: ubuntu-latest
+    environment: dev
+    env:
+      DATABRICKS_HOST: ${{ vars.DBX_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-        - name: Set up Databricks CLI
-          uses: databricks/setup-cli@main
+      - name: Set up Databricks CLI
+        uses: databricks/setup-cli@main
 
         # The notebook will actually run both as set up atm but maybe its ok for now
-        - name: Check Dev Integration Tests Before Allowing Sing Deploy
-          run: |
-            echo "running the integration yml test job integration_test_job.yml"
-            databricks bundle run -t dev run_integration_tests
+      - name: Check Dev Integration Tests Before Allowing Sing Deploy
+        run: |
+          echo "running the integration yml test job integration_test_job.yml"
+          databricks bundle run -t dev run_integration_tests

databricks.yml

@@ -82,7 +82,7 @@ variables:
   staging_env_sp_id:
     default: "d588f2c8-0c0a-4ded-9da2-0663bf8dd994"
   prod_env_sp_id:
-    default: "my-sp-id-jfsdkjhfjsdhfkjh"
+    default: "811de200-7873-4ab3-a435-da3cddf13c36"
 
 # for external package maybe qqqq i dont think we will export there are poetry bundle example repos if we decide to
 # artifacts:
@@ -92,17 +92,18 @@ variables:
 
 # Deployment environments
 targets:
+  # qqqq should we have a personal section for deploying to own space ???
   dev: 
     # qqqq what is the process of developing and deploying dabs for own area.
     # qqqq would it actually be staging with features against folders !!!!???
-
+    # qqqq We do github checks against DEV and DEV would be analyst only env so do we actually want to deploy to dev shared workspace
     mode: development
     # Deploy under own user not service principle
     default: true
     # This is the the default deployment workspace
     workspace:
       # profile means in gets set values from [dev] section of databrickscfg useful on local machine, unneeded in databricks environment 
-      profile: dev
+      # profile: dev
       # specify hosts in the databricks.yml not config for clarity
       # dev databricks host
       host: https://adb-295718430158257.17.azuredatabricks.net
@@ -146,8 +147,8 @@ targets:
       # Staging storage account
       storage_account:  unifiedrptdeltalake
     permissions:
-      - user_name: philip.tate@nhs.net
-        level: CAN_MANAGE
+      #- user_name: philip.tate@nhs.net
+      #  level: CAN_MANAGE
       - group_name: staging_env_users
         level: CAN_VIEW
       - service_principal_name: "${var.staging_env_sp_id}" 
@@ -170,8 +171,8 @@ targets:
       # Prod storage account
       storage_account: unifiedrptdeltalake
     permissions:
-      - user_name: philip.tate@nhs.net
-        level: CAN_MANAGE
+      #- user_name: philip.tate@nhs.net
+      #  level: CAN_MANAGE
       - group_name: prod_env_users
         level: CAN_VIEW
       - service_principal_name: "${var.prod_env_sp_id}"