DatabricksPOC/.github/workflows/staging-cd.yml at main · TechnologyEnhancedLearning/DatabricksPOC · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
# Rely on ci.yml and branch rules to ensure bundle validation and linting
# we are not going from staging straight to prod, because we use staging for manual testing as well
# [use as a ref](https://github.com/evanaze/dbx-asset-bundle-deployment/blob/main/.github/workflows/dev-ci.yml)
name: Deploy to Databricks Staging and Trigger Staging Tests in Databricks

on:
  push:
    branches:
      - staging-data-test-team-shared
    #  - staging #longer name for communication in the poc reduce when using it

jobs:
  deploy_to_staging:
    name: "Deploy Bundle to Stage Environment"
    runs-on: ubuntu-latest
    environment: staging
    env:
      DATABRICKS_HOST: ${{ vars.DBX_HOST }}
      DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
      DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Databricks CLI
        uses: databricks/setup-cli@main

      # Just incase somethings changed there is no harm in it
      - name: Validate Staging Bundle
      # trigger target the staging deploy in databricks.yml
        run: databricks bundle validate -t staging #--profile ""  # forces use of env vars only # didnt work

      - name: Deploy Staging Bundle
      # trigger target the staging deploy in databricks.yml
        run: databricks bundle deploy -t staging --auto-approve #--profile ""  # forces use of env vars only #??--auto-approve # didnt work
       # working-directory: .

  check-staging-is-stable:
    name: Check staging stable after deploy
    needs: deploy_to_staging
    runs-on: ubuntu-latest
    environment: staging
    env:
      DATABRICKS_HOST: ${{ vars.DBX_HOST }}
      DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
      DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Databricks CLI
        uses: databricks/setup-cli@main

      # The notebook will actually run both as set up atm but maybe its ok for now
      - name: Check Staging Integration Tests After Deploy
        run: |
          echo "running the integration yml test job integration_test_job.yml"
          databricks bundle run -t staging integration_test_job