Skip to content

Commit fd44032

Browse files
Phil-NHSPhil-NHS
authored
Merge pull request #13 from TechnologyEnhancedLearning/staging-data-test-team-shared
Staging data test team shared
2 parents 8b2765a + bf4d41a commit fd44032

6 files changed

Lines changed: 107 additions & 52 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 53 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,68 @@
1-
# We want to run CI processes that can run independent of databricks as branch rules so that we dont # deploy at cost code that we already should know needs changing
2-
# such as linting, and unit test for python, maybe dab? verify
3-
# we run these on all pull request because if there is a hot fix it may not have passed through
4-
# staging for example
5-
# qqqq check this is up to date
61
name: CI - Pull Request Checks
72

8-
# Run CI on all pull requests
3+
# Run CI on all pull requests just incase of hot fixes
94
on:
105
pull_request:
116
branches:
127
- '**' # all branches
8+
workflow_dispatch:
139

1410
jobs:
15-
ci_checks:
16-
name: "Linting, Unit Tests, DAB Verify"
11+
lint:
12+
name: "Linting"
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout code
16+
uses: actions/checkout@v3
17+
18+
- name: Commit lint placeholder
19+
# TODO: Implement commitlint with config from TELBlazor reference
20+
run: |
21+
echo "WARNING: Commit lint step is currently a placeholder."
22+
echo "Reference project: TELBlazor"
23+
# exit 0 <-- Change to 0 if you don't want to block the rest of the pipe yet
24+
25+
pytest:
1726
runs-on: ubuntu-latest
1827

1928
steps:
20-
# Checkout code
21-
- name: Checkout repository
29+
- name: Check out repository
2230
uses: actions/checkout@v4
2331

24-
# Set up Python
25-
- name: Setup Python
32+
- name: Set up Python
2633
uses: actions/setup-python@v5
2734
with:
28-
python-version: "3.x"
29-
30-
# Install dependencies used for linting and unit tests
31-
- name: Install dependencies
32-
run: pip install -r requirements-dev.txt
33-
34-
# Run python unit tests
35-
- name: Run Unit Tests
36-
run: pytest tests/unit
37-
38-
# Run python lint
39-
# qqqq on example used flake8 instead
40-
# pyproject.toml will need configuring
41-
- name: Run Linting
42-
run: pylint src
43-
44-
# qqqq to do run commit lint step and put in commit lint config
45-
# see TELBlazor
46-
- name: Commit lint
47-
run: |
48-
echo "Commit lint not implemented"
49-
exit 1
50-
51-
# qqqq to do run version generation step and put in commit lint config
52-
# see TELBlazor
53-
- name: Version Generation Test Run
54-
run: |
55-
echo "Version test run not implemented"
56-
exit 1
35+
python-version: "3.10"
36+
37+
- name: Upgrade pip
38+
run: python -m pip install --upgrade pip
39+
40+
- name: Install project + test deps
41+
run: |
42+
pip install -r requirements-dev.txt
43+
pip install -e .
44+
- name: Run pytest againt unit tests (exclude Databricks tests)
45+
run: |
46+
pytest tests/unit-tests -m "not databricks" -v
47+
48+
dab-validate:
49+
name: "Early warning of dab issues"
50+
runs-on: ubuntu-latest
51+
# Just dev we are not deploying so dont need access to more important service principle
52+
# !!!! TODO !!!!! Should be dev but git needs the service principle putting into its environment
53+
environment: staging
54+
env:
55+
DATABRICKS_HOST: ${{ vars.DBX_HOST }}
56+
DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
57+
DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
58+
59+
steps:
60+
- name: Checkout code
61+
uses: actions/checkout@v4
62+
63+
- name: Set up Databricks CLI
64+
uses: databricks/setup-cli@main
65+
66+
- name: Validate Bundle
67+
# Am i forced to have a target its after databrickcfg but shouldnt be
68+
run: databricks bundle validate -t staging

‎.github/workflows/main.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Just a POC Manual Integration Tests - Staging
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
integration_tests_staging:
8+
name: "Deploy & Test in Staging"
9+
runs-on: ubuntu-latest
10+
# Using the staging environment for Service Principal secrets
11+
environment: staging
12+
13+
env:
14+
# Assume the DAB bundle is already initialized in the repo but this step would actually come after dab deployment unless it can be run against the file directory
15+
# Using Staging Service Principal credentials from Git Secrets/Vars
16+
# using staging here because i havent set up env vars in git for dev yet
17+
DATABRICKS_HOST: ${{ vars.DBX_HOST }}
18+
DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }}
19+
DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }}
20+
21+
steps:
22+
- name: Checkout code
23+
uses: actions/checkout@v4
24+
25+
- name: Set up Databricks CLI
26+
uses: databricks/setup-cli@main
27+
28+
# this step would come after dab deploy? or would run first dev dab?
29+
# i dont want a fail really and the code to be sat there known to be bad so running locally via a hook could be good pre-push?
30+
# The notebook will actually run both as set up atm but maybe its ok for now
31+
- name: Run Integration Tests
32+
run: |
33+
echo "running the integration yml test job integration_test_job.yml"
34+
databricks bundle run -t staging run_integration_tests

‎.github/workflows/manual-integration-test-poc.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# This yml is so we can tell dbx to run integration tests
2+
name: Trying to trigger dbx integration tests in dbx environ
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
deploy_target:
8+
description: 'Which DAB target to deploy to?'
9+
required: true
10+
default: 'dev'
11+
type: choice
12+
options:
13+
- dev
14+
- staging

‎.github/workflows/manual-trigger-test-poc.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ jobs:
2424
pip install -r requirements-dev.txt
2525
pip install -e .
2626
27-
- name: Run pytest (exclude Databricks tests)
27+
- name: Run pytest againt unit tests (exclude Databricks tests)
2828
run: |
29-
pytest -m "not databricks" -v
29+
pytest tests/unit-tests -m "not databricks" -v

‎requirements-dev.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
pytest
1+
pytest
2+
pyspark

‎tests/unit-tests/transformations/test_date_transformations.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,7 @@
2525
# return session
2626

2727

28-
@pytest.fixture(scope="session")
29-
def spark():
30-
"""Provide Spark session for all tests"""
31-
session = SparkSession.getActiveSession()
32-
if session is None:
33-
raise RuntimeError("No active Spark session found. Running in Databricks?")
34-
return session
28+
3529

3630

3731
@pytest.fixture(scope="function")
@@ -226,4 +220,4 @@ def test_working_days_values_are_reasonable(spark, sample_dataframe_multiple_mon
226220
assert 19 <= working_days <= 23, \
227221
f"Working days should be between 19-23, got {working_days} for {row['start_date']}"
228222

229-
print("✅ All working days values are reasonable")
223+
print("✅ All working days values are reasonable")

0 commit comments

Comments
 (0)