Repository navigation
117 lines (102 loc) · 3.84 KB
/
Copy pathperformance.yml
File metadata and controls
117 lines (102 loc) · 3.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
name: Performance
on:
pull_request:
workflow_dispatch:
permissions:
contents: read
pull-requests: write
concurrency:
group: perf-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
benchmark:
name: Benchmark distro overhead
runs-on: ubuntu-latest
timeout-minutes: 40
env:
PERF_REGRESSION_THRESHOLD: "15"
steps:
- name: Check out PR branch
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: pr
fetch-depth: 0
- name: Check out base branch
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.base.sha || github.event.repository.default_branch }}
path: base
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
# ------------------------------------------------------------------
# Candidate (PR) run
# ------------------------------------------------------------------
- name: Install candidate (PR) distro
working-directory: pr
run: |
python -m pip install --upgrade pip
python -m pip install .
python -m pip install "pytest>=8.0" "pytest-benchmark>=4.0"
- name: Run candidate benchmarks
working-directory: pr
run: |
python -m pytest tests/perf \
--benchmark-only \
--benchmark-min-rounds=5 \
--benchmark-warmup=on \
--benchmark-json="$GITHUB_WORKSPACE/pr.json"
# ------------------------------------------------------------------
# Baseline (base branch) run.
# We always invoke the PR copy of the perf tests (tests/perf may not
# exist on base) but install the *base* distro into the same env first
# so the `microsoft.opentelemetry` import resolves to baseline code.
# ------------------------------------------------------------------
- name: Install baseline distro
working-directory: base
run: |
python -m pip install --force-reinstall --no-deps .
python -m pip install "pytest>=8.0" "pytest-benchmark>=4.0"
- name: Run baseline benchmarks
working-directory: pr
run: |
python -m pytest tests/perf \
--benchmark-only \
--benchmark-min-rounds=5 \
--benchmark-warmup=on \
--benchmark-json="$GITHUB_WORKSPACE/base.json"
# ------------------------------------------------------------------
# Compare + report
# ------------------------------------------------------------------
- name: Compare results
id: compare
working-directory: pr
run: |
set +e
python -m perf.compare \
--baseline "$GITHUB_WORKSPACE/base.json" \
--candidate "$GITHUB_WORKSPACE/pr.json" \
--threshold "$PERF_REGRESSION_THRESHOLD" \
--output "$GITHUB_WORKSPACE/report.md"
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
- name: Record PR number
if: always() && github.event_name == 'pull_request'
run: echo "${{ github.event.pull_request.number }}" > "$GITHUB_WORKSPACE/pr-number.txt"
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: perf-results
path: |
pr.json
base.json
report.md
pr-number.txt
if-no-files-found: ignore
- name: Fail on regression
if: steps.compare.outputs.exit_code != '0'
run: |
echo "Gating perf scenario regressed beyond ${PERF_REGRESSION_THRESHOLD}%"
exit 1