Querya-Desktop/.github/workflows/benchmark.yml at main · QueryaHub/Querya-Desktop · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: Benchmark

# Informational: compares frame timings of benchmark/grid_perf_bench.dart on the
# PR against its base branch, three interleaved runs per side, and comments when
# the Dart build time is slower in every pair (scripts/ci/bench_compare.py).
# Never blocks a merge.

on:
  pull_request:
    branches: [ dev, main ]
    paths:
      - 'lib/features/workspace/**'
      - 'lib/core/database/**'
      - 'lib/core/csv/**'
      - 'lib/features/erd/**'
      - 'lib/features/results/**'
      - 'benchmark/**'
      - 'scripts/ci/bench_compare.py'
      - 'scripts/ci/app_bench_report.py'
      - '.github/workflows/benchmark.yml'
  workflow_dispatch:

permissions:
  contents: read
  pull-requests: write

concurrency:
  group: benchmark-${{ github.ref }}
  cancel-in-progress: true

jobs:
  grid-bench:
    name: Grid performance
    # Forks get no write token and must never reach any self-hosted runner.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 75
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Linux dependencies
        timeout-minutes: 8
        run: |
          sudo apt-get -o Acquire::Retries=3 update
          sudo apt-get -o Acquire::Retries=3 install -y clang cmake ninja-build pkg-config libgtk-3-dev libsecret-1-dev libsqlite3-dev xvfb libgl1-mesa-dri

      - uses: subosito/flutter-action@v2
        with:
          flutter-version: '3.47.1'
          channel: 'stable'
          cache: true

      - name: Prepare the PR and the base checkouts
        run: |
          flutter pub get
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            git worktree add "$RUNNER_TEMP/base" "origin/${{ github.base_ref }}"
            # The base runs the PR's benchmark, so both sides measure the same thing.
            rm -rf "$RUNNER_TEMP/base/benchmark"
            cp -r benchmark "$RUNNER_TEMP/base/benchmark"
            (cd "$RUNNER_TEMP/base" && flutter pub get)
          fi

      # Base and PR alternate (base, PR, base, PR, base, PR), so a slow
      # stretch of the shared runner lands on both sides.
      - name: Run the benchmark, 3 interleaved runs per side
        timeout-minutes: 40
        run: |
          bench() {
            (cd "$1" && xvfb-run -a -s '-screen 0 1920x1080x24' flutter run --profile -d linux \
              -t benchmark/grid_perf_bench.dart --dart-define=MODE=scroll \
              2>&1 | tee "$2") || true
          }
          for i in 1 2 3; do
            if [ "${{ github.event_name }}" = "pull_request" ]; then
              bench "$RUNNER_TEMP/base" "$RUNNER_TEMP/base$i.log"
            fi
            bench "$GITHUB_WORKSPACE" "$RUNNER_TEMP/head$i.log"
          done

      - name: Compare
        id: compare
        run: |
          python3 scripts/ci/bench_compare.py \
            --base "$RUNNER_TEMP"/base*.log --head "$RUNNER_TEMP"/head*.log \
            > "$RUNNER_TEMP/report.txt"
          cat "$RUNNER_TEMP/report.txt"
          echo "regression=$(head -1 "$RUNNER_TEMP/report.txt" | cut -d= -f2)" >> "$GITHUB_OUTPUT"
          tail -n +2 "$RUNNER_TEMP/report.txt" > "$RUNNER_TEMP/report.md"
          cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"

      - name: Comment on regression
        if: github.event_name == 'pull_request' && steps.compare.outputs.regression == '1'
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh pr comment "${{ github.event.pull_request.number }}" --body-file "$RUNNER_TEMP/report.md"

  # #1173: ERD, charts / groupings and multi-tab SQL scenarios of the real app
  # (benchmark/app_perf_bench.dart, Demo Playground, no database server),
  # checked against per-scenario limits. Informational, like the grid bench.
  app-bench:
    name: App scenarios performance
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 30
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4

      - name: Install Linux dependencies
        timeout-minutes: 8
        run: |
          sudo apt-get -o Acquire::Retries=3 update
          sudo apt-get -o Acquire::Retries=3 install -y clang cmake ninja-build pkg-config libgtk-3-dev libsecret-1-dev libsqlite3-dev xvfb libgl1-mesa-dri

      - uses: subosito/flutter-action@v2
        with:
          flutter-version: '3.47.1'
          channel: 'stable'
          cache: true

      - name: Run the app scenarios
        timeout-minutes: 20
        run: |
          flutter pub get
          xvfb-run -a -s '-screen 0 1920x1080x24' flutter run --profile -d linux \
            -t benchmark/app_perf_bench.dart \
            --dart-define=ONLY=charts_5000,groupings_5000,multi_tab_run,erd_open_200,erd_hover,erd_drag,erd_auto_layout \
            --dart-define=STREAMS= \
            2>&1 | tee "$RUNNER_TEMP/app.log" || true

      - name: Report
        run: |
          python3 scripts/ci/app_bench_report.py "$RUNNER_TEMP/app.log" > "$RUNNER_TEMP/app_report.txt"
          cat "$RUNNER_TEMP/app_report.txt"
          tail -n +2 "$RUNNER_TEMP/app_report.txt" >> "$GITHUB_STEP_SUMMARY"