diff --git a/.github/actions/check-pr-ready-to-test/action.yml b/.github/actions/check-pr-ready-to-test/action.yml new file mode 100644 index 0000000000000..4b588c9a265f5 --- /dev/null +++ b/.github/actions/check-pr-ready-to-test/action.yml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: check PR readiness for running CI +description: > + Fails the workflow when a pull request isn't ready for consuming the shared CI resources of the + apache/pulsar repository. Draft pull requests and stacked pull requests which aren't at the bottom + of the stack are expected to be tested in the contributor's own fork. The check is overridden by + adding the ready-to-test label to the pull request. +inputs: + github-token: + description: "Token used for reading the pull request state" + required: false + default: ${{ github.token }} + ready-to-test-label: + description: "Label which overrides the readiness check" + required: false + default: 'ready-to-test' + trunk-branches: + description: > + Comma separated glob patterns of the branches that are considered trunk branches. Used for + detecting stacked pull requests when the GitHub stack API isn't available. + required: false + default: 'master,branch-*,pulsar-*' +runs: + using: composite + steps: + - uses: actions/github-script@v8 + env: + # github.action_path is passed in an environment variable since a relative require would be + # resolved against the github-script action's own directory instead of this action's directory + ACTION_PATH: ${{ github.action_path }} + READY_TO_TEST_LABEL: ${{ inputs.ready-to-test-label }} + TRUNK_BRANCHES: ${{ inputs.trunk-branches }} + with: + github-token: ${{ inputs.github-token }} + script: | + const checkPrReadyToTest = require(`${process.env.ACTION_PATH}/check-pr-ready-to-test.js`); + await checkPrReadyToTest({ github, context, core }); diff --git a/.github/actions/check-pr-ready-to-test/check-pr-ready-to-test.js b/.github/actions/check-pr-ready-to-test/check-pr-ready-to-test.js new file mode 100644 index 0000000000000..d3c66f0a758e9 --- /dev/null +++ b/.github/actions/check-pr-ready-to-test/check-pr-ready-to-test.js @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const DEFAULT_READY_TO_TEST_LABEL = 'ready-to-test'; +const DEFAULT_TRUNK_BRANCHES = 'master,branch-*,pulsar-*'; + +// GitHub stacked pull requests: https://docs.github.com/en/pull-requests/how-tos/stacked-pull-requests +// The entries keep their position when a pull request of the stack is merged, so the bottom of the +// stack is the lowest entry which is still open, not necessarily the entry at position 1. +const STACK_QUERY = ` + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $number) { + stackEntry { + position + } + stack { + number + size + baseRefName + entries(first: 100) { + nodes { + position + pullRequest { + number + url + state + } + } + } + } + } + } + }`; + +function parsePatterns(value) { + return (value || '').split(/[\s,]+/).filter(pattern => pattern.length > 0); +} + +function matchesAnyPattern(branch, patterns) { + return patterns.some(pattern => { + const regex = new RegExp(`^${pattern.split('*').map(escapeRegExp).join('.*')}$`); + return regex.test(branch); + }); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Resolves the place of the pull request within a GitHub stack. + * The stack fields aren't available in all GitHub deployments, in that case `available` is false and + * the caller falls back to inspecting the base branch of the pull request. + */ +async function resolveStack({ github, core, owner, repo, number, baseRef }) { + let response; + try { + response = await github.graphql(STACK_QUERY, { owner, repo, number }); + } catch (error) { + core.warning(`Couldn't resolve GitHub stack information for #${number}: ${error.message}`); + return { available: false }; + } + const stack = response?.repository?.pullRequest?.stack; + if (!stack) { + return { available: true, inStack: false }; + } + const entries = (stack.entries?.nodes || []).filter(entry => entry?.pullRequest); + const openEntries = entries.filter(entry => entry.pullRequest.state === 'OPEN'); + const bottomEntry = openEntries.reduce( + (bottom, entry) => (bottom == null || entry.position < bottom.position ? entry : bottom), null); + const position = response.repository.pullRequest.stackEntry?.position; + // The pull request is at the bottom of the stack when every pull request below it has been merged + // or closed. When such a pull request is merged, GitHub retargets the one above it to the branch + // that was merged into, so targeting the trunk branch of the stack means the same thing. Either + // condition is enough: the retargeting and the stack entries aren't necessarily updated at once. + const isLowestOpen = position != null + ? !entries.some(entry => entry.position < position && entry.pullRequest.state === 'OPEN') + : bottomEntry?.pullRequest?.number === number; + return { + available: true, + inStack: true, + isBottom: isLowestOpen || baseRef === stack.baseRefName, + position, + number: stack.number, + size: stack.size, + trunkBranch: stack.baseRefName, + bottomPullRequest: bottomEntry?.pullRequest + }; +} + +function renderSummary({ pullRequestUrl, blockers, label }) { + const steps = [ + ...blockers.map(blocker => blocker.remedy), + `Test the change in your own fork in the meantime. The full CI pipeline runs in a fork without ` + + `maintainer approval and GitHub Actions provides separate quota for it. See the ` + + `[Personal CI documentation](https://pulsar.apache.org/contribute/personal-ci/) for enabling ` + + `it: push the branch to your fork and let the CI run against the pull request opened in your ` + + `own fork. As the pull request author, you are responsible for following up on test failures. ` + + `Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues after ` + + `checking that the flaky test isn't already reported.`, + `An Apache Pulsar committer can add the \`${label}\` label to ${pullRequestUrl} to run the CI in ` + + `apache/pulsar regardless of the checks above.`, + `This workflow doesn't restart on its own when the pull request is marked as ready for review, ` + + `when the \`${label}\` label is added or when the pull request below this one in a stack is ` + + `merged. Once the checks above are addressed, start a new run by pushing to the branch, by ` + + `adding a "/pulsarbot rerun" comment to the pull request, or by re-running the failed jobs in ` + + `the GitHub Actions UI.` + ]; + return ` +## Pulsar CI didn't run for this pull request + +The apache/pulsar CI based on GitHub Actions has constrained resources and quota which are shared by +all contributors, so CI in apache/pulsar is reserved for pull requests that are ready to be tested: +draft pull requests, and the pull requests of a stack above the bottom one, are expected to be tested +with [Personal CI](https://pulsar.apache.org/contribute/personal-ci/) in the contributor's own fork. + +### Why this run was stopped + +${blockers.map(blocker => `- ${blocker.reason}`).join('\n')} + +### How to proceed + +${steps.map((step, index) => `${index + 1}. ${step}`).join('\n')} + +If you have any trouble you can get support in multiple ways: +* by sending email to the [dev mailing list](mailto:dev@pulsar.apache.org) ([subscribe](mailto:dev-subscribe@pulsar.apache.org)) +* on the [#dev channel on Pulsar Slack](https://apache-pulsar.slack.com/channels/dev) ([join](https://pulsar.apache.org/community#section-discussions)) +* in apache/pulsar [GitHub discussions Q&A](https://github.com/apache/pulsar/discussions/categories/q-a) +`; +} + +module.exports = async ({ github, context, core }) => { + const eventPullRequest = context.payload.pull_request; + if (!eventPullRequest) { + core.info(`The '${context.eventName}' event isn't a pull request event, skipping the check.`); + return; + } + const { owner, repo } = context.repo; + const number = eventPullRequest.number; + const label = process.env.READY_TO_TEST_LABEL || DEFAULT_READY_TO_TEST_LABEL; + const trunkBranches = parsePatterns(process.env.TRUNK_BRANCHES || DEFAULT_TRUNK_BRANCHES); + + // The event payload is a snapshot of the pull request from the time the workflow run was triggered. + // Refresh the state so that re-running the workflow picks up changes made after that, such as + // adding the label or marking the pull request as ready for review. + const { data: pullRequest } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + + if ((pullRequest.labels || []).some(prLabel => prLabel.name === label)) { + core.info(`Found the '${label}' label on #${number}.`); + return; + } + core.info(`There is no '${label}' label on #${number}.`); + + // Each blocker explains why the CI didn't run and what to do about it. The remedies are rendered as + // the first steps of the instructions so that they match the checks which actually failed. + const stackRemedy = 'Wait for this pull request to reach the bottom of the stack: once the pull ' + + 'requests below it have been merged or closed, it becomes the lowest open pull request of the ' + + 'stack and its CI runs in apache/pulsar.'; + const blockers = []; + if (pullRequest.draft) { + blockers.push({ + reason: 'The pull request is a **draft**, so it isn\'t ready to be reviewed and tested yet.', + remedy: 'Mark the pull request as ready for review once it is ready to be tested and reviewed.' + }); + } + + const stack = await resolveStack({ github, core, owner, repo, number, baseRef: pullRequest.base.ref }); + if (stack.available && stack.inStack) { + const positionText = stack.position != null + ? `entry ${stack.position} of ${stack.size} in stack #${stack.number}` + : `part of stack #${stack.number}`; + core.info(`#${number} is ${positionText}. At the bottom of the stack: ${stack.isBottom}.`); + if (!stack.isBottom) { + const bottom = stack.bottomPullRequest; + const bottomLink = bottom ? ` ([#${bottom.number}](${bottom.url}))` : ''; + blockers.push({ + reason: `The pull request is ${positionText}. Only the pull request at the bottom of the ` + + `stack, that is the lowest one which hasn't been merged or closed yet${bottomLink}, runs ` + + `the CI in apache/pulsar.`, + remedy: stackRemedy + }); + } + } else if (!matchesAnyPattern(pullRequest.base.ref, trunkBranches)) { + // Not resolved as a GitHub stack: a pull request that targets a branch which isn't a trunk branch + // is a dependent pull request stacked on top of another one. + core.info(`#${number} targets the '${pullRequest.base.ref}' branch which isn't a trunk branch.`); + blockers.push({ + reason: `The pull request targets the \`${pullRequest.base.ref}\` branch instead of a trunk ` + + `branch (${trunkBranches.map(pattern => `\`${pattern}\``).join(', ')}), so it is stacked on ` + + `top of another pull request. Only the pull request at the bottom of a stack runs the CI in ` + + `apache/pulsar.`, + remedy: stackRemedy + }); + } + + if (blockers.length === 0) { + core.info(`#${number} is ready for running the CI.`); + return; + } + + await core.summary + .addRaw(renderSummary({ pullRequestUrl: eventPullRequest.html_url, blockers, label })) + .write(); + core.setFailed(`#${number} isn't ready for running the CI in ${owner}/${repo}. ` + + `See the job summary for instructions on how to proceed.`); +}; diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 772b8521763b8..7fc882020823f 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -57,6 +57,10 @@ jobs: echo docs_only=false >> $GITHUB_OUTPUT fi + - name: Check if the PR is ready for running CI + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/check-pr-ready-to-test + check-style: needs: preconditions if: ${{ needs.preconditions.outputs.docs_only != 'true' }} diff --git a/.github/workflows/ci-python-functions.yaml b/.github/workflows/ci-python-functions.yaml new file mode 100644 index 0000000000000..d02ed5303ebde --- /dev/null +++ b/.github/workflows/ci-python-functions.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Python Functions +on: + pull_request: + branches: + - master + paths: + - '.github/workflows/**' + - 'pulsar-functions/instance/src/main/python/**' + - 'pulsar-functions/instance/src/test/python/**' + - 'pulsar-functions/instance/src/scripts/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + preconditions: + name: Preconditions + runs-on: ubuntu-24.04 + outputs: + docs_only: ${{ steps.check_changes.outputs.docs_only }} + steps: + - name: checkout + uses: actions/checkout@v6 + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + if [[ "${GITHUB_EVENT_NAME}" != "schedule" ]]; then + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + else + echo docs_only=false >> $GITHUB_OUTPUT + fi + + - name: Check if the PR is ready for running CI + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/check-pr-ready-to-test + + instance-tests: + needs: preconditions + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + name: Python ${{ matrix.python-version }} Functions instance tests + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ['3.12', '3.13'] + + steps: + - name: checkout + uses: actions/checkout@v6 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Run Python instance tests + run: | + ./pulsar-functions/instance/src/scripts/run_python_instance_tests.sh diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 105fa508b49fb..6e18be2060cb2 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -44,6 +44,7 @@ on: options: - '25' - '21' + - '26' default: '25' trace_test_resource_cleanup: description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' @@ -134,6 +135,10 @@ jobs: echo docs_only=false >> $GITHUB_OUTPUT fi + - name: Check if the PR is ready for running CI + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/check-pr-ready-to-test + - name: Check if coverage should be collected id: check_coverage run: | diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 68feda3776a1b..6e0c0e2335172 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -39,6 +39,7 @@ on: options: - '25' - '21' + - '26' default: '25' trace_test_resource_cleanup: description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' @@ -127,6 +128,10 @@ jobs: echo docs_only=false >> $GITHUB_OUTPUT fi + - name: Check if the PR is ready for running CI + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/check-pr-ready-to-test + - name: Set Netty leak detection mode id: netty_leak_detection run: | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cc064d1e91a2c..c5d4d6f77987c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -92,7 +92,7 @@ concurrency model. ## Build infrastructure Apache Pulsar uses a **Gradle** build (migrated from Maven via PIP-463; some older tooling and docs -elsewhere still reference Maven). The wrapper `./gradlew` requires **JDK 21 or 25** (bytecode targets +elsewhere still reference Maven). The wrapper `./gradlew` requires **JDK 21, 25 or 26** (bytecode targets Java 17). See [`CONTRIBUTING.md` → Building](CONTRIBUTING.md#building) for the build and lint commands. - `settings.gradle.kts` — all modules, organized in dependency tiers (Tier 0 has no internal deps, @@ -133,7 +133,10 @@ When editing `build-logic/`, `settings.gradle.kts`, a module `build.gradle.kts`, sources, and verify with `--configuration-cache`. Tasks reached by the common flows (`assemble`, `test`, `integrationTest`, `rat` / `spotlessCheck` / `checkstyle*`, `checkBinaryLicense`, `docker*`) must be compatible; one-off tooling tasks not part of those flows (e.g. `verifyTestGroups`, ad-hoc - report tasks) may be exempt. + report tasks) may be exempt. When a third-party plugin is incompatible and cannot be fixed locally, + opt its tasks out with `notCompatibleWithConfigurationCache("")` (see + `tests/pulsar-client-native-image/build.gradle.kts` for the GraalVM native-build-tools case) so the + build degrades to running without the cache instead of failing. - **Published modules must not depend on internal modules** at compile/runtime scope — the artifact would be unresolvable from Maven Central. A module is published only when it applies `pulsar.public-java-library-conventions`. diff --git a/CODING.md b/CODING.md index 892b5a051687a..75dace9439a46 100644 --- a/CODING.md +++ b/CODING.md @@ -236,6 +236,8 @@ defaulting to the safe/old behaviour. - **Back optimizations with evidence** — a JMH benchmark (see *Testing conventions*) or a profile, not intuition — measured on JIT-warmed code (see *Reproducing concurrency / memory-visibility bugs*). + `-PtestAsyncProfiler` profiles a test run with async-profiler (see + [`CONTRIBUTING.md`](CONTRIBUTING.md#profiling-tests-with-async-profiler)). - **On hot paths** (dispatch, IO, per-message): avoid `String.format` (build strings directly), `Enum.values()` (match explicitly), and unnecessary allocation/locking; prefer lock-free or single-writer designs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88b0cfa844708..6252033fca338 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ workflow (build, test, PR, CI). For the big-picture module map and the Gradle bu ## Building -**JDK 21 or 25** is required to build `master` (bytecode targets Java 17; `-PskipJavaVersionCheck` +**JDK 21, 25 or 26** is required to build `master` (bytecode targets Java 17; `-PskipJavaVersionCheck` bypasses the check); `zip` is also needed. Use the bundled wrapper `./gradlew` (Linux/macOS) or `gradlew.bat` (Windows) — no separate Gradle install. See the [build-tooling setup guide](https://pulsar.apache.org/contribute/setup-buildtools/) and the @@ -109,6 +109,184 @@ Failed tests are retried once by default (`testRetryCount=1`; `0` when running i running tests locally, prefer **`-PtestRetryCount=0`** to catch failures (including flakiness) early instead of having retries mask them. +### Micro benchmarks (JMH) + +For a **micro**-level question — what a single method, data structure or codec costs — write a +[JMH](https://openjdk.org/projects/code-tools/jmh/) benchmark under `microbench/`. That is the +preferred tool for micro benchmarks: JMH handles warm-up, dead-code elimination and measurement, none +of which hand-written timing code gets right. [`CODING.md`](CODING.md#performance) asks for +optimizations to be backed by evidence, and for a small self-contained change a benchmark is the +strongest kind. + +```bash +./gradlew :microbench:shadowJar # build the runnable jar +java -jar microbench/build/libs/microbench-*-benchmarks.jar -l # list the benchmarks +java -jar microbench/build/libs/microbench-*-benchmarks.jar ".*.*" # run the ones that match +``` + +A benchmark can be profiled directly through JMH's async-profiler integration, which writes forward +and reverse flame graphs per benchmark under `dir=`: + +```bash +export LIBASYNCPROFILER_PATH=$(ls $JAVA_HOME/lib/libasyncProfiler.*) # Corretto ships one +java -jar microbench/build/libs/microbench-*-benchmarks.jar \ + -prof async:libPath=$LIBASYNCPROFILER_PATH\;output=flamegraph\;dir=profile-results ".*.*" +``` + +See [`microbench/README.md`](microbench/README.md) for the rest: the async-profiler setup when the +JDK does not ship one, recording a benchmark to JFR and rendering it, JSON result files for +[JMH Visualizer](https://jmh.morethan.io/), and the `rawCommand` escape hatch for async-profiler +options the JMH plugin does not expose. + +### Profiling tests with async-profiler + +Profiling a *test run* answers the question a micro benchmark cannot: not what one method costs, but +where a broker actually spends its time end to end. + +> **When the numbers matter, profile and benchmark on Linux x86_64.** That is Pulsar's most common +> deployment target, and results from elsewhere do not carry over. Two differences bite in +> particular: async-profiler supports only some of its sampling engines on macOS, so a profile taken +> there is less reliable; and `System.nanoTime()` is far more expensive on macOS than on Linux, which +> skews the results in some cases — code that times itself, and JMH's own measurement loop, both pay +> that cost. Working on macOS or arm64 is fine for finding your way around the code — just treat what +> it tells you as provisional until it is confirmed on Linux x86_64. + +#### Profiling a module's tests + +Pass **`-PtestAsyncProfiler`** to run a test task under +[async-profiler](https://github.com/async-profiler/async-profiler). The build finds the profiler +library in `LIBASYNCPROFILER_PATH`, and falls back to the copy that Amazon Corretto ships inside the +JDK that runs the tests, so on Corretto no setup is needed at all: + +```bash +# only needed when the JDK does not ship async-profiler +export LIBASYNCPROFILER_PATH=$(ls $JAVA_HOME/lib/libasyncProfiler.*) +./gradlew :pulsar-broker:test -PtestAsyncProfiler --tests "" +``` + +Enabling it runs the tests in a single JVM with retries off and a pre-touched fixed-size heap, and +sends log4j output to a file, so that one run produces one profile that isn't distorted by console +logging or by heap resizing. The task always re-runs and is never cached. **Manual tests are enabled +automatically** whenever profiling is on — the long-running, load-generating tests that are otherwise +skipped are usually the ones worth profiling — so `ENABLE_MANUAL_TEST` does not have to be exported. Recordings are written to +`build/test-profiles/` in the repository root, named after the test task and stamped with the start +time and the pid — for example `test_profile_pulsar-broker-test_20260904-114040_23420.jfr`, next to +`test_profile_pulsar-broker-test.log`. See [Analyzing a JFR file](#analyzing-a-jfr-file) below. + +The defaults can be tuned with `-Ptest.asyncprofiler.event=` (the CPU sampling engine: +`cpu` on Linux, `itimer` elsewhere — see +[CPU sampling engines](https://github.com/async-profiler/async-profiler/blob/master/docs/CpuSamplingEngines.md)), +`-Ptest.asyncprofiler.opts=` (default `event=,all,alloc=2m,jfrsync=profile`), +`-Ptest.asyncprofiler.outputformat=jfr|html|collapsed`, `-Ptest.asyncprofiler.dir=` (use it to +keep the profiles of an A/B comparison apart) and `-Ptest.asyncprofiler.libpath=`. + +`outputformat` only sets the file extension; the format itself comes from the agent options, and the +default `jfrsync=profile` forces a JFR recording. To get something other than JFR, override both — to +profile exception creation, for example (Linux only): + +```bash +./gradlew :pulsar-broker:test -PtestAsyncProfiler --tests "" \ + "-Ptest.asyncprofiler.opts=event=Java_java_lang_Throwable_fillInStackTrace,tree,reverse" \ + -Ptest.asyncprofiler.outputformat=html +``` + +#### Profiling an integration-test cluster + +The module tests above profile the JVM the tests run in. To profile the **broker, bookies and +ZooKeeper inside the containers** of a real cluster, run: + +```bash +./gradlew :tests:integration:profilingIntegrationTest +``` + +That one task does everything the profiling run needs: it builds +`apachepulsar/java-test-image:latest-asyncprofiler` (the test image with async-profiler installed — +kept under its own tag so it never replaces the image the other integration tests use), relaxes the +kernel `perf_event` limits that the `cpu` sampling engine needs by way of a privileged throwaway +container, and runs +[`PulsarProfilingTest`](tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java) +against it with retries off. That test drives `pulsar-perf` against a single broker. + +**Any other integration test can be profiled without being modified**, by pointing the task at it and +naming the cluster components to attach the profiler to: + +```bash +./gradlew :tests:integration:profilingIntegrationTest --tests "" \ + -Pinttest.asyncprofiler.components=broker,bookie +``` + +`components` takes `broker`, `proxy`, `functionworker`, `bookie`, `zookeeper`, or `all`, and defaults +to `broker` for this task. It is what `PulsarClusterSpec.profileBroker` and its siblings fall back to, +so it has no effect on a test that sets those flags itself — `PulsarProfilingTest` does, which is why +it profiles the broker whatever you pass. Every other test leaves them at their default of off, so +without this property nothing would be profiled. Setting it also enables manual tests, so +`-Pinttest.asyncprofiler.components=<...>` profiles a cluster through the plain `integrationTest` +task too. + +Recordings land in `tests/integration/build/`, named `inttest_profile__