From d18e42d36dc7b2c9359e3fda4920fe3f65f1986a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Apr 2026 20:58:37 +0300 Subject: [PATCH 01/33] [improve][build] Improve the gradle build to use similar bestpractices as apache/pulsar build --- .github/actions/setup-gradle/action.yml | 71 ++++++ .github/actions/tune-runner-vm/action.yml | 80 ++++++ .github/workflows/ci.yaml | 39 ++- README.md | 37 ++- aerospike/build.gradle.kts | 3 +- alluxio/build.gradle.kts | 3 +- aws/build.gradle.kts | 4 + azure-data-explorer/build.gradle.kts | 3 +- build-logic/conventions/build.gradle.kts | 34 +++ ...ectors.code-quality-conventions.gradle.kts | 41 +++ ...sar-connectors.java-conventions.gradle.kts | 163 ++++++++++++ ...lsar-connectors.nar-conventions.gradle.kts | 93 +++++++ ...r-connectors.shadow-conventions.gradle.kts | 57 +++++ build-logic/settings.gradle.kts | 32 +++ build.gradle.kts | 233 +++--------------- canal/build.gradle.kts | 3 +- cassandra/build.gradle.kts | 3 +- debezium/core/build.gradle.kts | 3 + debezium/mongodb/build.gradle.kts | 3 +- debezium/mssql/build.gradle.kts | 3 +- debezium/mysql/build.gradle.kts | 3 +- debezium/oracle/build.gradle.kts | 3 +- debezium/postgres/build.gradle.kts | 3 +- distribution/io/build.gradle.kts | 6 +- docker/pulsar-all/build.gradle.kts | 6 +- docs/build.gradle.kts | 4 + dynamodb/build.gradle.kts | 3 +- elastic-search/build.gradle.kts | 3 +- file/build.gradle.kts | 3 +- gradle.properties | 1 + gradle/libs.versions.toml | 149 ++++------- hbase/build.gradle.kts | 3 +- hdfs3/build.gradle.kts | 3 +- http/build.gradle.kts | 3 +- influxdb/build.gradle.kts | 3 +- jdbc/clickhouse/build.gradle.kts | 3 +- jdbc/core/build.gradle.kts | 4 + jdbc/mariadb/build.gradle.kts | 3 +- jdbc/openmldb/build.gradle.kts | 3 +- jdbc/postgres/build.gradle.kts | 3 +- jdbc/sqlite/build.gradle.kts | 3 +- kafka-connect-adaptor-nar/build.gradle.kts | 3 +- kafka-connect-adaptor/build.gradle.kts | 3 + kafka/build.gradle.kts | 3 +- kinesis-kpl-shaded/build.gradle.kts | 12 +- kinesis/build.gradle.kts | 7 +- mongo/build.gradle.kts | 3 +- netty/build.gradle.kts | 3 +- nsq/build.gradle.kts | 3 +- .../build.gradle.kts | 23 +- rabbitmq/build.gradle.kts | 3 +- redis/build.gradle.kts | 3 +- settings.gradle.kts | 11 +- solr/build.gradle.kts | 5 +- src/license-header.txt | 16 ++ 55 files changed, 850 insertions(+), 371 deletions(-) create mode 100644 .github/actions/setup-gradle/action.yml create mode 100644 .github/actions/tune-runner-vm/action.yml create mode 100644 build-logic/conventions/build.gradle.kts create mode 100644 build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts create mode 100644 build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts create mode 100644 build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts create mode 100644 build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts create mode 100644 build-logic/settings.gradle.kts rename {pulsar-dependencies => pulsar-connectors-dependencies}/build.gradle.kts (56%) create mode 100644 src/license-header.txt diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml new file mode 100644 index 0000000000..ceac384023 --- /dev/null +++ b/.github/actions/setup-gradle/action.yml @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Setup Gradle +description: Sets up Gradle with Develocity or public build scan publishing by default +inputs: + develocity-access-key: + description: 'Develocity access key for authenticated build scans' + required: false + default: '' + build-scan-publish: + description: 'Whether to publish build scans or use Develocity when the access key is set' + required: false + default: 'true' + cache-read-only: + description: 'Whether the Gradle cache is read-only' + required: false + default: ${{ github.event.repository != null && github.ref_name != github.event.repository.default_branch }} + add-job-summary: + description: 'When to add a job summary' + required: false + default: 'always' +runs: + using: composite + steps: + - name: Set Develocity Project ID and configure custom settings + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + shell: bash + run: | + mkdir -p ~/.gradle + touch ~/.gradle/gradle.properties + grep -q 'systemProp.develocity.projectId=' ~/.gradle/gradle.properties || echo systemProp.develocity.projectId=pulsar >> ~/.gradle/gradle.properties + grep -q 'systemProp.scan.uploadInBackground=' ~/.gradle/gradle.properties || echo systemProp.scan.uploadInBackground=false >> ~/.gradle/gradle.properties + + - name: Setup Gradle with Develocity + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f + with: + develocity-injection-enabled: true + develocity-url: https://develocity.apache.org + # expected format is develocity.apache.org: + develocity-access-key: ${{ inputs.develocity-access-key }} + build-scan-publish: ${{ inputs.build-scan-publish }} + cache-read-only: ${{ inputs.cache-read-only }} + add-job-summary: ${{ inputs.add-job-summary }} + + - name: Setup Gradle + if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true') }} + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f + with: + build-scan-publish: ${{ inputs.build-scan-publish }} + build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service' + build-scan-terms-of-use-agree: 'yes' + cache-read-only: ${{ inputs.cache-read-only }} + add-job-summary: ${{ inputs.add-job-summary }} \ No newline at end of file diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml new file mode 100644 index 0000000000..00eb084990 --- /dev/null +++ b/.github/actions/tune-runner-vm/action.yml @@ -0,0 +1,80 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Tune Runner VM performance +description: tunes the GitHub Runner VM operation system +runs: + using: composite + steps: + - shell: bash + run: | + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "::group::Configure and tune OS" + # Ensure that reverse lookups for current hostname are handled properly + # Add the current IP address, long hostname and short hostname record to /etc/hosts file + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + + # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory + # consumption is high. + # Set vm.swappiness=1 to avoid swapping and allow high RAM usage + echo 1 | sudo tee /proc/sys/vm/swappiness + + # use "madvise" Linux Transparent HugePages (THP) setting + # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html + # "madvise" is generally a better option than the default "always" setting + # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b + echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled + echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled + echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag + echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag + + # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt + # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) + # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) + sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true + if mountpoint -q /mnt; then + sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true + fi + # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective + # https://www.spinics.net/lists/linux-ide/msg52562.html + for i in /sys/block/sd*/queue/discard_max_bytes; do + echo 0 | sudo tee $i + done + # disable unnecessary timers + sudo systemctl stop fstrim.timer fstrim.service \ + podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \ + phpsessionclean.timer man-db.timer motd-news.timer \ + dpkg-db-backup.timer e2scrub_all.timer \ + update-notifier-download.timer update-notifier-motd.timer || true + + # stop unnecessary services + sudo systemctl stop php8.3-fpm.service ModemManager.service \ + multipathd.socket multipathd.service udisks2.service walinuxagent.service || true + + echo '::endgroup::' + + # show memory + echo "::group::Available Memory" + free -m + echo '::endgroup::' + # show disk + echo "::group::Available diskspace" + df -BM + echo "::endgroup::" + fi \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 838b66779a..347b794409 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,17 +40,26 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - uses: actions/setup-java@v5 with: distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ env.JDK_VERSION }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c - - name: Build all modules - run: ./gradlew build -x test - - name: License check (RAT) - run: ./gradlew rat + - name: Setup Gradle + uses: ./.github/actions/setup-gradle + with: + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + cache-read-only: false + + - name: Build and check licenses + run: >- + ./gradlew -x test -x nar build rat spotlessCheck + --no-configuration-cache tests: name: Tests - ${{ matrix.name }} @@ -64,18 +73,26 @@ jobs: - name: Connectors tasks: test steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - uses: actions/setup-java@v5 with: distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ env.JDK_VERSION }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c + + - name: Setup Gradle + uses: ./.github/actions/setup-gradle + with: + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Run tests run: ./gradlew ${{ matrix.tasks }} - name: Upload test reports if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.name }}-test-reports path: '**/build/reports/tests/' diff --git a/README.md b/README.md index 6ecb2716f7..ea0694f752 100644 --- a/README.md +++ b/README.md @@ -66,26 +66,48 @@ mounting them into the `apachepulsar/pulsar` Docker image. |-----------|-------------| | Kafka Connect Adaptor | Run Kafka Connect connectors on Pulsar | +## Prerequisites + +- **JDK 17** or later — e.g. [Eclipse Temurin](https://adoptium.net/en-GB/temurin/releases?version=17&os=any&arch=any) + or [Amazon Corretto](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/what-is-corretto-17.html) + +> **Note**: This project includes a [Gradle Wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html) +> so no separate Gradle installation is needed. Use `./gradlew` on Linux/macOS and `gradlew.bat` on Windows. + ## Building +Compile and assemble all modules: + ```bash -./gradlew build -x test +./gradlew assemble ``` -To build all connector NARs: +NAR files are produced under each connector's `build/libs/` directory. + +Build a specific connector: ```bash -./gradlew build -x test +./gradlew :elastic-search:assemble ``` -NAR files are produced under each connector's `build/libs/` directory. - -To build the distribution tarball containing all connector NARs: +Build the distribution package containing all connector NARs: ```bash ./gradlew :distribution:pulsar-io-distribution:assemble ``` +Check source code license headers: + +```bash +./gradlew rat spotlessCheck +``` + +Auto-fix license headers: + +```bash +./gradlew spotlessApply +``` + ## Running Tests ```bash @@ -94,6 +116,9 @@ To build the distribution tarball containing all connector NARs: # Specific connector ./gradlew :elastic-search:test + +# Specific test class +./gradlew :elastic-search:test --tests "ElasticSearchSinkTests" ``` ## Using Connectors diff --git a/aerospike/build.gradle.kts b/aerospike/build.gradle.kts index eae094fd21..9fb452f446 100644 --- a/aerospike/build.gradle.kts +++ b/aerospike/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index be406b5ecb..0dd5ae199f 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/aws/build.gradle.kts b/aws/build.gradle.kts index b8456075be..81fc5e0711 100644 --- a/aws/build.gradle.kts +++ b/aws/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { implementation(libs.pulsar.io.core) implementation(libs.gson) diff --git a/azure-data-explorer/build.gradle.kts b/azure-data-explorer/build.gradle.kts index 7fcedd574e..0f0662b088 100644 --- a/azure-data-explorer/build.gradle.kts +++ b/azure-data-explorer/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } nar { narId.set("pulsar-io-azuredataexplorer") diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts new file mode 100644 index 0000000000..bc3d07a2be --- /dev/null +++ b/build-logic/conventions/build.gradle.kts @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `kotlin-dsl` +} + +dependencies { + implementation(libs.plugins.shadow.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) + implementation(libs.plugins.spotless.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) + implementation(libs.plugins.nar.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts new file mode 100644 index 0000000000..ad2ffaee23 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("com.diffplug.spotless") +} + +// ── License header check (Spotless) ──────────────────────────────────────── +val asfLicenseHeader = rootProject.file("src/license-header.txt").readText() +val asfLicenseHeaderJava = "/*\n" + asfLicenseHeader.lines() + .map { " * $it".trimEnd() } + .joinToString("\n") + "/\n" + +configure { + java { + targetExclude( + "**/generated/**", + "**/generated-sources/**", + // Generated FlatBuffers files (Kinesis) + "**/org/apache/pulsar/io/kinesis/fbs/*.java", + "build/**", + ) + licenseHeader(asfLicenseHeaderJava, "(\\n|package|import|public|class|module) ?") + } +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts new file mode 100644 index 0000000000..191a6aa9db --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `java-library` + id("pulsar-connectors.code-quality-conventions") +} + +val catalog = the().named("libs") + +group = "org.apache.pulsar" +version = catalog.findVersion("pulsar-connectors").get().requiredVersion + +// Add shared test resources (log4j2-test.xml) to the test classpath for all modules. +the()["test"].resources.srcDir(rootProject.file("gradle/test-resources")) + +tasks.withType().configureEach { + options.encoding = "UTF-8" + options.release.set(17) + options.compilerArgs.addAll(listOf("-parameters", "-Xlint:deprecation", "-Xlint:unchecked")) +} + +configurations.all { + // Exclude the old SLF4J 1.x bridge pulled in by transitive dependencies. + // Pulsar uses SLF4J 2.x with log4j-slf4j2-impl; having both causes + // NoSuchMethodError in Log4jLoggerFactory at test startup. + exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl") + + // Force Jackson version to match the version catalog. Transitive dependencies + // (e.g. from jackson-bom) can pull in newer versions that break API compatibility. + resolutionStrategy.eachDependency { + if (requested.group.startsWith("com.fasterxml.jackson")) { + useVersion(catalog.findVersion("jackson").get().requiredVersion) + } + } +} + +// Exclude bc-fips from modules that don't need it. bc-fips's CryptoServicesRegistrar +// conflicts with bcprov-jdk18on's version — having both causes NoSuchMethodError. +val modulesUsingBcFips = setOf("kafka-connect-adaptor") +if (project.name !in modulesUsingBcFips) { + configurations.all { + exclude(group = "org.bouncycastle", module = "bc-fips") + } +} + +dependencies { + // Enforced platform pins all dependency versions from the version catalog. + // This is the Gradle equivalent of Maven's dependencyManagement section. + "implementation"(enforcedPlatform(project(":pulsar-connectors-dependencies"))) + + // Resolve lz4-java capability conflict: at.yawk.lz4:lz4-java (used by Pulsar) and + // org.lz4:lz4-java (used by kafka-clients) both provide the org.lz4:lz4-java capability. + // Prefer at.yawk.lz4 which is the version Pulsar standardizes on. + configurations.all { + resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") { + select("at.yawk.lz4:lz4-java:0") + } + } + + // Annotation processing for Lombok + "compileOnly"(catalog.findLibrary("lombok").get()) + "annotationProcessor"(catalog.findLibrary("lombok").get()) + "testCompileOnly"(catalog.findLibrary("lombok").get()) + "testAnnotationProcessor"(catalog.findLibrary("lombok").get()) + + // Common test dependencies + "testImplementation"(catalog.findLibrary("testng").get()) + "testImplementation"(catalog.findLibrary("mockito-core").get()) + "testImplementation"(catalog.findLibrary("assertj-core").get()) + "testImplementation"(catalog.findLibrary("awaitility").get()) + "testImplementation"(catalog.findLibrary("system-lambda").get()) + "testImplementation"(catalog.findLibrary("slf4j-api").get()) + + // Logging runtime for tests — provides Log4j2 as the SLF4J backend. + // Some connectors (Alluxio minicluster, Solr embedded) require a logging + // implementation to be present at test runtime. + "testRuntimeOnly"(catalog.findLibrary("log4j-api").get()) + "testRuntimeOnly"(catalog.findLibrary("log4j-core").get()) + "testRuntimeOnly"(catalog.findLibrary("log4j-slf4j2-impl").get()) + "testRuntimeOnly"(catalog.findLibrary("jcl-over-slf4j").get()) +} + +tasks.withType().configureEach { + useTestNG { + // TestNG group filtering: -PtestGroups=group1,group2 -PexcludedTestGroups=flaky + providers.gradleProperty("testGroups").orNull?.let { groups -> + includeGroups(*groups.split(",").map { it.trim() }.toTypedArray()) + } + val excludedTestGroups = providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky") + excludeGroups(*(excludedTestGroups.split(",").map { it.trim() }.toTypedArray())) + } + testLogging { + events("FAILED") + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + showStackTraces = true + showExceptions = true + showCauses = true + } + maxHeapSize = "1300m" + maxParallelForks = 4 + val failFastValue = providers.gradleProperty("testFailFast").getOrElse("true").toBoolean() + failFast = failFastValue + systemProperty("testRetryCount", providers.gradleProperty("testRetryCount").getOrElse("1")) + systemProperty("testFailFast", failFastValue.toString()) + jvmArgs( + "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/sun.net=ALL-UNNAMED", + "--add-opens", "java.management/sun.management=ALL-UNNAMED", + "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", + "-XX:+EnableDynamicAgentLoading", + "-Xshare:off", + "-Dio.netty.tryReflectionSetAccessible=true", + "-Dpulsar.allocator.pooled=true", + "-Dpulsar.allocator.exit_on_oom=false", + "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", + "-Dpulsar.test.preventExit=true", + ) +} + +// Set archive names to match Maven artifactId for nested modules. +// Skip if the project name is already qualified (starts with parent name), +// which happens for sub-modules that use qualified names in settings.gradle.kts +// to avoid Gradle name clashes. +val parentProject = project.parent +if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject + && !project.name.startsWith(parentProject.name)) { + the().archivesName.set("${parentProject.name}-${project.name}") +} + +tasks.withType().configureEach { + manifest { + attributes( + "Implementation-Title" to project.name, + "Implementation-Version" to project.version, + ) + } +} + +// Add a task for viewing all configurations for all projects in a simple way +tasks.register("allDependencies"){} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts new file mode 100644 index 0000000000..d51ebc8ad2 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Convention plugin for NAR (Nifi Archive) modules. +// Configures platform module exclusions from runtimeClasspath, forces JAR artifacts +// for bundled-dependencies, and handles archive name qualification. + +plugins { + id("io.github.merlimat.nar") +} + +// NAR modules should not bundle Pulsar platform dependencies — they are provided +// at runtime by Pulsar's classloader hierarchy. +// Note: pulsar-io-common is NOT in java-instance.jar (runtime-all), so it must be +// bundled in each NAR that uses it (e.g., IOConfigUtils). +val pulsarPlatformModules = setOf( + "pulsar-client-api", + "pulsar-client-admin-api", + "pulsar-client-original", + "pulsar-client", + "pulsar-common", + "pulsar-config-validation", + "bouncy-castle-bc", + "pulsar-functions-api", + "pulsar-functions-instance", + "pulsar-functions-proto", + "pulsar-functions-secrets", + "pulsar-functions-utils", + "pulsar-io-core", + "pulsar-metadata", + "pulsar-opentelemetry", + "managed-ledger", + "pulsar-package-core", +) + +configurations.named("runtimeClasspath") { + exclude(group = "org.apache.bookkeeper") + // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it. + // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts. + exclude(group = "com.google.protobuf") + pulsarPlatformModules.forEach { module -> + exclude(group = "org.apache.pulsar", module = module) + } +} + +// The NAR plugin copies from runtimeClasspath which resolves project dependencies +// as class directories, not JARs. The NarClassLoader expects JARs in +// META-INF/bundled-dependencies/. Force the NAR task to use JAR artifacts. +// Use lazy resolution to avoid eagerly resolving the configuration at configuration +// time, which would cause configuration cache invalidation when JARs are created. +tasks.named("nar", Jar::class.java) { + val runtimeClasspath = configurations.named("runtimeClasspath") + into("META-INF/bundled-dependencies") { + from(runtimeClasspath.map { conf -> + conf.incoming.artifactView { + attributes { + attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + objects.named(LibraryElements::class.java, LibraryElements.JAR) + ) + } + }.files + }) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } +} + +// Set NAR-specific archive name qualification for nested modules. +val parentProject = project.parent +if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject + && !project.name.startsWith(parentProject.name)) { + val qualifiedName = "${parentProject.name}-${project.name}" + val narExt = extensions.getByName("nar") + @Suppress("UNCHECKED_CAST") + val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as org.gradle.api.provider.Property + narIdProp.set(qualifiedName) +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts new file mode 100644 index 0000000000..7a760e58e8 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Convention plugin for modules using the Shadow plugin. +// Applies the shadow plugin, disables the default jar task, and makes the +// shadow jar the primary artifact for both runtimeElements and apiElements, +// so plain project() dependencies resolve to the shadow jar. + +plugins { + id("com.gradleup.shadow") +} + +shadow { + addShadowVariantIntoJavaComponent.set(false) +} + +tasks.named("shadowJar") { + archiveClassifier.set("") + mergeServiceFiles() +} + +tasks.named("jar") { + enabled = false +} + +configurations { + named("runtimeElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("shadowJar")) + variants.clear() + } + } + named("apiElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("shadowJar")) + variants.clear() + } + } +} diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts new file mode 100644 index 0000000000..ec81a012a5 --- /dev/null +++ b/build-logic/settings.gradle.kts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +dependencyResolutionManagement { + repositories { + gradlePluginPortal() + mavenCentral() + } + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} +rootProject.name = "build-logic" +include("conventions") diff --git a/build.gradle.kts b/build.gradle.kts index 9aefca6122..ffd7684efb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,10 +17,35 @@ * under the License. */ +import org.jetbrains.gradle.ext.copyright +import org.jetbrains.gradle.ext.settings + plugins { alias(libs.plugins.rat) + alias(libs.plugins.version.catalog.update) + alias(libs.plugins.versions) + alias(libs.plugins.idea.ext) + alias(libs.plugins.spotless) apply false // workaround for https://github.com/diffplug/spotless/issues/2877 +} + +versionCatalogUpdate { + sortByKey = false + keep { + keepUnusedVersions.set(true) + } +} + +tasks.named("dependencyUpdates") { + outputFormatter = "html" + rejectVersionIf { + val nonStable = candidate.version.contains("alpha") || candidate.version.contains("beta") || candidate.version.contains("rc") + // OpenTelemetry publishes stable releases with -alpha suffix for some modules + val isOpenTelemetry = candidate.group.startsWith("io.opentelemetry") + nonStable && !(isOpenTelemetry && candidate.version.contains("alpha")) + } } +// ── Apache RAT (Release Audit Tool) ───────────────────────────────────────── tasks.named("rat").configure { val excludesProp = this.javaClass.getMethod("getExcludes").invoke(this) @Suppress("UNCHECKED_CAST") @@ -81,205 +106,19 @@ allprojects { version = pulsarConnectorsVersion } -subprojects { - // Platform modules use java-platform which is mutually exclusive with java-library. - if (project.name == "pulsar-dependencies") { - return@subprojects - } - - // Parent modules and non-Java modules that have no source code of their own - val skipModules = setOf("jdbc", "debezium", "distribution", "docker") - if (project.name in skipModules && project.childProjects.isNotEmpty()) { - return@subprojects - } - - apply(plugin = "java-library") - - // Add shared test resources (log4j2-test.xml) to the test classpath for all modules. - the()["test"].resources.srcDir(rootProject.file("gradle/test-resources")) - - tasks.withType { - options.encoding = "UTF-8" - options.release.set(17) - options.compilerArgs.addAll(listOf("-parameters")) - } - - configurations.all { - // Exclude the old SLF4J 1.x bridge - exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl") - - // Force Jackson version to match the version catalog - resolutionStrategy.eachDependency { - if (requested.group.startsWith("com.fasterxml.jackson")) { - useVersion(rootProject.libs.versions.jackson.get()) - } - } - } - - // Exclude bc-fips from modules that don't need it. - val modulesUsingBcFips = setOf("kafka-connect-adaptor") - if (project.name !in modulesUsingBcFips) { - configurations.all { - exclude(group = "org.bouncycastle", module = "bc-fips") - } - } - - dependencies { - // Enforced platform pins all dependency versions from the version catalog. - "implementation"(enforcedPlatform(project(":pulsar-dependencies"))) - - // Resolve lz4-java capability conflict - configurations.all { - resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") { - select("at.yawk.lz4:lz4-java:0") - } - } - - // Annotation processing for Lombok - "compileOnly"(rootProject.libs.lombok) - "annotationProcessor"(rootProject.libs.lombok) - "testCompileOnly"(rootProject.libs.lombok) - "testAnnotationProcessor"(rootProject.libs.lombok) - - // Common test dependencies - "testImplementation"(rootProject.libs.testng) - "testImplementation"(rootProject.libs.mockito.core) - "testImplementation"(rootProject.libs.assertj.core) - "testImplementation"(rootProject.libs.awaitility) - "testImplementation"(rootProject.libs.system.lambda) - "testImplementation"(rootProject.libs.slf4j.api) - - // Logging runtime for tests — provides Log4j2 as the SLF4J backend. - // Some connectors (Alluxio minicluster, Solr embedded) require a logging - // implementation to be present at test runtime. - "testRuntimeOnly"(rootProject.libs.log4j.api) - "testRuntimeOnly"(rootProject.libs.log4j.core) - "testRuntimeOnly"(rootProject.libs.log4j.slf4j2.impl) - "testRuntimeOnly"(rootProject.libs.jcl.over.slf4j) - } - - tasks.withType { - useTestNG { - // TestNG group filtering - providers.gradleProperty("testGroups").orNull?.let { groups -> - includeGroups(*groups.split(",").map { it.trim() }.toTypedArray()) - } - val excludedTestGroups = providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky") - excludeGroups(*(excludedTestGroups.split(",").map { it.trim() }.toTypedArray())) - } - maxHeapSize = "1300m" - maxParallelForks = 4 - systemProperty("testRetryCount", System.getProperty("testRetryCount", "1")) - systemProperty("testFailFast", System.getProperty("testFailFast", "true")) - jvmArgs( - "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", - "--add-opens", "java.base/java.lang=ALL-UNNAMED", - "--add-opens", "java.base/java.io=ALL-UNNAMED", - "--add-opens", "java.base/java.util=ALL-UNNAMED", - "--add-opens", "java.base/sun.net=ALL-UNNAMED", - "--add-opens", "java.management/sun.management=ALL-UNNAMED", - "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED", - "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED", - "--add-opens", "java.base/java.nio=ALL-UNNAMED", - "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", - "-XX:+EnableDynamicAgentLoading", - "-Xshare:off", - "-Dio.netty.tryReflectionSetAccessible=true", - "-Dpulsar.allocator.pooled=true", - "-Dpulsar.allocator.exit_on_oom=false", - "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", - "-Dpulsar.test.preventExit=true", - ) - } - - // Shadow JAR modules: expose the shadow JAR as a consumable configuration so other - // projects can depend on it via project(path = "...", configuration = "shadowElements") - pluginManager.withPlugin("com.gradleup.shadow") { - val shadowElements by configurations.creating { - isCanBeConsumed = true - isCanBeResolved = false - attributes { - attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.JAVA_RUNTIME)) - attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling.SHADOWED)) - } - } - artifacts.add("shadowElements", tasks.named("shadowJar")) - } - - // NAR modules should not bundle Pulsar platform dependencies — they are provided - // at runtime by Pulsar's classloader hierarchy. - pluginManager.withPlugin("io.github.merlimat.nar") { - val pulsarPlatformModules = setOf( - "pulsar-client-api", - "pulsar-client-admin-api", - "pulsar-client-original", - "pulsar-client", - "pulsar-common", - "pulsar-config-validation", - "bouncy-castle-bc", - "pulsar-functions-api", - "pulsar-functions-instance", - "pulsar-functions-proto", - "pulsar-functions-secrets", - "pulsar-functions-utils", - "pulsar-io-core", - "pulsar-metadata", - "pulsar-opentelemetry", - "managed-ledger", - "pulsar-package-core", - ) - configurations.named("runtimeClasspath") { - exclude(group = "org.apache.bookkeeper") - exclude(group = "com.google.protobuf") - pulsarPlatformModules.forEach { module -> - exclude(group = "org.apache.pulsar", module = module) - } - } - - // The NAR plugin copies from runtimeClasspath which resolves project dependencies - // as class directories, not JARs. The NarClassLoader expects JARs in - // META-INF/bundled-dependencies/. Force the NAR task to use JAR artifacts. - val jarView = configurations.named("runtimeClasspath").get() - .incoming.artifactView { - attributes { - attribute( - LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, - objects.named(LibraryElements::class.java, LibraryElements.JAR) - ) +idea { + project { + settings { + // add ASL2 copyright profile to IntelliJ + copyright { + useDefault = "ASL2" + profiles { + create("ASL2") { + notice = rootProject.file("src/license-header.txt").readText().trimEnd() + keyword = "Copyright" + } } - }.files - tasks.named("nar", Jar::class.java) { - into("META-INF/bundled-dependencies") { - from(jarView) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE } } } - - // Set archive names to match Maven artifactId for nested modules. - val parentProject = project.parent - if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject - && !project.name.startsWith(parentProject.name)) { - val qualifiedName = "${parentProject.name}-${project.name}" - the().archivesName.set(qualifiedName) - pluginManager.withPlugin("io.github.merlimat.nar") { - @Suppress("UNCHECKED_CAST") - val narExt = extensions.getByName("nar") - val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as Property - narIdProp.set(qualifiedName) - } - } - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version, - ) - } - } } - -// Access version catalog from subprojects -val Project.libs: org.gradle.accessors.dm.LibrariesForLibs - get() = rootProject.extensions.getByType() diff --git a/canal/build.gradle.kts b/canal/build.gradle.kts index 2a20556272..29397beeb3 100644 --- a/canal/build.gradle.kts +++ b/canal/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/cassandra/build.gradle.kts b/cassandra/build.gradle.kts index 8c2fab2c5b..da76cc01b8 100644 --- a/cassandra/build.gradle.kts +++ b/cassandra/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/core/build.gradle.kts b/debezium/core/build.gradle.kts index df1cc24cb4..746dc05543 100644 --- a/debezium/core/build.gradle.kts +++ b/debezium/core/build.gradle.kts @@ -17,6 +17,9 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} dependencies { compileOnly(libs.pulsar.io.core) diff --git a/debezium/mongodb/build.gradle.kts b/debezium/mongodb/build.gradle.kts index 53c2670166..d4292a9876 100644 --- a/debezium/mongodb/build.gradle.kts +++ b/debezium/mongodb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/mssql/build.gradle.kts b/debezium/mssql/build.gradle.kts index 1cfcc2c88a..07aac7f6ca 100644 --- a/debezium/mssql/build.gradle.kts +++ b/debezium/mssql/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/mysql/build.gradle.kts b/debezium/mysql/build.gradle.kts index bc79ff6e2a..33e30485a1 100644 --- a/debezium/mysql/build.gradle.kts +++ b/debezium/mysql/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/oracle/build.gradle.kts b/debezium/oracle/build.gradle.kts index 35bf12bcb4..dd1ef02d83 100644 --- a/debezium/oracle/build.gradle.kts +++ b/debezium/oracle/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/postgres/build.gradle.kts b/debezium/postgres/build.gradle.kts index f49f8a81fa..8dc37a5f32 100644 --- a/debezium/postgres/build.gradle.kts +++ b/debezium/postgres/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/distribution/io/build.gradle.kts b/distribution/io/build.gradle.kts index 91c7721e9f..9aa7b81470 100644 --- a/distribution/io/build.gradle.kts +++ b/distribution/io/build.gradle.kts @@ -18,9 +18,9 @@ */ // Distribution module — no Java compilation needed -tasks.named("compileJava") { enabled = false } -tasks.named("compileTestJava") { enabled = false } -tasks.named("jar") { enabled = false } +plugins { + base +} val pulsarVersion = project.version.toString() diff --git a/docker/pulsar-all/build.gradle.kts b/docker/pulsar-all/build.gradle.kts index 5ea1aa5f7a..448d3d1f3c 100644 --- a/docker/pulsar-all/build.gradle.kts +++ b/docker/pulsar-all/build.gradle.kts @@ -18,9 +18,9 @@ */ // Docker image module — no Java compilation needed -tasks.named("compileJava") { enabled = false } -tasks.named("compileTestJava") { enabled = false } -tasks.named("jar") { enabled = false } +plugins { + base +} val catalog = extensions.getByType().named("libs") val pulsarConnectorsVersion = project.version.toString() diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 62cabf83a4..ab0677af46 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { implementation(libs.pulsar.io.core) implementation(libs.guava) diff --git a/dynamodb/build.gradle.kts b/dynamodb/build.gradle.kts index 7697d16bae..980e18c0ba 100644 --- a/dynamodb/build.gradle.kts +++ b/dynamodb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/elastic-search/build.gradle.kts b/elastic-search/build.gradle.kts index 620cb7759f..1b2022cd8e 100644 --- a/elastic-search/build.gradle.kts +++ b/elastic-search/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { compileOnly(libs.pulsar.io.core) diff --git a/file/build.gradle.kts b/file/build.gradle.kts index a22c7ac9fb..3725b4f536 100644 --- a/file/build.gradle.kts +++ b/file/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/gradle.properties b/gradle.properties index 20a29b3a26..657c24b120 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,6 +18,7 @@ # org.gradle.configuration-cache=true +org.gradle.configureondemand=true org.gradle.parallel=true org.gradle.caching=true org.gradle.jvmargs=-Xmx4g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0e87504f16..da8c28fe9a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,20 +16,18 @@ # specific language governing permissions and limitations # under the License. # - [versions] # Core pulsar = "4.1.3" pulsar-connectors = "4.3.0-SNAPSHOT" java = "17" - # Docker docker-jdk = "21" pulsar-client-python = "3.10.0" - # Code quality -checkstyle = "10.14.2" - +checkstyle = "13.3.0" +spotless = "8.4.0" +idea-ext = "1.4.1" # Major frameworks bookkeeper = "4.17.3" zookeeper = "3.9.5" @@ -43,16 +41,14 @@ grpc = "1.75.0" slf4j = "2.0.17" log4j2 = "2.25.3" lombok = "1.18.42" - # OpenTelemetry opentelemetry = "1.56.0" opentelemetry-alpha = "1.56.0-alpha" opentelemetry-instrumentation = "2.21.0" opentelemetry-instrumentation-alpha = "2.21.0-alpha" opentelemetry-semconv = "1.37.0" - # Apache Commons -commons-lang3 = "3.19.0" +commons-lang3 = "3.20.0" commons-io = "2.21.0" commons-codec = "1.20.0" commons-compress = "1.28.0" @@ -63,7 +59,6 @@ commons-math3 = "3.6.1" commons-logging = "1.3.5" commons-beanutils = "1.11.0" commons-configuration2 = "2.12.0" - # BouncyCastle bouncycastle-bcprov = "1.78.1" bouncycastle-bcpkix = "1.81" @@ -71,15 +66,12 @@ bouncycastle-bcutil = "1.81" bouncycastle-bcprov-ext = "1.78.1" bouncycastle-bcpkix-fips = "2.0.10" bouncycastle-bc-fips = "2.0.1" - # Serialization avro = "1.12.0" gson = "2.13.2" snakeyaml = "2.0" - # Vert.x vertx = "4.5.24" - # Networking / HTTP asynchttpclient = "2.12.4" conscrypt = "2.5.2" @@ -88,32 +80,27 @@ okio = "3.16.3" netty-tcnative = "2.0.75.Final" httpcomponents-httpclient = "4.5.13" httpcomponents-httpcore = "4.4.15" - # Google libraries (transitive deps, versions managed to match Maven) google-auth = "1.24.1" google-http-client = "1.41.0" j2objc-annotations = "1.3" opencensus = "0.28.0" opentelemetry-gcp-resources = "1.48.0-alpha" - # Data structures / Utils guava = "33.4.8-jre" caffeine = "3.2.3" -fastutil = "8.5.16" jctools = "4.0.5" roaringbitmap = "1.6.9" hppc = "0.9.1" aircompressor = "2.0.3" completable-futures = "0.3.6" re2j = "1.8" - # Metrics / Observability prometheus = "0.16.0" prometheus-jmx = "0.16.1" dropwizardmetrics = "4.1.12.1" hdrHistogram = "2.1.9" perfmark = "0.26.0" - # Auth jsonwebtoken = "0.13.0" athenz = "1.10.62" @@ -121,18 +108,15 @@ jose4j = "0.9.6" nimbus-jose-jwt = "9.37.4" auth0-java-jwt = "4.3.0" auth0-jwks-rsa = "0.22.0" - # CLI picocli = "4.7.5" jline3 = "3.21.0" jline2 = "2.14.6" - javassist = "3.25.0-GA" rocksdb = "7.9.2" kotlin-stdlib = "1.8.20" audience-annotations = "0.12.0" jetbrains-annotations = "13.0" - # Misc curator = "5.7.1" reflections = "0.10.2" @@ -151,7 +135,6 @@ guice = "5.1.0" snappy = "1.1.10.8" ipaddress = "5.5.0" zt-zip = "1.17" - # Jakarta jakarta-ws-rs = "2.1.6" jakarta-annotation = "1.3.5" @@ -159,17 +142,14 @@ jakarta-activation = "1.2.2" jakarta-xml-bind = "2.3.3" jakarta-validation = "2.0.2" javax-servlet = "3.1.0" - # Oxia / etcd oxia = "0.7.4" - # Build plugins -lightproto = "0.6.2" +lightproto = "0.6.6" errorprone = "2.45.0" spotbugs = "4.9.6" checkerframework = "3.33.0" jsr305 = "3.0.2" - # Test testng = "7.7.1" mockito = "5.19.0" @@ -189,7 +169,6 @@ skyscreamer = "1.5.0" zstd-jni = "1.5.7-3" lz4java = "1.10.3" spring = "6.2.12" - # Connectors / IO kafka-client = "4.1.1" confluent = "8.1.1" @@ -198,7 +177,6 @@ elasticsearch-java = "8.15.3" debezium = "3.4.2.Final" debezium-mysql-connector = "9.4.0" kubernetesclient = "23.0.0" - # IO connector specific aerospike-client = "4.5.0" aws-sdk = "1.12.788" @@ -214,25 +192,24 @@ solr = "9.8.0" hbase = "2.6.4-hadoop3" hadoop3 = "3.4.3" jclouds = "2.6.0" - # Shading -shadow = "9.3.2" +shadow = "9.4.1" +com-4paradigm-openmldb = "0.4.4-hotfix1" [libraries] # SLF4J +slf4j-bom = { module = "org.slf4j:slf4j-bom", version.ref = "slf4j" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } - # Log4j2 +log4j-bom = { module = "org.apache.logging.log4j:log4j-bom", version.ref = "log4j2" } log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j2" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j2" } log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j2" } log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json", version.ref = "log4j2" } log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j2" } - # Lombok lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } - # Jackson jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson" } jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" } @@ -244,7 +221,6 @@ jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datat jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } jackson-module-jsonSchema = { module = "com.fasterxml.jackson.module:jackson-module-jsonSchema", version.ref = "jackson" } jackson-jaxrs-json-provider = { module = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", version.ref = "jackson" } - # Netty netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } netty-common = { module = "io.netty:netty-common", version.ref = "netty" } @@ -264,17 +240,16 @@ netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl- netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" } netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" } netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" } - # Protobuf / gRPC +protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref = "protobuf" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util", version.ref = "protobuf" } +grpc-bom = { module = "io.grpc:grpc-bom", version.ref = "grpc" } grpc-all = { module = "io.grpc:grpc-all", version.ref = "grpc" } grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref = "grpc" } grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } - # Guava guava = { module = "com.google.guava:guava", version.ref = "guava" } - # Apache Commons commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons-lang3" } commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } @@ -283,7 +258,6 @@ commons-compress = { module = "org.apache.commons:commons-compress", version.ref commons-collections4 = { module = "org.apache.commons:commons-collections4", version.ref = "commons-collections4" } commons-cli = { module = "commons-cli:commons-cli", version.ref = "commons-cli" } commons-math3 = { module = "org.apache.commons:commons-math3", version.ref = "commons-math3" } - # BookKeeper bookkeeper-server = { module = "org.apache.bookkeeper:bookkeeper-server", version.ref = "bookkeeper" } bookkeeper-common-allocator = { module = "org.apache.bookkeeper:bookkeeper-common-allocator", version.ref = "bookkeeper" } @@ -297,18 +271,15 @@ bookkeeper-stream-storage-service-impl = { module = "org.apache.bookkeeper:strea bookkeeper-tools-framework = { module = "org.apache.bookkeeper:bookkeeper-tools-framework", version.ref = "bookkeeper" } bookkeeper-http-vertx-server = { module = "org.apache.bookkeeper.http:vertx-http-server", version.ref = "bookkeeper" } distributedlog-core = { module = "org.apache.distributedlog:distributedlog-core", version.ref = "bookkeeper" } - # ZooKeeper zookeeper = { module = "org.apache.zookeeper:zookeeper", version.ref = "zookeeper" } zookeeper-tests = { module = "org.apache.zookeeper:zookeeper", version.ref = "zookeeper" } - # Curator curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "curator" } - # Conscrypt conscrypt-openjdk-uber = { module = "org.conscrypt:conscrypt-openjdk-uber", version.ref = "conscrypt" } - # Jetty +jetty-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty" } jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" } jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "jetty" } jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server", version.ref = "jetty" } @@ -321,7 +292,6 @@ jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy", version.re jetty-websocket-jetty-api = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-api", version.ref = "jetty" } jetty-websocket-jetty-client = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-client", version.ref = "jetty" } jetty-ee8-websocket-jetty-server = { module = "org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server", version.ref = "jetty" } - # Jersey jersey-server = { module = "org.glassfish.jersey.core:jersey-server", version.ref = "jersey" } jersey-container-servlet-core = { module = "org.glassfish.jersey.containers:jersey-container-servlet-core", version.ref = "jersey" } @@ -332,8 +302,8 @@ jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2", version.ref = jersey-media-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jersey-test-framework-core", version.ref = "jersey" } jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2", version.ref = "jersey" } - # Prometheus +simpleclient-bom = { module = "io.prometheus:simpleclient_bom", version.ref = "prometheus" } simpleclient = { module = "io.prometheus:simpleclient", version.ref = "prometheus" } simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot", version.ref = "prometheus" } simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine", version.ref = "prometheus" } @@ -342,8 +312,11 @@ simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet", version. simpleclient-common = { module = "io.prometheus:simpleclient_common", version.ref = "prometheus" } simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2", version.ref = "prometheus" } prometheus-jmx-collector = { module = "io.prometheus.jmx:collector", version.ref = "prometheus-jmx" } - # OpenTelemetry +opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version.ref = "opentelemetry" } +opentelemetry-bom-alpha = { module = "io.opentelemetry:opentelemetry-bom-alpha", version.ref = "opentelemetry-alpha" } +opentelemetry-instrumentation-bom = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom", version.ref = "opentelemetry-instrumentation" } +opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version.ref = "opentelemetry-instrumentation-alpha" } opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } opentelemetry-api-incubator = { module = "io.opentelemetry:opentelemetry-api-incubator", version.ref = "opentelemetry-alpha" } opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } @@ -355,28 +328,22 @@ opentelemetry-instrumentation-resources = { module = "io.opentelemetry.instrumen opentelemetry-instrumentation-runtime-telemetry-java17 = { module = "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17", version.ref = "opentelemetry-instrumentation-alpha" } opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version.ref = "opentelemetry-semconv" } opentelemetry-gcp-resources = { module = "io.opentelemetry.contrib:opentelemetry-gcp-resources", version.ref = "opentelemetry-gcp-resources" } - # BouncyCastle bcpkix-jdk18on = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle-bcpkix" } bcprov-ext-jdk18on = { module = "org.bouncycastle:bcprov-ext-jdk18on", version.ref = "bouncycastle-bcprov-ext" } bcpkix-fips = { module = "org.bouncycastle:bcpkix-fips", version.ref = "bouncycastle-bcpkix-fips" } bc-fips = { module = "org.bouncycastle:bc-fips", version.ref = "bouncycastle-bc-fips" } -bcutil-fips = { module = "org.bouncycastle:bcutil-fips", version = "2.0.5" } - +bcutil-fips = "org.bouncycastle:bcutil-fips:2.0.5" # RocksDB rocksdbjni = { module = "org.rocksdb:rocksdbjni", version.ref = "rocksdb" } - # Error Prone error-prone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "errorprone" } - # Data structures caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } -fastutil = { module = "it.unimi.dsi:fastutil", version.ref = "fastutil" } jctools-core = { module = "org.jctools:jctools-core", version.ref = "jctools" } roaringbitmap = { module = "org.roaringbitmap:RoaringBitmap", version.ref = "roaringbitmap" } hppc = { module = "com.carrotsearch:hppc", version.ref = "hppc" } aircompressor = { module = "io.airlift:aircompressor", version.ref = "aircompressor" } - # Misc libs gson = { module = "com.google.code.gson:gson", version.ref = "gson" } re2j = { module = "com.google.re2j:re2j", version.ref = "re2j" } @@ -391,10 +358,12 @@ javassist = { module = "org.javassist:javassist", version.ref = "javassist" } commons-text = { module = "org.apache.commons:commons-text", version.ref = "commons-text" } typetools = { module = "net.jodah:typetools", version.ref = "typetools" } perfmark-api = { module = "io.perfmark:perfmark-api", version.ref = "perfmark" } +okhttp3-bom = { module = "com.squareup.okhttp3:okhttp-bom", version.ref = "okhttp3" } okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm", version.ref = "okhttp3" } okhttp3-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp3" } +okio-bom = { module = "com.squareup.okio:okio-bom", version.ref = "okio" } okio = { module = "com.squareup.okio:okio", version.ref = "okio" } -# Transitive dep version pins (enforced by pulsar-dependencies platform) +# Transitive dep version pins (enforced by pulsar-connectors-dependencies platform) google-auth-library-credentials = { module = "com.google.auth:google-auth-library-credentials", version.ref = "google-auth" } google-auth-library-oauth2-http = { module = "com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth" } google-http-client = { module = "com.google.http-client:google-http-client", version.ref = "google-http-client" } @@ -406,7 +375,6 @@ httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", v httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "httpcomponents-httpcore" } jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version.ref = "jakarta-annotation" } kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin-stdlib" } - snakeyaml = { module = "org.yaml:snakeyaml", version.ref = "snakeyaml" } ant = { module = "org.apache.ant:ant", version.ref = "ant" } guice = { module = "com.google.inject:guice", version.ref = "guice" } @@ -416,7 +384,7 @@ vertx-core = { module = "io.vertx:vertx-core", version.ref = "vertx" } vertx-web = { module = "io.vertx:vertx-web", version.ref = "vertx" } avro = { module = "org.apache.avro:avro", version.ref = "avro" } avro-protobuf = { module = "org.apache.avro:avro-protobuf", version.ref = "avro" } -joda-time = { module = "joda-time:joda-time", version = "2.10.10" } +joda-time = "joda-time:joda-time:2.10.10" sketches-core = { module = "com.yahoo.datasketches:sketches-core", version.ref = "sketches" } java-semver = { module = "com.github.zafarkhaja:java-semver", version.ref = "java-semver" } oshi-core = { module = "com.github.oshi:oshi-core-java11", version.ref = "oshi" } @@ -426,16 +394,14 @@ dropwizardmetrics-core = { module = "io.dropwizard.metrics:metrics-core", versio dropwizardmetrics-graphite = { module = "io.dropwizard.metrics:metrics-graphite", version.ref = "dropwizardmetrics" } dropwizardmetrics-jvm = { module = "io.dropwizard.metrics:metrics-jvm", version.ref = "dropwizardmetrics" } snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy" } -jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" } +jspecify = "org.jspecify:jspecify:1.0.0" reflections = { module = "org.reflections:reflections", version.ref = "reflections" } - # Auth / Security jjwt-api = { module = "io.jsonwebtoken:jjwt-api", version.ref = "jsonwebtoken" } jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl", version.ref = "jsonwebtoken" } jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson", version.ref = "jsonwebtoken" } auth0-java-jwt = { module = "com.auth0:java-jwt", version.ref = "auth0-java-jwt" } auth0-jwks-rsa = { module = "com.auth0:jwks-rsa", version.ref = "auth0-jwks-rsa" } - # Jakarta jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "jakarta-ws-rs" } jakarta-activation-api = { module = "jakarta.activation:jakarta.activation-api", version.ref = "jakarta-activation" } @@ -444,17 +410,15 @@ jakarta-xml-bind-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", versi javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version.ref = "javax-servlet" } zt-zip = { module = "org.zeroturnaround:zt-zip", version.ref = "zt-zip" } ipaddress = { module = "com.github.seancfoley:ipaddress", version.ref = "ipaddress" } - # Oxia / etcd oxia-client = { module = "io.github.oxia-db:oxia-client", version.ref = "oxia" } oxia-testcontainers = { module = "io.github.oxia-db:oxia-testcontainers", version.ref = "oxia" } - # Static analysis spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version.ref = "spotbugs" } jsr305 = { module = "com.google.code.findbugs:jsr305", version.ref = "jsr305" } - # Test testng = { module = "org.testng:testng", version.ref = "testng" } +mockito-bom = { module = "org.mockito:mockito-bom", version.ref = "mockito" } mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } awaitility = { module = "org.awaitility:awaitility", version.ref = "awaitility" } assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } @@ -475,6 +439,7 @@ spring-jdbc = { module = "org.springframework:spring-jdbc", version.ref = "sprin spring-orm = { module = "org.springframework:spring-orm", version.ref = "spring" } kubernetes-client-java = { module = "io.kubernetes:client-java", version.ref = "kubernetesclient" } kubernetes-client-java-api-fluent = { module = "io.kubernetes:client-java-api-fluent", version.ref = "kubernetesclient" } +testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version.ref = "testcontainers" } testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "testcontainers" } testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy", version.ref = "testcontainers" } @@ -489,19 +454,18 @@ testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.re testcontainers-k3s = { module = "org.testcontainers:k3s", version.ref = "testcontainers" } kerby-simplekdc = { module = "org.apache.kerby:kerb-simplekdc", version.ref = "kerby" } json = { module = "org.json:json", version.ref = "json" } - # AWS SDKs +aws-java-sdk-bom = { module = "com.amazonaws:aws-java-sdk-bom", version.ref = "aws-sdk" } aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core", version.ref = "aws-sdk" } aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts", version.ref = "aws-sdk" } aws-sdk2-regions = { module = "software.amazon.awssdk:regions", version.ref = "aws-sdk2" } aws-sdk2-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws-sdk2" } aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis", version.ref = "aws-sdk2" } aws-sdk2-utils = { module = "software.amazon.awssdk:utils", version.ref = "aws-sdk2" } -dynamodb-streams-kinesis-adapter = { module = "com.amazonaws:dynamodb-streams-kinesis-adapter", version = "1.6.0" } -amazon-kinesis-client = { module = "software.amazon.kinesis:amazon-kinesis-client", version = "2.6.0" } -amazon-kinesis-client-v3 = { module = "software.amazon.kinesis:amazon-kinesis-client", version = "3.1.2" } -amazon-kinesis-producer = { module = "software.amazon.kinesis:amazon-kinesis-producer", version = "1.0.4" } - +dynamodb-streams-kinesis-adapter = "com.amazonaws:dynamodb-streams-kinesis-adapter:1.6.0" +amazon-kinesis-client = "software.amazon.kinesis:amazon-kinesis-client:2.6.0" +amazon-kinesis-client-v3 = "software.amazon.kinesis:amazon-kinesis-client:3.1.2" +amazon-kinesis-producer = "software.amazon.kinesis:amazon-kinesis-producer:1.0.4" # Kafka / Confluent kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka-client" } kafka-connect-runtime = { module = "org.apache.kafka:connect-runtime", version.ref = "kafka-client" } @@ -512,11 +476,9 @@ kafka-connect-file = { module = "org.apache.kafka:connect-file", version.ref = " kafka-schema-registry-client = { module = "io.confluent:kafka-schema-registry-client", version.ref = "confluent" } kafka-avro-serializer = { module = "io.confluent:kafka-avro-serializer", version.ref = "confluent" } kafka-connect-avro-converter = { module = "io.confluent:kafka-connect-avro-converter", version.ref = "confluent" } - # ElasticSearch / OpenSearch opensearch-rest-high-level-client = { module = "org.opensearch.client:opensearch-rest-high-level-client", version.ref = "opensearch" } elasticsearch-java = { module = "co.elastic.clients:elasticsearch-java", version.ref = "elasticsearch-java" } - # Debezium debezium-core = { module = "io.debezium:debezium-core", version.ref = "debezium" } debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium" } @@ -524,72 +486,61 @@ debezium-connector-mongodb = { module = "io.debezium:debezium-connector-mongodb" debezium-connector-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium" } debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle", version.ref = "debezium" } debezium-connector-sqlserver = { module = "io.debezium:debezium-connector-sqlserver", version.ref = "debezium" } - # Database drivers -postgresql-jdbc = { module = "org.postgresql:postgresql", version = "42.7.10" } -sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version = "3.47.1.0" } -clickhouse-jdbc = { module = "com.clickhouse:clickhouse-jdbc", version = "0.4.6" } -mariadb-jdbc = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.5.5" } -openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version = "0.4.4-hotfix1" } -openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version = "0.4.4-hotfix1" } - +postgresql-jdbc = "org.postgresql:postgresql:42.7.10" +sqlite-jdbc = "org.xerial:sqlite-jdbc:3.47.1.0" +clickhouse-jdbc = "com.clickhouse:clickhouse-jdbc:0.4.6" +mariadb-jdbc = "org.mariadb.jdbc:mariadb-java-client:3.5.5" +openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref = "com-4paradigm-openmldb" } +openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version.ref = "com-4paradigm-openmldb" } # JClouds jclouds-allblobstore = { module = "org.apache.jclouds:jclouds-allblobstore", version.ref = "jclouds" } jclouds-blobstore = { module = "org.apache.jclouds:jclouds-blobstore", version.ref = "jclouds" } - # Hadoop hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" } hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version.ref = "hadoop3" } hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" } hadoop-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop3" } - # HBase hbase-client = { module = "org.apache.hbase:hbase-client", version.ref = "hbase" } hbase-common = { module = "org.apache.hbase:hbase-common", version.ref = "hbase" } - # NoSQL / Search aerospike-client = { module = "com.aerospike:aerospike-client-bc", version.ref = "aerospike-client" } cassandra-driver = { module = "com.datastax.cassandra:cassandra-driver-core", version.ref = "cassandra-driver" } failsafe = { module = "dev.failsafe:failsafe", version.ref = "failsafe" } -docker-java-core = { module = "com.github.docker-java:docker-java-core", version = "3.4.1" } +docker-java-core = "com.github.docker-java:docker-java-core:3.4.1" mongodb-driver-reactivestreams = { module = "org.mongodb:mongodb-driver-reactivestreams", version.ref = "mongodb-driver" } mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version.ref = "mongodb-driver" } lettuce-core = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" } solr-solrj = { module = "org.apache.solr:solr-solrj", version.ref = "solr" } solr-test-framework = { module = "org.apache.solr:solr-test-framework", version.ref = "solr" } solr-core = { module = "org.apache.solr:solr-core", version.ref = "solr" } - # Messaging rabbitmq-amqp-client = { module = "com.rabbitmq:amqp-client", version.ref = "rabbitmq-client" } nsq-j = { module = "com.sproutsocial:nsq-j", version.ref = "nsq-client" } - # Time series influxdb-client-java = { module = "com.influxdb:influxdb-client-java", version.ref = "influxdb-client" } influxdb-java = { module = "org.influxdb:influxdb-java", version.ref = "influxdb-java" } - # IO specific jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref = "jackson" } -json-smart = { module = "net.minidev:json-smart", version = "2.5.2" } -json-flattener = { module = "com.github.wnameless.json:json-flattener", version = "0.16.4" } -flatbuffers-java = { module = "com.google.flatbuffers:flatbuffers-java", version = "1.9.0" } -jfairy = { module = "io.codearte.jfairy:jfairy", version = "0.5.9" } -embedded-redis = { module = "com.github.kstyrc:embedded-redis", version = "0.6" } - +json-smart = "net.minidev:json-smart:2.5.2" +json-flattener = "com.github.wnameless.json:json-flattener:0.16.4" +flatbuffers-java = "com.google.flatbuffers:flatbuffers-java:1.9.0" +jfairy = "io.codearte.jfairy:jfairy:0.5.9" +embedded-redis = "com.github.kstyrc:embedded-redis:0.6" # Auth athenz-zts-java-client = { module = "com.yahoo.athenz:athenz-zts-java-client", version.ref = "athenz" } athenz-cert-refresher = { module = "com.yahoo.athenz:athenz-cert-refresher", version.ref = "athenz" } athenz-auth-core = { module = "com.yahoo.athenz:athenz-auth-core", version.ref = "athenz" } athenz-zpe-java-client = { module = "com.yahoo.athenz:athenz-zpe-java-client", version.ref = "athenz" } - # Misc bcprov-jdk18on = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bouncycastle-bcprov" } commons-logging = { module = "commons-logging:commons-logging", version.ref = "commons-logging" } commons-beanutils = { module = "commons-beanutils:commons-beanutils", version.ref = "commons-beanutils" } commons-configuration2 = { module = "org.apache.commons:commons-configuration2", version.ref = "commons-configuration2" } bookkeeper-stats-api = { module = "org.apache.bookkeeper.stats:bookkeeper-stats-api", version.ref = "bookkeeper" } -datasketches-memory = { module = "org.apache.datasketches:datasketches-memory", version = "2.2.0" } -datasketches-java = { module = "org.apache.datasketches:datasketches-java", version = "6.1.1" } - +datasketches-memory = "org.apache.datasketches:datasketches-memory:2.2.0" +datasketches-java = "org.apache.datasketches:datasketches-java:6.1.1" # Pulsar core modules (published Maven artifacts, used by connectors) pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core", version.ref = "pulsar" } pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common", version.ref = "pulsar" } @@ -608,8 +559,12 @@ pulsar-buildtools = { module = "org.apache.pulsar:buildtools", version.ref = "pu [plugins] lightproto = { id = "io.streamnative.lightproto", version.ref = "lightproto" } -nar = { id = "io.github.merlimat.nar", version = "0.1.3" } -protobuf = { id = "com.google.protobuf", version = "0.9.6" } +nar = "io.github.merlimat.nar:0.1.3" +protobuf = "com.google.protobuf:0.9.6" shadow = { id = "com.gradleup.shadow", version.ref = "shadow" } -rat = { id = "org.nosphere.apache.rat", version = "0.8.1" } -license = { id = "com.github.hierynomus.license", version = "0.16.1" } +rat = "org.nosphere.apache.rat:0.8.1" +spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } +idea-ext = { id = "org.jetbrains.gradle.plugin.idea-ext", version.ref = "idea-ext" } +version-catalog-update = "nl.littlerobots.version-catalog-update:1.1.0" +versions = "com.github.ben-manes.versions:0.53.0" +license = "com.github.hierynomus.license:0.16.1" diff --git a/hbase/build.gradle.kts b/hbase/build.gradle.kts index caa37adf66..be1f1e5c5b 100644 --- a/hbase/build.gradle.kts +++ b/hbase/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/hdfs3/build.gradle.kts b/hdfs3/build.gradle.kts index 1f20430559..82e22541ca 100644 --- a/hdfs3/build.gradle.kts +++ b/hdfs3/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/http/build.gradle.kts b/http/build.gradle.kts index f4008b947f..9b101afd8d 100644 --- a/http/build.gradle.kts +++ b/http/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/influxdb/build.gradle.kts b/influxdb/build.gradle.kts index a29d19b6ec..d3a589ec8d 100644 --- a/influxdb/build.gradle.kts +++ b/influxdb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/jdbc/clickhouse/build.gradle.kts b/jdbc/clickhouse/build.gradle.kts index da66bc4802..7a36945ed9 100644 --- a/jdbc/clickhouse/build.gradle.kts +++ b/jdbc/clickhouse/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/core/build.gradle.kts b/jdbc/core/build.gradle.kts index 7d38cdfde2..5d40fba464 100644 --- a/jdbc/core/build.gradle.kts +++ b/jdbc/core/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { api(libs.pulsar.io.common) api(libs.pulsar.io.core) diff --git a/jdbc/mariadb/build.gradle.kts b/jdbc/mariadb/build.gradle.kts index 1691bff406..ce5fe962dd 100644 --- a/jdbc/mariadb/build.gradle.kts +++ b/jdbc/mariadb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/openmldb/build.gradle.kts b/jdbc/openmldb/build.gradle.kts index a9894a8ee8..7eae514afe 100644 --- a/jdbc/openmldb/build.gradle.kts +++ b/jdbc/openmldb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/postgres/build.gradle.kts b/jdbc/postgres/build.gradle.kts index 752366f894..3aa3da17c2 100644 --- a/jdbc/postgres/build.gradle.kts +++ b/jdbc/postgres/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/sqlite/build.gradle.kts b/jdbc/sqlite/build.gradle.kts index 4e1495a838..66c92cface 100644 --- a/jdbc/sqlite/build.gradle.kts +++ b/jdbc/sqlite/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/kafka-connect-adaptor-nar/build.gradle.kts b/kafka-connect-adaptor-nar/build.gradle.kts index 4ee233d59d..05ebf6307a 100644 --- a/kafka-connect-adaptor-nar/build.gradle.kts +++ b/kafka-connect-adaptor-nar/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } nar { narId.set("pulsar-io-kafka-connect-adaptor") diff --git a/kafka-connect-adaptor/build.gradle.kts b/kafka-connect-adaptor/build.gradle.kts index 4809b8269e..1349adb1a4 100644 --- a/kafka-connect-adaptor/build.gradle.kts +++ b/kafka-connect-adaptor/build.gradle.kts @@ -17,6 +17,9 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} dependencies { compileOnly(libs.pulsar.io.core) diff --git a/kafka/build.gradle.kts b/kafka/build.gradle.kts index 8046aa6af7..fb11565076 100644 --- a/kafka/build.gradle.kts +++ b/kafka/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } diff --git a/kinesis-kpl-shaded/build.gradle.kts b/kinesis-kpl-shaded/build.gradle.kts index 4197e5d5de..bf6ecce7cf 100644 --- a/kinesis-kpl-shaded/build.gradle.kts +++ b/kinesis-kpl-shaded/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.shadow) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.shadow-conventions") } dependencies { @@ -34,16 +35,7 @@ configurations.all { } } -// Disable the default jar task so the shadow JAR is the only artifact. -// This avoids Gradle's implicit dependency validation errors when consumers -// use project() to depend on this module. -tasks.jar { - enabled = false -} - tasks.shadowJar { - archiveClassifier.set("") - mergeServiceFiles() dependencies { include(dependency("software.amazon.kinesis:amazon-kinesis-producer")) include(dependency("com.google.protobuf:protobuf-java")) diff --git a/kinesis/build.gradle.kts b/kinesis/build.gradle.kts index 53f4995e27..2a801334bc 100644 --- a/kinesis/build.gradle.kts +++ b/kinesis/build.gradle.kts @@ -18,13 +18,14 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { // The shaded KPL project bundles amazon-kinesis-producer with relocated protobuf. - // Use shadowElements to get the shadow JAR (which has relocated protobuf). - implementation(project(path = ":kinesis-kpl-shaded", configuration = "shadowElements")) + // The shadow convention exposes the shadow JAR as the primary artifact. + implementation(project(":kinesis-kpl-shaded")) // CompileOnly: needed for compilation against KPL classes but NOT bundled in NAR. // At runtime, KPL classes come from the shaded project's shadow JAR. compileOnly(libs.amazon.kinesis.producer) diff --git a/mongo/build.gradle.kts b/mongo/build.gradle.kts index b760c25863..d06d6c8bc6 100644 --- a/mongo/build.gradle.kts +++ b/mongo/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/netty/build.gradle.kts b/netty/build.gradle.kts index be70778709..43e3e5a685 100644 --- a/netty/build.gradle.kts +++ b/netty/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/nsq/build.gradle.kts b/nsq/build.gradle.kts index 732c22c695..be75a00087 100644 --- a/nsq/build.gradle.kts +++ b/nsq/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/pulsar-dependencies/build.gradle.kts b/pulsar-connectors-dependencies/build.gradle.kts similarity index 56% rename from pulsar-dependencies/build.gradle.kts rename to pulsar-connectors-dependencies/build.gradle.kts index 7bead9941f..66f96c6642 100644 --- a/pulsar-dependencies/build.gradle.kts +++ b/pulsar-connectors-dependencies/build.gradle.kts @@ -19,7 +19,7 @@ // Enforced platform module that declares version constraints for all dependencies. // This is the Gradle equivalent of Maven's dependencyManagement section. -// All subprojects consume this via: implementation(enforcedPlatform(project(":pulsar-dependencies"))) +// All subprojects consume this via: implementation(enforcedPlatform(project(":pulsar-connectors-dependencies"))) plugins { `java-platform` } @@ -30,14 +30,19 @@ javaPlatform { } dependencies { - constraints { - // Iterate over all library declarations in the version catalog and add them as constraints. - // This ensures that any transitive dependency matching a catalog entry gets pinned to - // the version we specify, regardless of what version a transitive dependency requests. - val catalog = project.extensions.getByType().named("libs") - catalog.libraryAliases.forEach { alias -> - catalog.findLibrary(alias).ifPresent { provider -> - api(provider) + val catalog = project.extensions.getByType().named("libs") + // Iterate over all library declarations in the version catalog and add them as constraints. + // This ensures that any transitive dependency matching a catalog entry gets pinned to + // the version we specify, regardless of what version a transitive dependency requests. + // BOM entries (detected by module name) are imported as platforms rather than constraints. + catalog.libraryAliases.forEach { alias -> + catalog.findLibrary(alias).ifPresent { provider -> + val module = provider.get().module + if (module.name.endsWith("-bom") || module.name.endsWith("_bom") || module.name == "bom" + || module.name.contains("-bom-") || module.name.contains("_bom_")) { + api(platform(provider)) + } else { + constraints.api(provider) } } } diff --git a/rabbitmq/build.gradle.kts b/rabbitmq/build.gradle.kts index d7613cef2d..b15ee747ac 100644 --- a/rabbitmq/build.gradle.kts +++ b/rabbitmq/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/redis/build.gradle.kts b/redis/build.gradle.kts index eb57bee916..bfd82a9a68 100644 --- a/redis/build.gradle.kts +++ b/redis/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/settings.gradle.kts b/settings.gradle.kts index 9194ce066e..d186f01b9c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,6 +22,7 @@ pluginManagement { gradlePluginPortal() mavenCentral() } + includeBuild("build-logic") } dependencyResolutionManagement { @@ -49,8 +50,16 @@ dependencyResolutionManagement { rootProject.name = "pulsar-connectors" +// This build requires Java 17 or later. Version check can be skipped with -PskipJavaVersionCheck parameter. +val javaVersion = providers.provider { JavaVersion.current() } +require(providers.gradleProperty("skipJavaVersionCheck").isPresent + || javaVersion.get() >= JavaVersion.VERSION_17) { + "This build requires Java 17 or later, but is running on Java ${javaVersion.get()}. " + + "Pass -PskipJavaVersionCheck to skip this check." +} + // Enforced platform for dependency version management -include("pulsar-dependencies") +include("pulsar-connectors-dependencies") // Simple connectors (flat layout, top-level directories) include("aerospike") diff --git a/solr/build.gradle.kts b/solr/build.gradle.kts index d15cedc516..d338d67bef 100644 --- a/solr/build.gradle.kts +++ b/solr/build.gradle.kts @@ -18,10 +18,11 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } // Solr 9.x embeds Jetty 10.x, which is incompatible with Pulsar's Jetty 12. -// The pulsar-dependencies platform enforces Jetty 12 strict versions, which override +// The pulsar-connectors-dependencies platform enforces Jetty 12 strict versions, which override // enforcedPlatform("jetty-bom:10.0.24") because Gradle picks the highest strict version. // Use resolutionStrategy.force to downgrade Jetty to 10.0.24 for test configurations. val jetty10Version = "10.0.24" diff --git a/src/license-header.txt b/src/license-header.txt new file mode 100644 index 0000000000..60b675e310 --- /dev/null +++ b/src/license-header.txt @@ -0,0 +1,16 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. From 32d6848adb150309bdf9b78c89465d97e9e3c4b7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Apr 2026 22:52:53 +0300 Subject: [PATCH 02/33] Improve RAT check, also ignore .kotlin directory to fix CI --- .gitignore | 1 + .ratignore | 48 ++++++++++++++++++++++++++++ build.gradle.kts | 66 ++++++++++----------------------------- gradle/libs.versions.toml | 2 +- 4 files changed, 66 insertions(+), 51 deletions(-) create mode 100644 .ratignore diff --git a/.gitignore b/.gitignore index 91bd581484..33093cfbed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Gradle .gradle/ +.kotlin/ build/ **/build/ diff --git a/.ratignore b/.ratignore new file mode 100644 index 0000000000..ab06f4004d --- /dev/null +++ b/.ratignore @@ -0,0 +1,48 @@ +# Build artifacts +**/build/** +**/target/** +# Gradle files +.gradle/** +gradle/wrapper/** +**/.gradle/** +**/.kotlin/** +**/gradle/wrapper/** +gradlew +gradlew.bat +gradle/libs.versions.toml +# Generated Flatbuffer files (Kinesis) +**/org/apache/pulsar/io/kinesis/fbs/*.java +# Services files +**/META-INF/services/* +# Certificates and keys +**/*.crt +**/*.key +**/*.csr +**/*.pem +**/*.srl +**/certificate-authority/serial +**/certificate-authority/index.txt +**/*.json +**/*.txt +# Project/IDE files +**/*.md +.github/** +**/*.nar +**/.gitignore +**/.gitattributes +**/*.iml +**/.classpath +**/.project +**/.settings +**/.idea/** +**/.vscode/** +# Avro schemas +**/*.avsc +# Patch files +**/*.patch +# Hidden directories +.*/** +# Test output +**/test-output/** +# Log files +**/*.log diff --git a/build.gradle.kts b/build.gradle.kts index ffd7684efb..db427cb7b0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,6 +17,7 @@ * under the License. */ +import com.github.vlsi.gradle.git.dsl.gitignore import org.jetbrains.gradle.ext.copyright import org.jetbrains.gradle.ext.settings @@ -24,6 +25,7 @@ plugins { alias(libs.plugins.rat) alias(libs.plugins.version.catalog.update) alias(libs.plugins.versions) + alias(libs.plugins.crlf) apply false alias(libs.plugins.idea.ext) alias(libs.plugins.spotless) apply false // workaround for https://github.com/diffplug/spotless/issues/2877 } @@ -46,56 +48,20 @@ tasks.named(" } // ── Apache RAT (Release Audit Tool) ───────────────────────────────────────── -tasks.named("rat").configure { - val excludesProp = this.javaClass.getMethod("getExcludes").invoke(this) - @Suppress("UNCHECKED_CAST") - val excludes = excludesProp as MutableCollection - excludes.addAll(listOf( - // Build artifacts - "**/build/**", - "**/target/**", - // Gradle files - ".gradle/**", - "gradle/wrapper/**", - "**/.gradle/**", - "**/gradle/wrapper/**", - // Generated Flatbuffer files (Kinesis) - "**/org/apache/pulsar/io/kinesis/fbs/*.java", - // Services files - "**/META-INF/services/*", - // Certificates and keys - "**/*.crt", - "**/*.key", - "**/*.csr", - "**/*.pem", - "**/*.srl", - "**/certificate-authority/serial", - "**/certificate-authority/index.txt", - "**/*.json", - "**/*.txt", - // Project/IDE files - "**/*.md", - ".github/**", - "**/*.nar", - "**/.gitignore", - "**/.gitattributes", - "**/*.iml", - "**/.classpath", - "**/.project", - "**/.settings", - "**/.idea/**", - "**/.vscode/**", - // Avro schemas - "**/*.avsc", - // Patch files - "**/*.patch", - // Hidden directories - ".*/**", - // Test output - "**/test-output/**", - // Log files - "**/*.log", - )) +tasks.named("rat").configure { + // Honour .gitignore exclusions so RAT skips untracked/generated files. + // Register .gitignore files as inputs so the task re-runs when they change. + inputs.files(fileTree(rootDir) { + include("**/.gitignore") + exclude("**/build/**") + exclude("**/.gradle/**") + }) + // use crlf plugin's gitignore dsl + gitignore(rootDir) + // Apply additional RAT-specific exclusions from .ratignore. + val ratignoreFile = rootDir.resolve(".ratignore") + inputs.file(ratignoreFile) + exclude(ratignoreFile.readLines().map { it.trim() }.filter { it.isNotBlank() && !it.startsWith("#") }) } val catalog = the().named("libs") diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index da8c28fe9a..19e04d48c9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -567,4 +567,4 @@ spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } idea-ext = { id = "org.jetbrains.gradle.plugin.idea-ext", version.ref = "idea-ext" } version-catalog-update = "nl.littlerobots.version-catalog-update:1.1.0" versions = "com.github.ben-manes.versions:0.53.0" -license = "com.github.hierynomus.license:0.16.1" +crlf = "com.github.vlsi.crlf:3.0.1" From 55218e326c62392640c9ed240b0296accb467f27 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:13:24 +0300 Subject: [PATCH 03/33] Use Jetty 9 for Alluxio --- alluxio/build.gradle.kts | 1 + gradle/libs.versions.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index 0dd5ae199f..048ae4ed98 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -22,6 +22,7 @@ plugins { id("pulsar-connectors.nar-conventions") } dependencies { + enforcedPlatform(platform(libs.jetty9.bom)) implementation(libs.pulsar.io.core) implementation("org.alluxio:alluxio-core-client-fs:2.9.3") implementation(libs.jackson.dataformat.yaml) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 19e04d48c9..606a97d03a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -34,6 +34,7 @@ zookeeper = "3.9.5" netty = "4.1.132.Final" netty-iouring = "0.0.26.Final" jetty = "12.1.5" +jetty9 = "9.4.58.v20250814" jersey = "2.42" jackson = "2.18.6" protobuf = "3.25.5" @@ -280,6 +281,7 @@ curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = conscrypt-openjdk-uber = { module = "org.conscrypt:conscrypt-openjdk-uber", version.ref = "conscrypt" } # Jetty jetty-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty" } +jetty9-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty9" } jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" } jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "jetty" } jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server", version.ref = "jetty" } From 8976c583d2ca451d203ac43c1a2bb535b9ecb647 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:13:39 +0300 Subject: [PATCH 04/33] Log test execution --- .../main/kotlin/pulsar-connectors.java-conventions.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 191a6aa9db..23f16730b9 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -107,7 +107,7 @@ tasks.withType().configureEach { excludeGroups(*(excludedTestGroups.split(",").map { it.trim() }.toTypedArray())) } testLogging { - events("FAILED") + events("passed", "skipped", "failed") exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL showStackTraces = true showExceptions = true From c610ed5ab44637e31dec768a1740b152070b197b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:22:38 +0300 Subject: [PATCH 05/33] fail fast when tests fail --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 347b794409..424363a0c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -89,7 +89,7 @@ jobs: develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Run tests - run: ./gradlew ${{ matrix.tasks }} + run: ./gradlew ${{ matrix.tasks }} -PtestFailFast=true - name: Upload test reports if: failure() uses: actions/upload-artifact@v7 From f060868dae88b69b476afb13d12c4570ec662926 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:50:32 +0300 Subject: [PATCH 06/33] Test with IPv4 --- .../main/kotlin/pulsar-connectors.java-conventions.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 23f16730b9..eab0349ab7 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -119,6 +119,7 @@ tasks.withType().configureEach { failFast = failFastValue systemProperty("testRetryCount", providers.gradleProperty("testRetryCount").getOrElse("1")) systemProperty("testFailFast", failFastValue.toString()) + systemProperty("java.net.preferIPv4Stack", "true") jvmArgs( "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", From 0eb559b57588ac3205ed38dcf2c448e4bca75934 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:51:16 +0300 Subject: [PATCH 07/33] Support showing test output with -PtestShowOutput --- .../main/kotlin/pulsar-connectors.java-conventions.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index eab0349ab7..d333702459 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -112,6 +112,8 @@ tasks.withType().configureEach { showStackTraces = true showExceptions = true showCauses = true + showStandardStreams = providers.gradleProperty("testShowOutput") + .map { it.isBlank() || it.toBoolean() }.getOrElse(false) } maxHeapSize = "1300m" maxParallelForks = 4 From 2ad07d6c0932637619571fd2fb533521d322728a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 00:51:50 +0300 Subject: [PATCH 08/33] Add logging to starting --- .../org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java index 2f777b620d..1ba64ee16b 100644 --- a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java +++ b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java @@ -239,10 +239,13 @@ public Object getNativeObject() { private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { // Setup and start the local alluxio cluster + log.info("Configuring local Alluxio cluster"); LocalAlluxioCluster cluster = new LocalAlluxioCluster(); cluster.initConfiguration(getTestName(getClass().getSimpleName(), "test")); Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); + log.info("Starting local Alluxio cluster"); cluster.start(); + log.info("Alluxio cluster started"); return cluster; } From 8c36313c48fd2156c80fcd2b7e8e074a65213cce Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 01:27:55 +0300 Subject: [PATCH 09/33] Improve test --- .../io/alluxio/sink/AlluxioSinkTest.java | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java index 1ba64ee16b..84e9f95aad 100644 --- a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java +++ b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java @@ -27,6 +27,8 @@ import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; import alluxio.master.LocalAlluxioCluster; +import alluxio.security.authentication.AuthType; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,6 +49,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -64,6 +67,7 @@ public class AlluxioSinkTest { protected Map map; protected AlluxioSink sink; protected LocalAlluxioCluster cluster; + protected String alluxioDir; @Mock protected Record mockRecord; @@ -74,7 +78,7 @@ public class AlluxioSinkTest { static GenericRecord fooBar; @BeforeClass - public static void init() { + public void init() throws Exception { valueSchema = Schema.JSON(Foobar.class); genericSchema = Schema.generic(valueSchema.getSchemaInfo()); fooBar = genericSchema.newRecordBuilder() @@ -83,25 +87,30 @@ public static void init() { .set("age", 20) .build(); kvSchema = Schema.KeyValue(Schema.STRING, genericSchema, KeyValueEncodingType.SEPARATED); - } - @BeforeMethod - public final void setUp() throws Exception { try { cluster = setupSingleMasterCluster(); } catch (java.util.concurrent.TimeoutException e) { throw new org.testng.SkipException( "Skipping test: Alluxio local cluster failed to start within timeout", e); } + } + + @AfterClass + public void destroyCluster() throws Exception { + if (cluster != null) { + cluster.stop(); + } + } + + @BeforeMethod + public final void setUp(Method method) throws Exception { + alluxioDir = "/" + method.getName(); map = new HashMap<>(); - // alluxioMasterHost should be set via LocalAlluxioCluster#getHostname - // instead of using a fixed value "localhost", since it seems that - // LocalAlluxioCluster may bind other address than localhost - // when the node has multiple network interfaces. map.put("alluxioMasterHost", cluster.getHostname()); map.put("alluxioMasterPort", cluster.getMasterRpcPort()); - map.put("alluxioDir", "/pulsar"); + map.put("alluxioDir", alluxioDir); map.put("filePrefix", "prefix"); map.put("schemaEnable", "true"); @@ -131,8 +140,9 @@ public Object getNativeObject() { @AfterMethod public void tearDown() throws Exception { - if (cluster != null) { - cluster.stop(); + if (sink != null) { + sink.close(); + sink = null; } } @@ -143,8 +153,6 @@ public void openTest() throws Exception { map.put("lineSeparator", "\n"); map.put("rotationRecords", "100"); - String alluxioDir = "/pulsar"; - sink = new AlluxioSink(); sink.open(map, mockSinkContext); @@ -156,8 +164,6 @@ public void openTest() throws Exception { String alluxioTmpDir = FilenameUtils.concat(alluxioDir, "tmp"); AlluxioURI alluxioTmpURI = new AlluxioURI(alluxioTmpDir); Assert.assertTrue(client.exists(alluxioTmpURI)); - - sink.close(); } @Test @@ -167,9 +173,6 @@ public void writeAndCloseTest() throws Exception { map.put("lineSeparator", "\n"); map.put("rotationRecords", "1"); map.put("writeType", "THROUGH"); - map.put("alluxioDir", "/pulsar"); - - String alluxioDir = "/pulsar"; sink = new AlluxioSink(); sink.open(map, mockSinkContext); @@ -201,11 +204,9 @@ public Object getNativeObject() { for (String path : pathList) { if (path.contains("tmp")) { - // Ensure that the temporary file is rotated and the directory is empty - Assert.assertEquals(path, "/pulsar/tmp"); + Assert.assertEquals(path, alluxioDir + "/tmp"); } else { - // Ensure that all rotated files conform the naming convention - Assert.assertTrue(path.startsWith("/pulsar/TopicA-")); + Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-")); } } @@ -228,13 +229,11 @@ public Object getNativeObject() { for (String path : pathList) { if (path.contains("tmp")) { - Assert.assertEquals(path, "/pulsar/tmp"); + Assert.assertEquals(path, alluxioDir + "/tmp"); } else { - Assert.assertTrue(path.startsWith("/pulsar/TopicA-")); + Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-")); } } - - sink.close(); } private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { @@ -243,6 +242,16 @@ private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { LocalAlluxioCluster cluster = new LocalAlluxioCluster(); cluster.initConfiguration(getTestName(getClass().getSimpleName(), "test")); Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); + // Disable auth handshake overhead + Configuration.set(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.NOSASL); + Configuration.set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, false); + // Disable unnecessary services + Configuration.set(PropertyKey.USER_METRICS_COLLECTION_ENABLED, false); + Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false); + Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false); + Configuration.set(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED, false); + // Wait for workers to register before master proceeds (avoids "Failed to get worker info" spam) + Configuration.set(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, "5sec"); log.info("Starting local Alluxio cluster"); cluster.start(); log.info("Alluxio cluster started"); From e0022f682b7a1ed2ef3b14f951f1dad2eacc856c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 01:39:35 +0300 Subject: [PATCH 10/33] Exclude jetty-bom from enforcedPlatform added by java-conventions --- alluxio/build.gradle.kts | 6 ++- ...sar-connectors.java-conventions.gradle.kts | 40 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index 048ae4ed98..29d51afbef 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -21,8 +21,12 @@ plugins { id("pulsar-connectors.java-conventions") id("pulsar-connectors.nar-conventions") } +pulsarConnectorsDependencies { + exclude(group = "org.eclipse.jetty", module = "jetty-bom") +} + dependencies { - enforcedPlatform(platform(libs.jetty9.bom)) + implementation(enforcedPlatform(libs.jetty9.bom)) implementation(libs.pulsar.io.core) implementation("org.alluxio:alluxio-core-client-fs:2.9.3") implementation(libs.jackson.dataformat.yaml) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index d333702459..89932732e1 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -60,10 +60,44 @@ if (project.name !in modulesUsingBcFips) { } } +// Extension to control how the shared dependency platform is applied. +data class DependencyExclusion(val group: String, val module: String) + +open class PulsarConnectorsDependenciesExtension { + /** When true (default), uses enforcedPlatform; when false, uses platform. */ + var enforced: Boolean = true + + internal val excludes: MutableList = mutableListOf() + + fun exclude(group: String, module: String) { + excludes.add(DependencyExclusion(group, module)) + } +} + +val pulsarConnectorsDependencies = extensions.create("pulsarConnectorsDependencies") + +// Use withDependencies to lazily add the platform dependency after subproject build scripts +// have configured the extension. This avoids afterEvaluate which is incompatible with +// configuration cache. +val depsHandler = dependencies +configurations["implementation"].withDependencies { + val platformNotation = project(":pulsar-connectors-dependencies") + val configureAction = Action { + (this as ModuleDependency).apply { + pulsarConnectorsDependencies.excludes.forEach { exc -> + exclude(group = exc.group, module = exc.module) + } + } + } + val dep = if (pulsarConnectorsDependencies.enforced) { + depsHandler.enforcedPlatform(platformNotation, configureAction) + } else { + depsHandler.platform(platformNotation, configureAction) + } + add(dep) +} + dependencies { - // Enforced platform pins all dependency versions from the version catalog. - // This is the Gradle equivalent of Maven's dependencyManagement section. - "implementation"(enforcedPlatform(project(":pulsar-connectors-dependencies"))) // Resolve lz4-java capability conflict: at.yawk.lz4:lz4-java (used by Pulsar) and // org.lz4:lz4-java (used by kafka-clients) both provide the org.lz4:lz4-java capability. From 993033b677d895209910a2348ca36d60e92843de Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 01:47:07 +0300 Subject: [PATCH 11/33] Make alluxio dependencies match the versions used in branch-4.2 --- alluxio/build.gradle.kts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index 29d51afbef..a2660fb3b0 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -21,17 +21,34 @@ plugins { id("pulsar-connectors.java-conventions") id("pulsar-connectors.nar-conventions") } + +val alluxioVersion = "2.9.4" + +// Alluxio requires older versions of netty, grpc, and jetty than the shared platform provides. +// Exclude those BOMs so we can pin alluxio-compatible versions below. pulsarConnectorsDependencies { + exclude(group = "io.netty", module = "netty-bom") + exclude(group = "io.grpc", module = "grpc-bom") exclude(group = "org.eclipse.jetty", module = "jetty-bom") + exclude(group = "io.dropwizard.metrics", module = "metrics-jvm") } dependencies { + // Alluxio-compatible BOMs — these override the shared platform versions. implementation(enforcedPlatform(libs.jetty9.bom)) + implementation(enforcedPlatform("io.netty:netty-bom:4.1.100.Final")) + implementation(enforcedPlatform("io.grpc:grpc-bom:1.37.0")) + + // Pin metrics-jvm to the version alluxio expects. + implementation("io.dropwizard.metrics:metrics-jvm:4.1.11") + implementation(libs.pulsar.io.core) - implementation("org.alluxio:alluxio-core-client-fs:2.9.3") + implementation("org.alluxio:alluxio-core-client-fs:$alluxioVersion") implementation(libs.jackson.dataformat.yaml) implementation(libs.guava) testImplementation(libs.pulsar.client) - testImplementation("org.alluxio:alluxio-minicluster:2.9.3") -} + testImplementation("org.alluxio:alluxio-minicluster:$alluxioVersion") { + exclude(group = "org.glassfish", module = "javax.el") + } +} \ No newline at end of file From 80a59095412f619c2c8b01021cc8e8b24cf1682d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:07:39 +0300 Subject: [PATCH 12/33] Fix overriding versions --- alluxio/build.gradle.kts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index a2660fb3b0..3daa42a0a0 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -24,13 +24,11 @@ plugins { val alluxioVersion = "2.9.4" -// Alluxio requires older versions of netty, grpc, and jetty than the shared platform provides. -// Exclude those BOMs so we can pin alluxio-compatible versions below. +// Alluxio requires older versions of netty, grpc, jetty, and metrics than the shared platform +// provides. Use non-enforced (non-strict) shared platform so the alluxio-specific enforced BOMs +// below can override individual version constraints. pulsarConnectorsDependencies { - exclude(group = "io.netty", module = "netty-bom") - exclude(group = "io.grpc", module = "grpc-bom") - exclude(group = "org.eclipse.jetty", module = "jetty-bom") - exclude(group = "io.dropwizard.metrics", module = "metrics-jvm") + enforced = false } dependencies { @@ -39,9 +37,6 @@ dependencies { implementation(enforcedPlatform("io.netty:netty-bom:4.1.100.Final")) implementation(enforcedPlatform("io.grpc:grpc-bom:1.37.0")) - // Pin metrics-jvm to the version alluxio expects. - implementation("io.dropwizard.metrics:metrics-jvm:4.1.11") - implementation(libs.pulsar.io.core) implementation("org.alluxio:alluxio-core-client-fs:$alluxioVersion") implementation(libs.jackson.dataformat.yaml) From e19b38fc12927f1158f51a7292eb888f4d01148b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:09:11 +0300 Subject: [PATCH 13/33] Fix a bug in the sink implementation --- .../org/apache/pulsar/io/alluxio/sink/AlluxioSink.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java index 3b72dc9666..8a6f263096 100644 --- a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java +++ b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java @@ -224,10 +224,11 @@ private void createTmpFile() throws AlluxioException, IOException { } private void closeAndCommitTmpFile() throws AlluxioException, IOException { - // close the tmpFile - if (fileOutStream != null) { - fileOutStream.close(); + if (fileOutStream == null) { + return; } + // close the tmpFile + fileOutStream.close(); // commit the tmpFile String filePrefix = alluxioSinkConfig.getFilePrefix(); String fileExtension = alluxioSinkConfig.getFileExtension(); From 720db60225b0d0feb78d63b515b7d10fb65e825c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:14:51 +0300 Subject: [PATCH 14/33] Fix another bug --- .../pulsar/io/alluxio/sink/AlluxioSink.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java index 8a6f263096..bde3cadcfd 100644 --- a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java +++ b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java @@ -151,11 +151,7 @@ public void write(Record record) { } catch (AlluxioException | IOException e) { log.error("Unable to flush records to alluxio.", e); failRecords(); - try { - deleteTmpFile(); - } catch (AlluxioException | IOException e1) { - log.error("Failed to delete tmp cache file.", e); - } + deleteTmpFile(); break; } case FILE_COMMITTED: @@ -241,9 +237,14 @@ private void closeAndCommitTmpFile() throws AlluxioException, IOException { lastRotationTime = System.currentTimeMillis(); } - private void deleteTmpFile() throws AlluxioException, IOException { - if (!tmpFilePath.equals("")) { + private void deleteTmpFile() { + if (tmpFilePath == null || tmpFilePath.isEmpty()) { + return; + } + try { fileSystem.delete(new AlluxioURI(tmpFilePath)); + } catch (Exception e) { + log.warn("Failed to delete tmp file {}", tmpFilePath, e); } } From f5e2ef8400ec581c1eb4b900b5dc9e30742c181e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:15:03 +0300 Subject: [PATCH 15/33] Fix issue with test --- .../org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java index 84e9f95aad..b1b1d0f15d 100644 --- a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java +++ b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java @@ -250,8 +250,10 @@ private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false); Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false); Configuration.set(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED, false); - // Wait for workers to register before master proceeds (avoids "Failed to get worker info" spam) - Configuration.set(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, "5sec"); + // Minimal safe mode wait — in a local minicluster the worker registers almost instantly. + // A longer wait (e.g. 5s) causes "master is in safe mode" failures when tests write right + // after startup. + Configuration.set(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, "500ms"); log.info("Starting local Alluxio cluster"); cluster.start(); log.info("Alluxio cluster started"); From fcbae7b01d4d017a17befc6ded231c301e386fbc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:21:13 +0300 Subject: [PATCH 16/33] Simplify --- .../kotlin/pulsar-connectors.java-conventions.gradle.kts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 89932732e1..a32287ae70 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -79,9 +79,8 @@ val pulsarConnectorsDependencies = extensions.create { (this as ModuleDependency).apply { pulsarConnectorsDependencies.excludes.forEach { exc -> @@ -90,9 +89,9 @@ configurations["implementation"].withDependencies { } } val dep = if (pulsarConnectorsDependencies.enforced) { - depsHandler.enforcedPlatform(platformNotation, configureAction) + dependencies.enforcedPlatform(platformProject, configureAction) } else { - depsHandler.platform(platformNotation, configureAction) + dependencies.platform(platformProject, configureAction) } add(dep) } From cd2a74845f85b80131260ff325ba6a2e3f2cad1c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 02:28:34 +0300 Subject: [PATCH 17/33] Document the extension --- ...sar-connectors.java-conventions.gradle.kts | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index a32287ae70..499c48f862 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -60,11 +60,28 @@ if (project.name !in modulesUsingBcFips) { } } -// Extension to control how the shared dependency platform is applied. -data class DependencyExclusion(val group: String, val module: String) - +/** + * Configures how the shared `pulsar-connectors-dependencies` platform is applied. + * + * By default, the platform is applied as `enforcedPlatform`, which pins (strictly enforces) all + * dependency versions from the version catalog. Subprojects can customize this behavior: + * + * ```kotlin + * pulsarConnectorsDependencies { + * // Exclude specific dependencies from the platform so they can be overridden locally. + * // Useful when a module needs an older version of a BOM or library (e.g. alluxio needs + * // older netty/grpc). + * exclude(group = "io.netty", module = "netty-bom") + * + * // Set enforced = false to use platform() instead of enforcedPlatform(). This makes all + * // version constraints non-strict: the platform versions are used only as defaults (allowing + * // version omission when declaring dependencies), but can be overridden by the module's own + * // enforcedPlatform or strictly-versioned dependencies. + * enforced = false + * } + * ``` + */ open class PulsarConnectorsDependenciesExtension { - /** When true (default), uses enforcedPlatform; when false, uses platform. */ var enforced: Boolean = true internal val excludes: MutableList = mutableListOf() @@ -74,11 +91,12 @@ open class PulsarConnectorsDependenciesExtension { } } +data class DependencyExclusion(val group: String, val module: String) + val pulsarConnectorsDependencies = extensions.create("pulsarConnectorsDependencies") -// Use withDependencies to lazily add the platform dependency after subproject build scripts -// have configured the extension. This avoids afterEvaluate which is incompatible with -// configuration cache. +// withDependencies runs lazily after subproject build scripts have configured the extension. +// This is configuration-cache compatible (unlike afterEvaluate). configurations["implementation"].withDependencies { val platformProject = project(":pulsar-connectors-dependencies") val configureAction = Action { From 8df1002d91a13fe0b36f17658408e6ebb3975ae7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 03:06:22 +0300 Subject: [PATCH 18/33] Fix nar project artifacts --- ...lsar-connectors.nar-conventions.gradle.kts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts index d51ebc8ad2..a5d803319a 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts @@ -91,3 +91,24 @@ if (parentProject != null && parentProject != rootProject && parentProject.paren val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as org.gradle.api.provider.Property narIdProp.set(qualifiedName) } + +tasks.named("jar") { + enabled = false +} + +configurations { + named("runtimeElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("nar")) + variants.clear() + } + } + named("apiElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("nar")) + variants.clear() + } + } +} \ No newline at end of file From 1501462cdade62f4fb6e4feca79311076fa7df60 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 03:08:13 +0300 Subject: [PATCH 19/33] Fix ConnectorDocGenerator --- .../pulsar/io/docs/ConnectorDocGenerator.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java index 2e9d6a9f27..e159c889ac 100644 --- a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java +++ b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java @@ -28,6 +28,7 @@ import java.lang.reflect.Modifier; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -51,8 +52,11 @@ public class ConnectorDocGenerator implements Callable { private static Reflections newReflections() throws Exception { final String[] classpathList = System.getProperty("java.class.path").split(":"); final List urlList = new ArrayList<>(); - for (String file : classpathList) { - urlList.add(new File(file).toURI().toURL()); + for (String cp : classpathList) { + File file = new File(cp); + if (file.isFile() && cp.endsWith(".nar")) { + urlList.add(new URL("jar:" + file.toURI() + "!/")); + } } return new Reflections(new ConfigurationBuilder().setUrls(urlList)); } @@ -70,7 +74,7 @@ private void generateConnectorYamlFile(Class configClass, PrintWriter writer) Field[] fields = configClass.getDeclaredFields(); for (Field field : fields) { - if (Modifier.isStatic(field.getModifiers())) { + if (Modifier.isStatic(field.getModifiers()) || Modifier.isTransient(field.getModifiers())) { continue; } FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class); @@ -107,12 +111,16 @@ private void generatorConnectorYamlFiles(String outputDir) throws IOException { Set> connectorClasses = reflections.getTypesAnnotatedWith(Connector.class); log.info("Retrieve all `Connector` annotated classes : {}", connectorClasses); + Path outputDirPath = Path.of(outputDir); + if (!Files.exists(outputDirPath)) { + Files.createDirectories(outputDirPath); + } for (Class connectorClass : connectorClasses) { final Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); final String name = connectorDef.name().toLowerCase(); final String type = connectorDef.type().name().toLowerCase(); final String filename = "pulsar-io-%s-%s.yml".formatted(name, type); - final Path outputPath = Path.of(outputDir, filename); + final Path outputPath = outputDirPath.resolve(filename); try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); generateConnectorYamlFile(connectorClass, connectorDef, pw); @@ -125,7 +133,7 @@ private void generatorConnectorYamlFiles(String outputDir) throws IOException { names = {"-o", "--output-dir"}, description = "The output dir to dump connector docs", required = true) - String outputDir = null; + String outputDir; @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") boolean help = false; From c803d8ef31c9f71d9c1f990c334a6f74fc8c3b6b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 03:08:30 +0300 Subject: [PATCH 20/33] Fix alluxio dependency in docs projects to avoid conflicts --- docs/build.gradle.kts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index ab0677af46..8a93a266fa 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -55,6 +55,10 @@ dependencies { implementation(project(":rabbitmq")) implementation(project(":redis")) implementation(project(":solr")) - implementation(project(":alluxio")) + implementation(project(":alluxio")) { + exclude("org.eclipse.jetty", "jetty-bom") + exclude("io.netty", "netty-bom") + exclude("io.grpc", "grpc-bom") + } implementation(project(":azure-data-explorer")) } From d06f91ab77fb09adc4cd173e937536e48d5f05de Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 03:28:13 +0300 Subject: [PATCH 21/33] Add pulsar-io-gen.sh adapted to gradle --- docs/build.gradle.kts | 24 ++++++++++++++++++++++++ docs/pulsar-io-gen.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100755 docs/pulsar-io-gen.sh diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 8a93a266fa..0f89397b3f 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -62,3 +62,27 @@ dependencies { } implementation(project(":azure-data-explorer")) } + +val exportClasspath by tasks.registering { + dependsOn(tasks.classes) + val classpath = sourceSets.main.get().output + configurations.runtimeClasspath.get() + inputs.files(classpath) + val outputFile = layout.buildDirectory.file("classpath.txt") + outputs.file(outputFile) + doLast { + outputFile.get().asFile.apply { + parentFile.mkdirs() + writeText(classpath.asPath) + } + } +} + +tasks.register("generateConnectorDocs") { + dependsOn(tasks.classes) + mainClass.set("org.apache.pulsar.io.docs.ConnectorDocGenerator") + classpath = sourceSets.main.get().output + configurations.runtimeClasspath.get() + inputs.files(classpath) + val outputDir = layout.buildDirectory.dir("connector-docs") + outputs.dir(outputDir) + args("-o", outputDir.get().asFile.absolutePath) +} \ No newline at end of file diff --git a/docs/pulsar-io-gen.sh b/docs/pulsar-io-gen.sh new file mode 100755 index 0000000000..0f2416a7e3 --- /dev/null +++ b/docs/pulsar-io-gen.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# Check for the java to use +if [[ -z $JAVA_HOME ]]; then + JAVA=$(which java) + if [ $? != 0 ]; then + echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 + exit 1 + fi +else + JAVA=$JAVA_HOME/bin/java +fi +CLASSPATH_FILE="$SCRIPT_DIR/build/classpath.txt" +$SCRIPT_DIR/../gradlew exportClasspath --console=plain > /dev/null +$JAVA -cp "$(cat $CLASSPATH_FILE)" org.apache.pulsar.io.docs.ConnectorDocGenerator "$@" \ No newline at end of file From c925780f6f58119af6d3b31022bc5da5c06e1317 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 12:00:24 +0300 Subject: [PATCH 22/33] Rely on BOMs for versions --- ...sar-connectors.java-conventions.gradle.kts | 8 - gradle/libs.versions.toml | 197 +++++++++--------- .../build.gradle.kts | 7 +- 3 files changed, 105 insertions(+), 107 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 499c48f862..ec43768be9 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -41,14 +41,6 @@ configurations.all { // Pulsar uses SLF4J 2.x with log4j-slf4j2-impl; having both causes // NoSuchMethodError in Log4jLoggerFactory at test startup. exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl") - - // Force Jackson version to match the version catalog. Transitive dependencies - // (e.g. from jackson-bom) can pull in newer versions that break API compatibility. - resolutionStrategy.eachDependency { - if (requested.group.startsWith("com.fasterxml.jackson")) { - useVersion(catalog.findVersion("jackson").get().requiredVersion) - } - } } // Exclude bc-fips from modules that don't need it. bc-fips's CryptoServicesRegistrar diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 606a97d03a..f112d88a8b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -200,55 +200,56 @@ com-4paradigm-openmldb = "0.4.4-hotfix1" [libraries] # SLF4J slf4j-bom = { module = "org.slf4j:slf4j-bom", version.ref = "slf4j" } -slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } -jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } +slf4j-api = { module = "org.slf4j:slf4j-api" } +jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j" } # Log4j2 log4j-bom = { module = "org.apache.logging.log4j:log4j-bom", version.ref = "log4j2" } -log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j2" } -log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j2" } -log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j2" } -log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json", version.ref = "log4j2" } -log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j2" } +log4j-api = { module = "org.apache.logging.log4j:log4j-api" } +log4j-core = { module = "org.apache.logging.log4j:log4j-core" } +log4j-web = { module = "org.apache.logging.log4j:log4j-web" } +log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json" } +log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl" } # Lombok lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } # Jackson jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson" } -jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" } -jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson" } -jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } -jackson-module-parameter-names = { module = "com.fasterxml.jackson.module:jackson-module-parameter-names", version.ref = "jackson" } -jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } -jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", version.ref = "jackson" } -jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } -jackson-module-jsonSchema = { module = "com.fasterxml.jackson.module:jackson-module-jsonSchema", version.ref = "jackson" } -jackson-jaxrs-json-provider = { module = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", version.ref = "jackson" } +jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations" } +jackson-core = { module = "com.fasterxml.jackson.core:jackson-core" } +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } +jackson-module-parameter-names = { module = "com.fasterxml.jackson.module:jackson-module-parameter-names" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } +jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8" } +jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } +jackson-module-jsonSchema = { module = "com.fasterxml.jackson.module:jackson-module-jsonSchema" } +jackson-jaxrs-json-provider = { module = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider" } +jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor" } # Netty netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } -netty-common = { module = "io.netty:netty-common", version.ref = "netty" } -netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty" } -netty-handler = { module = "io.netty:netty-handler", version.ref = "netty" } -netty-transport = { module = "io.netty:netty-transport", version.ref = "netty" } -netty-codec-http = { module = "io.netty:netty-codec-http", version.ref = "netty" } -netty-codec-http2 = { module = "io.netty:netty-codec-http2", version.ref = "netty" } -netty-codec-haproxy = { module = "io.netty:netty-codec-haproxy", version.ref = "netty" } -netty-handler-proxy = { module = "io.netty:netty-handler-proxy", version.ref = "netty" } -netty-codec-socks = { module = "io.netty:netty-codec-socks", version.ref = "netty" } -netty-resolver-dns = { module = "io.netty:netty-resolver-dns", version.ref = "netty" } -netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native-macos", version.ref = "netty" } -netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" } -netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common", version.ref = "netty" } +netty-common = { module = "io.netty:netty-common" } +netty-buffer = { module = "io.netty:netty-buffer" } +netty-handler = { module = "io.netty:netty-handler" } +netty-transport = { module = "io.netty:netty-transport" } +netty-codec-http = { module = "io.netty:netty-codec-http" } +netty-codec-http2 = { module = "io.netty:netty-codec-http2" } +netty-codec-haproxy = { module = "io.netty:netty-codec-haproxy" } +netty-handler-proxy = { module = "io.netty:netty-handler-proxy" } +netty-codec-socks = { module = "io.netty:netty-codec-socks" } +netty-resolver-dns = { module = "io.netty:netty-resolver-dns" } +netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native-macos" } +netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll" } +netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common" } netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" } netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" } netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" } netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" } # Protobuf / gRPC protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref = "protobuf" } -protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } -protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util", version.ref = "protobuf" } +protobuf-java = { module = "com.google.protobuf:protobuf-java" } +protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util" } grpc-bom = { module = "io.grpc:grpc-bom", version.ref = "grpc" } -grpc-all = { module = "io.grpc:grpc-all", version.ref = "grpc" } -grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref = "grpc" } -grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } +grpc-all = { module = "io.grpc:grpc-all" } +grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded" } +grpc-stub = { module = "io.grpc:grpc-stub" } # Guava guava = { module = "com.google.guava:guava", version.ref = "guava" } # Apache Commons @@ -282,52 +283,53 @@ conscrypt-openjdk-uber = { module = "org.conscrypt:conscrypt-openjdk-uber", vers # Jetty jetty-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty" } jetty9-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty9" } -jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" } -jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "jetty" } -jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server", version.ref = "jetty" } -jetty-compression-server = { module = "org.eclipse.jetty.compression:jetty-compression-server", version.ref = "jetty" } -jetty-compression-gzip = { module = "org.eclipse.jetty.compression:jetty-compression-gzip", version.ref = "jetty" } -jetty-client = { module = "org.eclipse.jetty:jetty-client", version.ref = "jetty" } -jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet", version.ref = "jetty" } -jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets", version.ref = "jetty" } -jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy", version.ref = "jetty" } -jetty-websocket-jetty-api = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-api", version.ref = "jetty" } -jetty-websocket-jetty-client = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-client", version.ref = "jetty" } -jetty-ee8-websocket-jetty-server = { module = "org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server", version.ref = "jetty" } +jetty-server = { module = "org.eclipse.jetty:jetty-server" } +jetty-util = { module = "org.eclipse.jetty:jetty-util" } +jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server" } +jetty-compression-server = { module = "org.eclipse.jetty.compression:jetty-compression-server" } +jetty-compression-gzip = { module = "org.eclipse.jetty.compression:jetty-compression-gzip" } +jetty-client = { module = "org.eclipse.jetty:jetty-client" } +jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet" } +jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets" } +jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy" } +jetty-websocket-jetty-api = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-api" } +jetty-websocket-jetty-client = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-client" } +jetty-ee8-websocket-jetty-server = { module = "org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server" } # Jersey -jersey-server = { module = "org.glassfish.jersey.core:jersey-server", version.ref = "jersey" } -jersey-container-servlet-core = { module = "org.glassfish.jersey.containers:jersey-container-servlet-core", version.ref = "jersey" } -jersey-container-servlet = { module = "org.glassfish.jersey.containers:jersey-container-servlet", version.ref = "jersey" } -jersey-media-json-jackson = { module = "org.glassfish.jersey.media:jersey-media-json-jackson", version.ref = "jersey" } -jersey-client = { module = "org.glassfish.jersey.core:jersey-client", version.ref = "jersey" } -jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2", version.ref = "jersey" } -jersey-media-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } -jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jersey-test-framework-core", version.ref = "jersey" } -jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2", version.ref = "jersey" } +jersey-bom = { module = "org.glassfish.jersey:jersey-bom", version.ref = "jersey" } +jersey-server = { module = "org.glassfish.jersey.core:jersey-server" } +jersey-container-servlet-core = { module = "org.glassfish.jersey.containers:jersey-container-servlet-core" } +jersey-container-servlet = { module = "org.glassfish.jersey.containers:jersey-container-servlet" } +jersey-media-json-jackson = { module = "org.glassfish.jersey.media:jersey-media-json-jackson" } +jersey-client = { module = "org.glassfish.jersey.core:jersey-client" } +jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2" } +jersey-media-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart" } +jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jersey-test-framework-core" } +jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2" } # Prometheus simpleclient-bom = { module = "io.prometheus:simpleclient_bom", version.ref = "prometheus" } -simpleclient = { module = "io.prometheus:simpleclient", version.ref = "prometheus" } -simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot", version.ref = "prometheus" } -simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine", version.ref = "prometheus" } -simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver", version.ref = "prometheus" } -simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet", version.ref = "prometheus" } -simpleclient-common = { module = "io.prometheus:simpleclient_common", version.ref = "prometheus" } -simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2", version.ref = "prometheus" } +simpleclient = { module = "io.prometheus:simpleclient" } +simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot" } +simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine" } +simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver" } +simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet" } +simpleclient-common = { module = "io.prometheus:simpleclient_common" } +simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2" } prometheus-jmx-collector = { module = "io.prometheus.jmx:collector", version.ref = "prometheus-jmx" } # OpenTelemetry opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version.ref = "opentelemetry" } opentelemetry-bom-alpha = { module = "io.opentelemetry:opentelemetry-bom-alpha", version.ref = "opentelemetry-alpha" } opentelemetry-instrumentation-bom = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom", version.ref = "opentelemetry-instrumentation" } opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version.ref = "opentelemetry-instrumentation-alpha" } -opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } -opentelemetry-api-incubator = { module = "io.opentelemetry:opentelemetry-api-incubator", version.ref = "opentelemetry-alpha" } -opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } -opentelemetry-sdk-extension-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure", version.ref = "opentelemetry" } -opentelemetry-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing", version.ref = "opentelemetry" } -opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version.ref = "opentelemetry" } -opentelemetry-exporter-prometheus = { module = "io.opentelemetry:opentelemetry-exporter-prometheus", version.ref = "opentelemetry-alpha" } -opentelemetry-instrumentation-resources = { module = "io.opentelemetry.instrumentation:opentelemetry-resources", version.ref = "opentelemetry-instrumentation-alpha" } -opentelemetry-instrumentation-runtime-telemetry-java17 = { module = "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17", version.ref = "opentelemetry-instrumentation-alpha" } +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api" } +opentelemetry-api-incubator = { module = "io.opentelemetry:opentelemetry-api-incubator" } +opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk" } +opentelemetry-sdk-extension-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure" } +opentelemetry-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp" } +opentelemetry-exporter-prometheus = { module = "io.opentelemetry:opentelemetry-exporter-prometheus" } +opentelemetry-instrumentation-resources = { module = "io.opentelemetry.instrumentation:opentelemetry-resources" } +opentelemetry-instrumentation-runtime-telemetry-java17 = { module = "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17" } opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version.ref = "opentelemetry-semconv" } opentelemetry-gcp-resources = { module = "io.opentelemetry.contrib:opentelemetry-gcp-resources", version.ref = "opentelemetry-gcp-resources" } # BouncyCastle @@ -361,10 +363,10 @@ commons-text = { module = "org.apache.commons:commons-text", version.ref = "comm typetools = { module = "net.jodah:typetools", version.ref = "typetools" } perfmark-api = { module = "io.perfmark:perfmark-api", version.ref = "perfmark" } okhttp3-bom = { module = "com.squareup.okhttp3:okhttp-bom", version.ref = "okhttp3" } -okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm", version.ref = "okhttp3" } -okhttp3-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp3" } +okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm" } +okhttp3-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor" } okio-bom = { module = "com.squareup.okio:okio-bom", version.ref = "okio" } -okio = { module = "com.squareup.okio:okio", version.ref = "okio" } +okio = { module = "com.squareup.okio:okio" } # Transitive dep version pins (enforced by pulsar-connectors-dependencies platform) google-auth-library-credentials = { module = "com.google.auth:google-auth-library-credentials", version.ref = "google-auth" } google-auth-library-oauth2-http = { module = "com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth" } @@ -399,9 +401,10 @@ snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy" jspecify = "org.jspecify:jspecify:1.0.0" reflections = { module = "org.reflections:reflections", version.ref = "reflections" } # Auth / Security -jjwt-api = { module = "io.jsonwebtoken:jjwt-api", version.ref = "jsonwebtoken" } -jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl", version.ref = "jsonwebtoken" } -jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson", version.ref = "jsonwebtoken" } +jjwt-bom = { module = "io.jsonwebtoken:jjwt-bom", version.ref = "jsonwebtoken" } +jjwt-api = { module = "io.jsonwebtoken:jjwt-api" } +jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl" } +jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson" } auth0-java-jwt = { module = "com.auth0:java-jwt", version.ref = "auth0-java-jwt" } auth0-jwks-rsa = { module = "com.auth0:jwks-rsa", version.ref = "auth0-jwks-rsa" } # Jakarta @@ -421,7 +424,7 @@ jsr305 = { module = "com.google.code.findbugs:jsr305", version.ref = "jsr305" } # Test testng = { module = "org.testng:testng", version.ref = "testng" } mockito-bom = { module = "org.mockito:mockito-bom", version.ref = "mockito" } -mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } +mockito-core = { module = "org.mockito:mockito-core" } awaitility = { module = "org.awaitility:awaitility", version.ref = "awaitility" } assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" } @@ -442,28 +445,29 @@ spring-orm = { module = "org.springframework:spring-orm", version.ref = "spring" kubernetes-client-java = { module = "io.kubernetes:client-java", version.ref = "kubernetesclient" } kubernetes-client-java-api-fluent = { module = "io.kubernetes:client-java-api-fluent", version.ref = "kubernetesclient" } testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version.ref = "testcontainers" } -testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } -testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "testcontainers" } -testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy", version.ref = "testcontainers" } -testcontainers-localstack = { module = "org.testcontainers:localstack", version.ref = "testcontainers" } -testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq", version.ref = "testcontainers" } -testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } -testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref = "testcontainers" } -testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } -testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "testcontainers" } -testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "testcontainers" } -testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "testcontainers" } -testcontainers-k3s = { module = "org.testcontainers:k3s", version.ref = "testcontainers" } +testcontainers = { module = "org.testcontainers:testcontainers" } +testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch" } +testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy" } +testcontainers-localstack = { module = "org.testcontainers:localstack" } +testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq" } +testcontainers-kafka = { module = "org.testcontainers:kafka" } +testcontainers-mysql = { module = "org.testcontainers:mysql" } +testcontainers-postgresql = { module = "org.testcontainers:postgresql" } +testcontainers-mongodb = { module = "org.testcontainers:mongodb" } +testcontainers-pulsar = { module = "org.testcontainers:pulsar" } +testcontainers-cassandra = { module = "org.testcontainers:cassandra" } +testcontainers-k3s = { module = "org.testcontainers:k3s" } kerby-simplekdc = { module = "org.apache.kerby:kerb-simplekdc", version.ref = "kerby" } json = { module = "org.json:json", version.ref = "json" } # AWS SDKs aws-java-sdk-bom = { module = "com.amazonaws:aws-java-sdk-bom", version.ref = "aws-sdk" } -aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core", version.ref = "aws-sdk" } -aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts", version.ref = "aws-sdk" } -aws-sdk2-regions = { module = "software.amazon.awssdk:regions", version.ref = "aws-sdk2" } -aws-sdk2-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws-sdk2" } -aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis", version.ref = "aws-sdk2" } -aws-sdk2-utils = { module = "software.amazon.awssdk:utils", version.ref = "aws-sdk2" } +aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core" } +aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts" } +aws-sdk2-bom = { module = "software.amazon.awssdk:bom", version.ref = "aws-sdk2" } +aws-sdk2-regions = { module = "software.amazon.awssdk:regions" } +aws-sdk2-sts = { module = "software.amazon.awssdk:sts" } +aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis" } +aws-sdk2-utils = { module = "software.amazon.awssdk:utils" } dynamodb-streams-kinesis-adapter = "com.amazonaws:dynamodb-streams-kinesis-adapter:1.6.0" amazon-kinesis-client = "software.amazon.kinesis:amazon-kinesis-client:2.6.0" amazon-kinesis-client-v3 = "software.amazon.kinesis:amazon-kinesis-client:3.1.2" @@ -524,7 +528,6 @@ nsq-j = { module = "com.sproutsocial:nsq-j", version.ref = "nsq-client" } influxdb-client-java = { module = "com.influxdb:influxdb-client-java", version.ref = "influxdb-client" } influxdb-java = { module = "org.influxdb:influxdb-java", version.ref = "influxdb-java" } # IO specific -jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref = "jackson" } json-smart = "net.minidev:json-smart:2.5.2" json-flattener = "com.github.wnameless.json:json-flattener:0.16.4" flatbuffers-java = "com.google.flatbuffers:flatbuffers-java:1.9.0" diff --git a/pulsar-connectors-dependencies/build.gradle.kts b/pulsar-connectors-dependencies/build.gradle.kts index 66f96c6642..7252f010f3 100644 --- a/pulsar-connectors-dependencies/build.gradle.kts +++ b/pulsar-connectors-dependencies/build.gradle.kts @@ -37,11 +37,14 @@ dependencies { // BOM entries (detected by module name) are imported as platforms rather than constraints. catalog.libraryAliases.forEach { alias -> catalog.findLibrary(alias).ifPresent { provider -> - val module = provider.get().module + val dep = provider.get() + val module = dep.module if (module.name.endsWith("-bom") || module.name.endsWith("_bom") || module.name == "bom" || module.name.contains("-bom-") || module.name.contains("_bom_")) { api(platform(provider)) - } else { + } else if (dep.versionConstraint.requiredVersion.isNotEmpty()) { + // Only add constraints for entries with explicit versions. + // Versionless entries are managed by BOMs imported above. constraints.api(provider) } } From 6c36ca826092924432b7b6ac9c4d13a545376bc9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 12:15:37 +0300 Subject: [PATCH 23/33] Further cleanup --- gradle/libs.versions.toml | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f112d88a8b..1fb9b7631b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,9 +21,6 @@ pulsar = "4.1.3" pulsar-connectors = "4.3.0-SNAPSHOT" java = "17" -# Docker -docker-jdk = "21" -pulsar-client-python = "3.10.0" # Code quality checkstyle = "13.3.0" spotless = "8.4.0" @@ -78,7 +75,6 @@ asynchttpclient = "2.12.4" conscrypt = "2.5.2" okhttp3 = "5.3.1" okio = "3.16.3" -netty-tcnative = "2.0.75.Final" httpcomponents-httpclient = "4.5.13" httpcomponents-httpcore = "4.4.15" # Google libraries (transitive deps, versions managed to match Maven) @@ -193,9 +189,10 @@ solr = "9.8.0" hbase = "2.6.4-hadoop3" hadoop3 = "3.4.3" jclouds = "2.6.0" +openmldb = "0.4.4-hotfix1" +docker-java = "3.7.1" # Shading shadow = "9.4.1" -com-4paradigm-openmldb = "0.4.4-hotfix1" [libraries] # SLF4J @@ -206,39 +203,20 @@ jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j" } log4j-bom = { module = "org.apache.logging.log4j:log4j-bom", version.ref = "log4j2" } log4j-api = { module = "org.apache.logging.log4j:log4j-api" } log4j-core = { module = "org.apache.logging.log4j:log4j-core" } -log4j-web = { module = "org.apache.logging.log4j:log4j-web" } -log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json" } log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl" } # Lombok lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } # Jackson jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson" } -jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations" } -jackson-core = { module = "com.fasterxml.jackson.core:jackson-core" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } -jackson-module-parameter-names = { module = "com.fasterxml.jackson.module:jackson-module-parameter-names" } jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } -jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8" } jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } -jackson-module-jsonSchema = { module = "com.fasterxml.jackson.module:jackson-module-jsonSchema" } -jackson-jaxrs-json-provider = { module = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider" } jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor" } # Netty netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } -netty-common = { module = "io.netty:netty-common" } netty-buffer = { module = "io.netty:netty-buffer" } netty-handler = { module = "io.netty:netty-handler" } -netty-transport = { module = "io.netty:netty-transport" } netty-codec-http = { module = "io.netty:netty-codec-http" } -netty-codec-http2 = { module = "io.netty:netty-codec-http2" } -netty-codec-haproxy = { module = "io.netty:netty-codec-haproxy" } -netty-handler-proxy = { module = "io.netty:netty-handler-proxy" } -netty-codec-socks = { module = "io.netty:netty-codec-socks" } -netty-resolver-dns = { module = "io.netty:netty-resolver-dns" } -netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native-macos" } -netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll" } -netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common" } -netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" } netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" } netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" } netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" } @@ -308,13 +286,6 @@ jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jer jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2" } # Prometheus simpleclient-bom = { module = "io.prometheus:simpleclient_bom", version.ref = "prometheus" } -simpleclient = { module = "io.prometheus:simpleclient" } -simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot" } -simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine" } -simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver" } -simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet" } -simpleclient-common = { module = "io.prometheus:simpleclient_common" } -simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2" } prometheus-jmx-collector = { module = "io.prometheus.jmx:collector", version.ref = "prometheus-jmx" } # OpenTelemetry opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version.ref = "opentelemetry" } @@ -444,6 +415,7 @@ spring-jdbc = { module = "org.springframework:spring-jdbc", version.ref = "sprin spring-orm = { module = "org.springframework:spring-orm", version.ref = "spring" } kubernetes-client-java = { module = "io.kubernetes:client-java", version.ref = "kubernetesclient" } kubernetes-client-java-api-fluent = { module = "io.kubernetes:client-java-api-fluent", version.ref = "kubernetesclient" } +docker-java-bom = { module = "com.github.docker-java:docker-java-bom", version.ref = "docker-java" } testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version.ref = "testcontainers" } testcontainers = { module = "org.testcontainers:testcontainers" } testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch" } @@ -497,8 +469,8 @@ postgresql-jdbc = "org.postgresql:postgresql:42.7.10" sqlite-jdbc = "org.xerial:sqlite-jdbc:3.47.1.0" clickhouse-jdbc = "com.clickhouse:clickhouse-jdbc:0.4.6" mariadb-jdbc = "org.mariadb.jdbc:mariadb-java-client:3.5.5" -openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref = "com-4paradigm-openmldb" } -openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version.ref = "com-4paradigm-openmldb" } +openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref = "openmldb" } +openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version.ref = "openmldb" } # JClouds jclouds-allblobstore = { module = "org.apache.jclouds:jclouds-allblobstore", version.ref = "jclouds" } jclouds-blobstore = { module = "org.apache.jclouds:jclouds-blobstore", version.ref = "jclouds" } @@ -514,7 +486,6 @@ hbase-common = { module = "org.apache.hbase:hbase-common", version.ref = "hbase" aerospike-client = { module = "com.aerospike:aerospike-client-bc", version.ref = "aerospike-client" } cassandra-driver = { module = "com.datastax.cassandra:cassandra-driver-core", version.ref = "cassandra-driver" } failsafe = { module = "dev.failsafe:failsafe", version.ref = "failsafe" } -docker-java-core = "com.github.docker-java:docker-java-core:3.4.1" mongodb-driver-reactivestreams = { module = "org.mongodb:mongodb-driver-reactivestreams", version.ref = "mongodb-driver" } mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version.ref = "mongodb-driver" } lettuce-core = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" } From d92a8369ecd9761bd92610dcfcbf38fd658f96ea Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 12:48:48 +0300 Subject: [PATCH 24/33] Cleanup --- ...sar-connectors.java-conventions.gradle.kts | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index ec43768be9..2095359326 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -24,6 +24,11 @@ plugins { val catalog = the().named("libs") +fun lib(alias: String): Provider = + catalog.findLibrary(alias).orElseThrow { + GradleException("Library alias '$alias' not found in version catalog 'libs'") + } + group = "org.apache.pulsar" version = catalog.findVersion("pulsar-connectors").get().requiredVersion @@ -118,26 +123,26 @@ dependencies { } // Annotation processing for Lombok - "compileOnly"(catalog.findLibrary("lombok").get()) - "annotationProcessor"(catalog.findLibrary("lombok").get()) - "testCompileOnly"(catalog.findLibrary("lombok").get()) - "testAnnotationProcessor"(catalog.findLibrary("lombok").get()) + "compileOnly"(lib("lombok")) + "annotationProcessor"(lib("lombok")) + "testCompileOnly"(lib("lombok")) + "testAnnotationProcessor"(lib("lombok")) // Common test dependencies - "testImplementation"(catalog.findLibrary("testng").get()) - "testImplementation"(catalog.findLibrary("mockito-core").get()) - "testImplementation"(catalog.findLibrary("assertj-core").get()) - "testImplementation"(catalog.findLibrary("awaitility").get()) - "testImplementation"(catalog.findLibrary("system-lambda").get()) - "testImplementation"(catalog.findLibrary("slf4j-api").get()) + "testImplementation"(lib("testng")) + "testImplementation"(lib("mockito-core")) + "testImplementation"(lib("assertj-core")) + "testImplementation"(lib("awaitility")) + "testImplementation"(lib("system-lambda")) + "testImplementation"(lib("slf4j-api")) // Logging runtime for tests — provides Log4j2 as the SLF4J backend. // Some connectors (Alluxio minicluster, Solr embedded) require a logging // implementation to be present at test runtime. - "testRuntimeOnly"(catalog.findLibrary("log4j-api").get()) - "testRuntimeOnly"(catalog.findLibrary("log4j-core").get()) - "testRuntimeOnly"(catalog.findLibrary("log4j-slf4j2-impl").get()) - "testRuntimeOnly"(catalog.findLibrary("jcl-over-slf4j").get()) + "testRuntimeOnly"(lib("log4j-api")) + "testRuntimeOnly"(lib("log4j-core")) + "testRuntimeOnly"(lib("log4j-slf4j2-impl")) + "testRuntimeOnly"(lib("jcl-over-slf4j")) } tasks.withType().configureEach { From 1c425bca8bce6148d4275650fe8c6cac3807a5a3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:07:13 +0300 Subject: [PATCH 25/33] Define project group and version in the correct location --- .../kotlin/pulsar-connectors.java-conventions.gradle.kts | 3 --- build.gradle.kts | 8 -------- gradle.properties | 3 +++ gradle/libs.versions.toml | 1 - 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 2095359326..d1cfa01bb3 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -29,9 +29,6 @@ fun lib(alias: String): Provider = GradleException("Library alias '$alias' not found in version catalog 'libs'") } -group = "org.apache.pulsar" -version = catalog.findVersion("pulsar-connectors").get().requiredVersion - // Add shared test resources (log4j2-test.xml) to the test classpath for all modules. the()["test"].resources.srcDir(rootProject.file("gradle/test-resources")) diff --git a/build.gradle.kts b/build.gradle.kts index db427cb7b0..9f0827841c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -64,14 +64,6 @@ tasks.named("rat").configure { exclude(ratignoreFile.readLines().map { it.trim() }.filter { it.isNotBlank() && !it.startsWith("#") }) } -val catalog = the().named("libs") -val pulsarConnectorsVersion = catalog.findVersion("pulsar-connectors").get().requiredVersion - -allprojects { - group = "org.apache.pulsar" - version = pulsarConnectorsVersion -} - idea { project { settings { diff --git a/gradle.properties b/gradle.properties index 657c24b120..fc83eeeece 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,6 +17,9 @@ # under the License. # +group=org.apache.pulsar +version=4.3.0-SNAPSHOT + org.gradle.configuration-cache=true org.gradle.configureondemand=true org.gradle.parallel=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1fb9b7631b..4c75fed878 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,6 @@ [versions] # Core pulsar = "4.1.3" -pulsar-connectors = "4.3.0-SNAPSHOT" java = "17" # Code quality checkstyle = "13.3.0" From 50a92c2087953ec18caa48d845970d6456031df0 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:07:35 +0300 Subject: [PATCH 26/33] configure the shaded client netty property --- .../main/kotlin/pulsar-connectors.java-conventions.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index d1cfa01bb3..41a0bf91fc 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -181,6 +181,7 @@ tasks.withType().configureEach { "-XX:+EnableDynamicAgentLoading", "-Xshare:off", "-Dio.netty.tryReflectionSetAccessible=true", + "-Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true", "-Dpulsar.allocator.pooled=true", "-Dpulsar.allocator.exit_on_oom=false", "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", From e837a15bb404e9c13dd9ae6c0425a34d4e73690d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:14:36 +0300 Subject: [PATCH 27/33] Cleanup debezium by using bom --- gradle/libs.versions.toml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4c75fed878..ec94812fbd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -171,7 +171,6 @@ confluent = "8.1.1" opensearch = "2.19.4" elasticsearch-java = "8.15.3" debezium = "3.4.2.Final" -debezium-mysql-connector = "9.4.0" kubernetesclient = "23.0.0" # IO connector specific aerospike-client = "4.5.0" @@ -457,17 +456,19 @@ kafka-connect-avro-converter = { module = "io.confluent:kafka-connect-avro-conve opensearch-rest-high-level-client = { module = "org.opensearch.client:opensearch-rest-high-level-client", version.ref = "opensearch" } elasticsearch-java = { module = "co.elastic.clients:elasticsearch-java", version.ref = "elasticsearch-java" } # Debezium -debezium-core = { module = "io.debezium:debezium-core", version.ref = "debezium" } -debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium" } -debezium-connector-mongodb = { module = "io.debezium:debezium-connector-mongodb", version.ref = "debezium" } -debezium-connector-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium" } -debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle", version.ref = "debezium" } -debezium-connector-sqlserver = { module = "io.debezium:debezium-connector-sqlserver", version.ref = "debezium" } +debezium-bom = { module = "io.debezium:debezium-bom", version.ref = "debezium" } +debezium-core = { module = "io.debezium:debezium-core" } +debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql" } +debezium-connector-mongodb = { module = "io.debezium:debezium-connector-mongodb" } +debezium-connector-postgres = { module = "io.debezium:debezium-connector-postgres" } +debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle" } +debezium-connector-sqlserver = { module = "io.debezium:debezium-connector-sqlserver" } # Database drivers postgresql-jdbc = "org.postgresql:postgresql:42.7.10" sqlite-jdbc = "org.xerial:sqlite-jdbc:3.47.1.0" clickhouse-jdbc = "com.clickhouse:clickhouse-jdbc:0.4.6" mariadb-jdbc = "org.mariadb.jdbc:mariadb-java-client:3.5.5" +mysql-connector-j = "com.mysql:mysql-connector-j:9.4.0" openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref = "openmldb" } openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version.ref = "openmldb" } # JClouds From f875916d4c90903f1593d03f50f7098b40786e0c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:16:48 +0300 Subject: [PATCH 28/33] Use pulsar-bom to align versions --- gradle/libs.versions.toml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ec94812fbd..32d1ebd758 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -518,20 +518,21 @@ bookkeeper-stats-api = { module = "org.apache.bookkeeper.stats:bookkeeper-stats- datasketches-memory = "org.apache.datasketches:datasketches-memory:2.2.0" datasketches-java = "org.apache.datasketches:datasketches-java:6.1.1" # Pulsar core modules (published Maven artifacts, used by connectors) -pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core", version.ref = "pulsar" } -pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common", version.ref = "pulsar" } -pulsar-common = { module = "org.apache.pulsar:pulsar-common", version.ref = "pulsar" } -pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api", version.ref = "pulsar" } -pulsar-client = { module = "org.apache.pulsar:pulsar-client-original", version.ref = "pulsar" } -pulsar-client-admin = { module = "org.apache.pulsar:pulsar-client-admin-original", version.ref = "pulsar" } -pulsar-config-validation = { module = "org.apache.pulsar:pulsar-config-validation", version.ref = "pulsar" } -pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api", version.ref = "pulsar" } -pulsar-functions-instance = { module = "org.apache.pulsar:pulsar-functions-instance", version.ref = "pulsar" } -pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils", version.ref = "pulsar" } -pulsar-broker = { module = "org.apache.pulsar:pulsar-broker", version.ref = "pulsar" } -pulsar-broker-test = { module = "org.apache.pulsar:pulsar-broker", version.ref = "pulsar" } -pulsar-testmocks = { module = "org.apache.pulsar:testmocks", version.ref = "pulsar" } -pulsar-buildtools = { module = "org.apache.pulsar:buildtools", version.ref = "pulsar" } +pulsar-bom = { module = "org.apache.pulsar:pulsar-bom", version.ref = "pulsar" } +pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core" } +pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common" } +pulsar-common = { module = "org.apache.pulsar:pulsar-common" } +pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api" } +pulsar-client = { module = "org.apache.pulsar:pulsar-client-original" } +pulsar-client-admin = { module = "org.apache.pulsar:pulsar-client-admin-original" } +pulsar-config-validation = { module = "org.apache.pulsar:pulsar-config-validation" } +pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api" } +pulsar-functions-instance = { module = "org.apache.pulsar:pulsar-functions-instance" } +pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils" } +pulsar-broker = { module = "org.apache.pulsar:pulsar-broker" } +pulsar-broker-test = { module = "org.apache.pulsar:pulsar-broker" } +pulsar-testmocks = { module = "org.apache.pulsar:testmocks" } +pulsar-buildtools = { module = "org.apache.pulsar:buildtools" } [plugins] lightproto = { id = "io.streamnative.lightproto", version.ref = "lightproto" } From 3e1db0a877dc122bf360194616f1cdb5d2b52ca7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:20:02 +0300 Subject: [PATCH 29/33] Assume alias ends with .bom (-bom in .toml file) --- pulsar-connectors-dependencies/build.gradle.kts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pulsar-connectors-dependencies/build.gradle.kts b/pulsar-connectors-dependencies/build.gradle.kts index 7252f010f3..0be83b0dad 100644 --- a/pulsar-connectors-dependencies/build.gradle.kts +++ b/pulsar-connectors-dependencies/build.gradle.kts @@ -34,13 +34,11 @@ dependencies { // Iterate over all library declarations in the version catalog and add them as constraints. // This ensures that any transitive dependency matching a catalog entry gets pinned to // the version we specify, regardless of what version a transitive dependency requests. - // BOM entries (detected by module name) are imported as platforms rather than constraints. + // BOM entries (detected by alias name) are imported as platforms rather than constraints. catalog.libraryAliases.forEach { alias -> catalog.findLibrary(alias).ifPresent { provider -> val dep = provider.get() - val module = dep.module - if (module.name.endsWith("-bom") || module.name.endsWith("_bom") || module.name == "bom" - || module.name.contains("-bom-") || module.name.contains("_bom_")) { + if (alias.endsWith(".bom")) { api(platform(provider)) } else if (dep.versionConstraint.requiredVersion.isNotEmpty()) { // Only add constraints for entries with explicit versions. From f89a5d97a65326ea7bb4e521a7ca17d66eece6bc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 13:46:25 +0300 Subject: [PATCH 30/33] Cleanup --- gradle/libs.versions.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 32d1ebd758..4e581ae16f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -530,7 +530,6 @@ pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api" } pulsar-functions-instance = { module = "org.apache.pulsar:pulsar-functions-instance" } pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils" } pulsar-broker = { module = "org.apache.pulsar:pulsar-broker" } -pulsar-broker-test = { module = "org.apache.pulsar:pulsar-broker" } pulsar-testmocks = { module = "org.apache.pulsar:testmocks" } pulsar-buildtools = { module = "org.apache.pulsar:buildtools" } From 61701b262b46ee30cb5df258033a34432aea818e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 14:02:23 +0300 Subject: [PATCH 31/33] Add solution for including pulsar modules in nar files --- ...lsar-connectors.nar-conventions.gradle.kts | 35 +++++++++++++++++-- kafka/build.gradle.kts | 16 ++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts index a5d803319a..29d79e3be2 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts @@ -25,11 +25,35 @@ plugins { id("io.github.merlimat.nar") } +/** + * Extension for customizing NAR packaging behavior. + * + * By default, Pulsar platform modules (pulsar-common, pulsar-client, etc.) are excluded + * from the NAR's bundled dependencies since they are provided at runtime by Pulsar's + * classloader hierarchy. Subprojects can include specific modules when needed: + * + * ```kotlin + * pulsarConnectorsNar { + * // Include pulsar-common in the NAR bundle (e.g., for SchemaInfoImpl) + * includePulsarModule("pulsar-common") + * } + * ``` + */ +open class PulsarConnectorsNarExtension { + internal val includedPulsarModules: MutableSet = mutableSetOf() + + fun includePulsarModule(module: String) { + includedPulsarModules.add(module) + } +} + +val pulsarConnectorsNar = extensions.create("pulsarConnectorsNar") + // NAR modules should not bundle Pulsar platform dependencies — they are provided // at runtime by Pulsar's classloader hierarchy. // Note: pulsar-io-common is NOT in java-instance.jar (runtime-all), so it must be // bundled in each NAR that uses it (e.g., IOConfigUtils). -val pulsarPlatformModules = setOf( +val defaultExcludedPulsarModules = setOf( "pulsar-client-api", "pulsar-client-admin-api", "pulsar-client-original", @@ -49,13 +73,18 @@ val pulsarPlatformModules = setOf( "pulsar-package-core", ) +// Use withDependencies to defer exclusion logic until after subproject build scripts +// have configured the extension. configurations.named("runtimeClasspath") { exclude(group = "org.apache.bookkeeper") // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it. // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts. exclude(group = "com.google.protobuf") - pulsarPlatformModules.forEach { module -> - exclude(group = "org.apache.pulsar", module = module) + withDependencies { + val excludedModules = defaultExcludedPulsarModules - pulsarConnectorsNar.includedPulsarModules + excludedModules.forEach { module -> + exclude(group = "org.apache.pulsar", module = module) + } } } diff --git a/kafka/build.gradle.kts b/kafka/build.gradle.kts index fb11565076..6d219d0460 100644 --- a/kafka/build.gradle.kts +++ b/kafka/build.gradle.kts @@ -22,13 +22,10 @@ plugins { id("pulsar-connectors.nar-conventions") } - // KafkaBytesSource uses SchemaInfoImpl from pulsar-common, which is excluded from -// NAR runtimeClasspath by the global exclusion. Bundle it via a separate configuration -// since the NAR classloader's parent (rootClassLoader) only has java-instance.jar. -val narExtraDeps by configurations.creating { - isCanBeResolved = true - isCanBeConsumed = false +// NAR runtimeClasspath by default. Include it in the NAR bundle. +pulsarConnectorsNar { + includePulsarModule("pulsar-common") } dependencies { @@ -36,7 +33,6 @@ dependencies { implementation(libs.pulsar.io.core) implementation(libs.pulsar.common) implementation(libs.pulsar.client) - narExtraDeps(libs.pulsar.common) implementation(libs.jackson.databind) implementation(libs.jackson.dataformat.yaml) implementation(libs.guava) @@ -53,9 +49,3 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.bcpkix.jdk18on) } - -tasks.named("nar") { - from(narExtraDeps) { into("META-INF/bundled-dependencies") } - bundledDependencies.from(narExtraDeps) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} From 623525807c92f8c1dd290eccee2d251fac1ca56b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 14:07:10 +0300 Subject: [PATCH 32/33] Use pulsar 4.2.0 pulsar-io libraries --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4e581ae16f..743d543fc1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ # [versions] # Core -pulsar = "4.1.3" +pulsar = "4.2.0" java = "17" # Code quality checkstyle = "13.3.0" From 4dab40971892264c552ddd87e1a0ac0a7a0c0221 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Apr 2026 14:19:43 +0300 Subject: [PATCH 33/33] Improve pulsarConnectorsDependencies extension --- alluxio/build.gradle.kts | 9 ++++---- ...sar-connectors.java-conventions.gradle.kts | 21 ++++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index 3daa42a0a0..9d2be5b15f 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -24,11 +24,12 @@ plugins { val alluxioVersion = "2.9.4" -// Alluxio requires older versions of netty, grpc, jetty, and metrics than the shared platform -// provides. Use non-enforced (non-strict) shared platform so the alluxio-specific enforced BOMs -// below can override individual version constraints. +// Alluxio requires older versions of netty, grpc, and jetty than the shared platform provides. +// Exclude these BOMs from the enforced platform so the alluxio-specific versions below can apply. pulsarConnectorsDependencies { - enforced = false + exclude(libs.jetty.bom) + exclude(libs.netty.bom) + exclude(libs.grpc.bom) } dependencies { diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts index 41a0bf91fc..a118e33d1d 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -62,9 +62,13 @@ if (project.name !in modulesUsingBcFips) { * * ```kotlin * pulsarConnectorsDependencies { - * // Exclude specific dependencies from the platform so they can be overridden locally. - * // Useful when a module needs an older version of a BOM or library (e.g. alluxio needs - * // older netty/grpc). + * // Exclude using a version catalog reference: + * exclude(libs.netty.bom) + * + * // Exclude using "group:module" notation: + * exclude("io.netty:netty-bom") + * + * // Exclude using named parameters: * exclude(group = "io.netty", module = "netty-bom") * * // Set enforced = false to use platform() instead of enforcedPlatform(). This makes all @@ -83,6 +87,17 @@ open class PulsarConnectorsDependenciesExtension { fun exclude(group: String, module: String) { excludes.add(DependencyExclusion(group, module)) } + + fun exclude(dependency: Provider) { + val dep = dependency.get() + excludes.add(DependencyExclusion(dep.module.group, dep.module.name)) + } + + fun exclude(notation: String) { + val parts = notation.split(":") + require(parts.size == 2) { "Expected 'group:module' format, got: $notation" } + excludes.add(DependencyExclusion(parts[0], parts[1])) + } } data class DependencyExclusion(val group: String, val module: String)