diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml new file mode 100644 index 0000000000..ceac384023 --- /dev/null +++ b/.github/actions/setup-gradle/action.yml @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Setup Gradle +description: Sets up Gradle with Develocity or public build scan publishing by default +inputs: + develocity-access-key: + description: 'Develocity access key for authenticated build scans' + required: false + default: '' + build-scan-publish: + description: 'Whether to publish build scans or use Develocity when the access key is set' + required: false + default: 'true' + cache-read-only: + description: 'Whether the Gradle cache is read-only' + required: false + default: ${{ github.event.repository != null && github.ref_name != github.event.repository.default_branch }} + add-job-summary: + description: 'When to add a job summary' + required: false + default: 'always' +runs: + using: composite + steps: + - name: Set Develocity Project ID and configure custom settings + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + shell: bash + run: | + mkdir -p ~/.gradle + touch ~/.gradle/gradle.properties + grep -q 'systemProp.develocity.projectId=' ~/.gradle/gradle.properties || echo systemProp.develocity.projectId=pulsar >> ~/.gradle/gradle.properties + grep -q 'systemProp.scan.uploadInBackground=' ~/.gradle/gradle.properties || echo systemProp.scan.uploadInBackground=false >> ~/.gradle/gradle.properties + + - name: Setup Gradle with Develocity + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f + with: + develocity-injection-enabled: true + develocity-url: https://develocity.apache.org + # expected format is develocity.apache.org: + develocity-access-key: ${{ inputs.develocity-access-key }} + build-scan-publish: ${{ inputs.build-scan-publish }} + cache-read-only: ${{ inputs.cache-read-only }} + add-job-summary: ${{ inputs.add-job-summary }} + + - name: Setup Gradle + if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true') }} + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f + with: + build-scan-publish: ${{ inputs.build-scan-publish }} + build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service' + build-scan-terms-of-use-agree: 'yes' + cache-read-only: ${{ inputs.cache-read-only }} + add-job-summary: ${{ inputs.add-job-summary }} \ No newline at end of file diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml new file mode 100644 index 0000000000..00eb084990 --- /dev/null +++ b/.github/actions/tune-runner-vm/action.yml @@ -0,0 +1,80 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Tune Runner VM performance +description: tunes the GitHub Runner VM operation system +runs: + using: composite + steps: + - shell: bash + run: | + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "::group::Configure and tune OS" + # Ensure that reverse lookups for current hostname are handled properly + # Add the current IP address, long hostname and short hostname record to /etc/hosts file + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + + # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory + # consumption is high. + # Set vm.swappiness=1 to avoid swapping and allow high RAM usage + echo 1 | sudo tee /proc/sys/vm/swappiness + + # use "madvise" Linux Transparent HugePages (THP) setting + # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html + # "madvise" is generally a better option than the default "always" setting + # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b + echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled + echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled + echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag + echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag + + # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt + # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) + # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) + sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true + if mountpoint -q /mnt; then + sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true + fi + # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective + # https://www.spinics.net/lists/linux-ide/msg52562.html + for i in /sys/block/sd*/queue/discard_max_bytes; do + echo 0 | sudo tee $i + done + # disable unnecessary timers + sudo systemctl stop fstrim.timer fstrim.service \ + podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \ + phpsessionclean.timer man-db.timer motd-news.timer \ + dpkg-db-backup.timer e2scrub_all.timer \ + update-notifier-download.timer update-notifier-motd.timer || true + + # stop unnecessary services + sudo systemctl stop php8.3-fpm.service ModemManager.service \ + multipathd.socket multipathd.service udisks2.service walinuxagent.service || true + + echo '::endgroup::' + + # show memory + echo "::group::Available Memory" + free -m + echo '::endgroup::' + # show disk + echo "::group::Available diskspace" + df -BM + echo "::endgroup::" + fi \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 838b66779a..424363a0c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,17 +40,26 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - uses: actions/setup-java@v5 with: distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ env.JDK_VERSION }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c - - name: Build all modules - run: ./gradlew build -x test - - name: License check (RAT) - run: ./gradlew rat + - name: Setup Gradle + uses: ./.github/actions/setup-gradle + with: + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + cache-read-only: false + + - name: Build and check licenses + run: >- + ./gradlew -x test -x nar build rat spotlessCheck + --no-configuration-cache tests: name: Tests - ${{ matrix.name }} @@ -64,18 +73,26 @@ jobs: - name: Connectors tasks: test steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v6 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - uses: actions/setup-java@v5 with: distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ env.JDK_VERSION }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c + + - name: Setup Gradle + uses: ./.github/actions/setup-gradle + with: + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Run tests - run: ./gradlew ${{ matrix.tasks }} + run: ./gradlew ${{ matrix.tasks }} -PtestFailFast=true - name: Upload test reports if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.name }}-test-reports path: '**/build/reports/tests/' diff --git a/.gitignore b/.gitignore index 91bd581484..33093cfbed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Gradle .gradle/ +.kotlin/ build/ **/build/ diff --git a/.ratignore b/.ratignore new file mode 100644 index 0000000000..ab06f4004d --- /dev/null +++ b/.ratignore @@ -0,0 +1,48 @@ +# Build artifacts +**/build/** +**/target/** +# Gradle files +.gradle/** +gradle/wrapper/** +**/.gradle/** +**/.kotlin/** +**/gradle/wrapper/** +gradlew +gradlew.bat +gradle/libs.versions.toml +# Generated Flatbuffer files (Kinesis) +**/org/apache/pulsar/io/kinesis/fbs/*.java +# Services files +**/META-INF/services/* +# Certificates and keys +**/*.crt +**/*.key +**/*.csr +**/*.pem +**/*.srl +**/certificate-authority/serial +**/certificate-authority/index.txt +**/*.json +**/*.txt +# Project/IDE files +**/*.md +.github/** +**/*.nar +**/.gitignore +**/.gitattributes +**/*.iml +**/.classpath +**/.project +**/.settings +**/.idea/** +**/.vscode/** +# Avro schemas +**/*.avsc +# Patch files +**/*.patch +# Hidden directories +.*/** +# Test output +**/test-output/** +# Log files +**/*.log diff --git a/README.md b/README.md index 6ecb2716f7..ea0694f752 100644 --- a/README.md +++ b/README.md @@ -66,26 +66,48 @@ mounting them into the `apachepulsar/pulsar` Docker image. |-----------|-------------| | Kafka Connect Adaptor | Run Kafka Connect connectors on Pulsar | +## Prerequisites + +- **JDK 17** or later — e.g. [Eclipse Temurin](https://adoptium.net/en-GB/temurin/releases?version=17&os=any&arch=any) + or [Amazon Corretto](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/what-is-corretto-17.html) + +> **Note**: This project includes a [Gradle Wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html) +> so no separate Gradle installation is needed. Use `./gradlew` on Linux/macOS and `gradlew.bat` on Windows. + ## Building +Compile and assemble all modules: + ```bash -./gradlew build -x test +./gradlew assemble ``` -To build all connector NARs: +NAR files are produced under each connector's `build/libs/` directory. + +Build a specific connector: ```bash -./gradlew build -x test +./gradlew :elastic-search:assemble ``` -NAR files are produced under each connector's `build/libs/` directory. - -To build the distribution tarball containing all connector NARs: +Build the distribution package containing all connector NARs: ```bash ./gradlew :distribution:pulsar-io-distribution:assemble ``` +Check source code license headers: + +```bash +./gradlew rat spotlessCheck +``` + +Auto-fix license headers: + +```bash +./gradlew spotlessApply +``` + ## Running Tests ```bash @@ -94,6 +116,9 @@ To build the distribution tarball containing all connector NARs: # Specific connector ./gradlew :elastic-search:test + +# Specific test class +./gradlew :elastic-search:test --tests "ElasticSearchSinkTests" ``` ## Using Connectors diff --git a/aerospike/build.gradle.kts b/aerospike/build.gradle.kts index eae094fd21..9fb452f446 100644 --- a/aerospike/build.gradle.kts +++ b/aerospike/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts index be406b5ecb..9d2be5b15f 100644 --- a/alluxio/build.gradle.kts +++ b/alluxio/build.gradle.kts @@ -18,14 +18,33 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } + +val alluxioVersion = "2.9.4" + +// Alluxio requires older versions of netty, grpc, and jetty than the shared platform provides. +// Exclude these BOMs from the enforced platform so the alluxio-specific versions below can apply. +pulsarConnectorsDependencies { + exclude(libs.jetty.bom) + exclude(libs.netty.bom) + exclude(libs.grpc.bom) +} + dependencies { + // Alluxio-compatible BOMs — these override the shared platform versions. + implementation(enforcedPlatform(libs.jetty9.bom)) + implementation(enforcedPlatform("io.netty:netty-bom:4.1.100.Final")) + implementation(enforcedPlatform("io.grpc:grpc-bom:1.37.0")) + implementation(libs.pulsar.io.core) - implementation("org.alluxio:alluxio-core-client-fs:2.9.3") + implementation("org.alluxio:alluxio-core-client-fs:$alluxioVersion") implementation(libs.jackson.dataformat.yaml) implementation(libs.guava) testImplementation(libs.pulsar.client) - testImplementation("org.alluxio:alluxio-minicluster:2.9.3") -} + testImplementation("org.alluxio:alluxio-minicluster:$alluxioVersion") { + exclude(group = "org.glassfish", module = "javax.el") + } +} \ No newline at end of file diff --git a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java index 3b72dc9666..bde3cadcfd 100644 --- a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java +++ b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java @@ -151,11 +151,7 @@ public void write(Record record) { } catch (AlluxioException | IOException e) { log.error("Unable to flush records to alluxio.", e); failRecords(); - try { - deleteTmpFile(); - } catch (AlluxioException | IOException e1) { - log.error("Failed to delete tmp cache file.", e); - } + deleteTmpFile(); break; } case FILE_COMMITTED: @@ -224,10 +220,11 @@ private void createTmpFile() throws AlluxioException, IOException { } private void closeAndCommitTmpFile() throws AlluxioException, IOException { - // close the tmpFile - if (fileOutStream != null) { - fileOutStream.close(); + if (fileOutStream == null) { + return; } + // close the tmpFile + fileOutStream.close(); // commit the tmpFile String filePrefix = alluxioSinkConfig.getFilePrefix(); String fileExtension = alluxioSinkConfig.getFileExtension(); @@ -240,9 +237,14 @@ private void closeAndCommitTmpFile() throws AlluxioException, IOException { lastRotationTime = System.currentTimeMillis(); } - private void deleteTmpFile() throws AlluxioException, IOException { - if (!tmpFilePath.equals("")) { + private void deleteTmpFile() { + if (tmpFilePath == null || tmpFilePath.isEmpty()) { + return; + } + try { fileSystem.delete(new AlluxioURI(tmpFilePath)); + } catch (Exception e) { + log.warn("Failed to delete tmp file {}", tmpFilePath, e); } } diff --git a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java index 2f777b620d..b1b1d0f15d 100644 --- a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java +++ b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java @@ -27,6 +27,8 @@ import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; import alluxio.master.LocalAlluxioCluster; +import alluxio.security.authentication.AuthType; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,6 +49,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -64,6 +67,7 @@ public class AlluxioSinkTest { protected Map map; protected AlluxioSink sink; protected LocalAlluxioCluster cluster; + protected String alluxioDir; @Mock protected Record mockRecord; @@ -74,7 +78,7 @@ public class AlluxioSinkTest { static GenericRecord fooBar; @BeforeClass - public static void init() { + public void init() throws Exception { valueSchema = Schema.JSON(Foobar.class); genericSchema = Schema.generic(valueSchema.getSchemaInfo()); fooBar = genericSchema.newRecordBuilder() @@ -83,25 +87,30 @@ public static void init() { .set("age", 20) .build(); kvSchema = Schema.KeyValue(Schema.STRING, genericSchema, KeyValueEncodingType.SEPARATED); - } - @BeforeMethod - public final void setUp() throws Exception { try { cluster = setupSingleMasterCluster(); } catch (java.util.concurrent.TimeoutException e) { throw new org.testng.SkipException( "Skipping test: Alluxio local cluster failed to start within timeout", e); } + } + + @AfterClass + public void destroyCluster() throws Exception { + if (cluster != null) { + cluster.stop(); + } + } + + @BeforeMethod + public final void setUp(Method method) throws Exception { + alluxioDir = "/" + method.getName(); map = new HashMap<>(); - // alluxioMasterHost should be set via LocalAlluxioCluster#getHostname - // instead of using a fixed value "localhost", since it seems that - // LocalAlluxioCluster may bind other address than localhost - // when the node has multiple network interfaces. map.put("alluxioMasterHost", cluster.getHostname()); map.put("alluxioMasterPort", cluster.getMasterRpcPort()); - map.put("alluxioDir", "/pulsar"); + map.put("alluxioDir", alluxioDir); map.put("filePrefix", "prefix"); map.put("schemaEnable", "true"); @@ -131,8 +140,9 @@ public Object getNativeObject() { @AfterMethod public void tearDown() throws Exception { - if (cluster != null) { - cluster.stop(); + if (sink != null) { + sink.close(); + sink = null; } } @@ -143,8 +153,6 @@ public void openTest() throws Exception { map.put("lineSeparator", "\n"); map.put("rotationRecords", "100"); - String alluxioDir = "/pulsar"; - sink = new AlluxioSink(); sink.open(map, mockSinkContext); @@ -156,8 +164,6 @@ public void openTest() throws Exception { String alluxioTmpDir = FilenameUtils.concat(alluxioDir, "tmp"); AlluxioURI alluxioTmpURI = new AlluxioURI(alluxioTmpDir); Assert.assertTrue(client.exists(alluxioTmpURI)); - - sink.close(); } @Test @@ -167,9 +173,6 @@ public void writeAndCloseTest() throws Exception { map.put("lineSeparator", "\n"); map.put("rotationRecords", "1"); map.put("writeType", "THROUGH"); - map.put("alluxioDir", "/pulsar"); - - String alluxioDir = "/pulsar"; sink = new AlluxioSink(); sink.open(map, mockSinkContext); @@ -201,11 +204,9 @@ public Object getNativeObject() { for (String path : pathList) { if (path.contains("tmp")) { - // Ensure that the temporary file is rotated and the directory is empty - Assert.assertEquals(path, "/pulsar/tmp"); + Assert.assertEquals(path, alluxioDir + "/tmp"); } else { - // Ensure that all rotated files conform the naming convention - Assert.assertTrue(path.startsWith("/pulsar/TopicA-")); + Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-")); } } @@ -228,21 +229,34 @@ public Object getNativeObject() { for (String path : pathList) { if (path.contains("tmp")) { - Assert.assertEquals(path, "/pulsar/tmp"); + Assert.assertEquals(path, alluxioDir + "/tmp"); } else { - Assert.assertTrue(path.startsWith("/pulsar/TopicA-")); + Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-")); } } - - sink.close(); } private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { // Setup and start the local alluxio cluster + log.info("Configuring local Alluxio cluster"); LocalAlluxioCluster cluster = new LocalAlluxioCluster(); cluster.initConfiguration(getTestName(getClass().getSimpleName(), "test")); Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); + // Disable auth handshake overhead + Configuration.set(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.NOSASL); + Configuration.set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, false); + // Disable unnecessary services + Configuration.set(PropertyKey.USER_METRICS_COLLECTION_ENABLED, false); + Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false); + Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false); + Configuration.set(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED, false); + // Minimal safe mode wait — in a local minicluster the worker registers almost instantly. + // A longer wait (e.g. 5s) causes "master is in safe mode" failures when tests write right + // after startup. + Configuration.set(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, "500ms"); + log.info("Starting local Alluxio cluster"); cluster.start(); + log.info("Alluxio cluster started"); return cluster; } diff --git a/aws/build.gradle.kts b/aws/build.gradle.kts index b8456075be..81fc5e0711 100644 --- a/aws/build.gradle.kts +++ b/aws/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { implementation(libs.pulsar.io.core) implementation(libs.gson) diff --git a/azure-data-explorer/build.gradle.kts b/azure-data-explorer/build.gradle.kts index 7fcedd574e..0f0662b088 100644 --- a/azure-data-explorer/build.gradle.kts +++ b/azure-data-explorer/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } nar { narId.set("pulsar-io-azuredataexplorer") diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts new file mode 100644 index 0000000000..bc3d07a2be --- /dev/null +++ b/build-logic/conventions/build.gradle.kts @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `kotlin-dsl` +} + +dependencies { + implementation(libs.plugins.shadow.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) + implementation(libs.plugins.spotless.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) + implementation(libs.plugins.nar.get().let { + "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" + }) +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts new file mode 100644 index 0000000000..ad2ffaee23 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("com.diffplug.spotless") +} + +// ── License header check (Spotless) ──────────────────────────────────────── +val asfLicenseHeader = rootProject.file("src/license-header.txt").readText() +val asfLicenseHeaderJava = "/*\n" + asfLicenseHeader.lines() + .map { " * $it".trimEnd() } + .joinToString("\n") + "/\n" + +configure { + java { + targetExclude( + "**/generated/**", + "**/generated-sources/**", + // Generated FlatBuffers files (Kinesis) + "**/org/apache/pulsar/io/kinesis/fbs/*.java", + "build/**", + ) + licenseHeader(asfLicenseHeaderJava, "(\\n|package|import|public|class|module) ?") + } +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts new file mode 100644 index 0000000000..a118e33d1d --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `java-library` + id("pulsar-connectors.code-quality-conventions") +} + +val catalog = the().named("libs") + +fun lib(alias: String): Provider = + catalog.findLibrary(alias).orElseThrow { + GradleException("Library alias '$alias' not found in version catalog 'libs'") + } + +// Add shared test resources (log4j2-test.xml) to the test classpath for all modules. +the()["test"].resources.srcDir(rootProject.file("gradle/test-resources")) + +tasks.withType().configureEach { + options.encoding = "UTF-8" + options.release.set(17) + options.compilerArgs.addAll(listOf("-parameters", "-Xlint:deprecation", "-Xlint:unchecked")) +} + +configurations.all { + // Exclude the old SLF4J 1.x bridge pulled in by transitive dependencies. + // Pulsar uses SLF4J 2.x with log4j-slf4j2-impl; having both causes + // NoSuchMethodError in Log4jLoggerFactory at test startup. + exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl") +} + +// Exclude bc-fips from modules that don't need it. bc-fips's CryptoServicesRegistrar +// conflicts with bcprov-jdk18on's version — having both causes NoSuchMethodError. +val modulesUsingBcFips = setOf("kafka-connect-adaptor") +if (project.name !in modulesUsingBcFips) { + configurations.all { + exclude(group = "org.bouncycastle", module = "bc-fips") + } +} + +/** + * Configures how the shared `pulsar-connectors-dependencies` platform is applied. + * + * By default, the platform is applied as `enforcedPlatform`, which pins (strictly enforces) all + * dependency versions from the version catalog. Subprojects can customize this behavior: + * + * ```kotlin + * pulsarConnectorsDependencies { + * // Exclude using a version catalog reference: + * exclude(libs.netty.bom) + * + * // Exclude using "group:module" notation: + * exclude("io.netty:netty-bom") + * + * // Exclude using named parameters: + * exclude(group = "io.netty", module = "netty-bom") + * + * // Set enforced = false to use platform() instead of enforcedPlatform(). This makes all + * // version constraints non-strict: the platform versions are used only as defaults (allowing + * // version omission when declaring dependencies), but can be overridden by the module's own + * // enforcedPlatform or strictly-versioned dependencies. + * enforced = false + * } + * ``` + */ +open class PulsarConnectorsDependenciesExtension { + var enforced: Boolean = true + + internal val excludes: MutableList = mutableListOf() + + fun exclude(group: String, module: String) { + excludes.add(DependencyExclusion(group, module)) + } + + fun exclude(dependency: Provider) { + val dep = dependency.get() + excludes.add(DependencyExclusion(dep.module.group, dep.module.name)) + } + + fun exclude(notation: String) { + val parts = notation.split(":") + require(parts.size == 2) { "Expected 'group:module' format, got: $notation" } + excludes.add(DependencyExclusion(parts[0], parts[1])) + } +} + +data class DependencyExclusion(val group: String, val module: String) + +val pulsarConnectorsDependencies = extensions.create("pulsarConnectorsDependencies") + +// withDependencies runs lazily after subproject build scripts have configured the extension. +// This is configuration-cache compatible (unlike afterEvaluate). +configurations["implementation"].withDependencies { + val platformProject = project(":pulsar-connectors-dependencies") + val configureAction = Action { + (this as ModuleDependency).apply { + pulsarConnectorsDependencies.excludes.forEach { exc -> + exclude(group = exc.group, module = exc.module) + } + } + } + val dep = if (pulsarConnectorsDependencies.enforced) { + dependencies.enforcedPlatform(platformProject, configureAction) + } else { + dependencies.platform(platformProject, configureAction) + } + add(dep) +} + +dependencies { + + // Resolve lz4-java capability conflict: at.yawk.lz4:lz4-java (used by Pulsar) and + // org.lz4:lz4-java (used by kafka-clients) both provide the org.lz4:lz4-java capability. + // Prefer at.yawk.lz4 which is the version Pulsar standardizes on. + configurations.all { + resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") { + select("at.yawk.lz4:lz4-java:0") + } + } + + // Annotation processing for Lombok + "compileOnly"(lib("lombok")) + "annotationProcessor"(lib("lombok")) + "testCompileOnly"(lib("lombok")) + "testAnnotationProcessor"(lib("lombok")) + + // Common test dependencies + "testImplementation"(lib("testng")) + "testImplementation"(lib("mockito-core")) + "testImplementation"(lib("assertj-core")) + "testImplementation"(lib("awaitility")) + "testImplementation"(lib("system-lambda")) + "testImplementation"(lib("slf4j-api")) + + // Logging runtime for tests — provides Log4j2 as the SLF4J backend. + // Some connectors (Alluxio minicluster, Solr embedded) require a logging + // implementation to be present at test runtime. + "testRuntimeOnly"(lib("log4j-api")) + "testRuntimeOnly"(lib("log4j-core")) + "testRuntimeOnly"(lib("log4j-slf4j2-impl")) + "testRuntimeOnly"(lib("jcl-over-slf4j")) +} + +tasks.withType().configureEach { + useTestNG { + // TestNG group filtering: -PtestGroups=group1,group2 -PexcludedTestGroups=flaky + providers.gradleProperty("testGroups").orNull?.let { groups -> + includeGroups(*groups.split(",").map { it.trim() }.toTypedArray()) + } + val excludedTestGroups = providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky") + excludeGroups(*(excludedTestGroups.split(",").map { it.trim() }.toTypedArray())) + } + testLogging { + events("passed", "skipped", "failed") + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + showStackTraces = true + showExceptions = true + showCauses = true + showStandardStreams = providers.gradleProperty("testShowOutput") + .map { it.isBlank() || it.toBoolean() }.getOrElse(false) + } + maxHeapSize = "1300m" + maxParallelForks = 4 + val failFastValue = providers.gradleProperty("testFailFast").getOrElse("true").toBoolean() + failFast = failFastValue + systemProperty("testRetryCount", providers.gradleProperty("testRetryCount").getOrElse("1")) + systemProperty("testFailFast", failFastValue.toString()) + systemProperty("java.net.preferIPv4Stack", "true") + jvmArgs( + "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/sun.net=ALL-UNNAMED", + "--add-opens", "java.management/sun.management=ALL-UNNAMED", + "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", + "-XX:+EnableDynamicAgentLoading", + "-Xshare:off", + "-Dio.netty.tryReflectionSetAccessible=true", + "-Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true", + "-Dpulsar.allocator.pooled=true", + "-Dpulsar.allocator.exit_on_oom=false", + "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", + "-Dpulsar.test.preventExit=true", + ) +} + +// Set archive names to match Maven artifactId for nested modules. +// Skip if the project name is already qualified (starts with parent name), +// which happens for sub-modules that use qualified names in settings.gradle.kts +// to avoid Gradle name clashes. +val parentProject = project.parent +if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject + && !project.name.startsWith(parentProject.name)) { + the().archivesName.set("${parentProject.name}-${project.name}") +} + +tasks.withType().configureEach { + manifest { + attributes( + "Implementation-Title" to project.name, + "Implementation-Version" to project.version, + ) + } +} + +// Add a task for viewing all configurations for all projects in a simple way +tasks.register("allDependencies"){} diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts new file mode 100644 index 0000000000..29d79e3be2 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Convention plugin for NAR (Nifi Archive) modules. +// Configures platform module exclusions from runtimeClasspath, forces JAR artifacts +// for bundled-dependencies, and handles archive name qualification. + +plugins { + id("io.github.merlimat.nar") +} + +/** + * Extension for customizing NAR packaging behavior. + * + * By default, Pulsar platform modules (pulsar-common, pulsar-client, etc.) are excluded + * from the NAR's bundled dependencies since they are provided at runtime by Pulsar's + * classloader hierarchy. Subprojects can include specific modules when needed: + * + * ```kotlin + * pulsarConnectorsNar { + * // Include pulsar-common in the NAR bundle (e.g., for SchemaInfoImpl) + * includePulsarModule("pulsar-common") + * } + * ``` + */ +open class PulsarConnectorsNarExtension { + internal val includedPulsarModules: MutableSet = mutableSetOf() + + fun includePulsarModule(module: String) { + includedPulsarModules.add(module) + } +} + +val pulsarConnectorsNar = extensions.create("pulsarConnectorsNar") + +// NAR modules should not bundle Pulsar platform dependencies — they are provided +// at runtime by Pulsar's classloader hierarchy. +// Note: pulsar-io-common is NOT in java-instance.jar (runtime-all), so it must be +// bundled in each NAR that uses it (e.g., IOConfigUtils). +val defaultExcludedPulsarModules = setOf( + "pulsar-client-api", + "pulsar-client-admin-api", + "pulsar-client-original", + "pulsar-client", + "pulsar-common", + "pulsar-config-validation", + "bouncy-castle-bc", + "pulsar-functions-api", + "pulsar-functions-instance", + "pulsar-functions-proto", + "pulsar-functions-secrets", + "pulsar-functions-utils", + "pulsar-io-core", + "pulsar-metadata", + "pulsar-opentelemetry", + "managed-ledger", + "pulsar-package-core", +) + +// Use withDependencies to defer exclusion logic until after subproject build scripts +// have configured the extension. +configurations.named("runtimeClasspath") { + exclude(group = "org.apache.bookkeeper") + // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it. + // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts. + exclude(group = "com.google.protobuf") + withDependencies { + val excludedModules = defaultExcludedPulsarModules - pulsarConnectorsNar.includedPulsarModules + excludedModules.forEach { module -> + exclude(group = "org.apache.pulsar", module = module) + } + } +} + +// The NAR plugin copies from runtimeClasspath which resolves project dependencies +// as class directories, not JARs. The NarClassLoader expects JARs in +// META-INF/bundled-dependencies/. Force the NAR task to use JAR artifacts. +// Use lazy resolution to avoid eagerly resolving the configuration at configuration +// time, which would cause configuration cache invalidation when JARs are created. +tasks.named("nar", Jar::class.java) { + val runtimeClasspath = configurations.named("runtimeClasspath") + into("META-INF/bundled-dependencies") { + from(runtimeClasspath.map { conf -> + conf.incoming.artifactView { + attributes { + attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + objects.named(LibraryElements::class.java, LibraryElements.JAR) + ) + } + }.files + }) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } +} + +// Set NAR-specific archive name qualification for nested modules. +val parentProject = project.parent +if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject + && !project.name.startsWith(parentProject.name)) { + val qualifiedName = "${parentProject.name}-${project.name}" + val narExt = extensions.getByName("nar") + @Suppress("UNCHECKED_CAST") + val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as org.gradle.api.provider.Property + narIdProp.set(qualifiedName) +} + +tasks.named("jar") { + enabled = false +} + +configurations { + named("runtimeElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("nar")) + variants.clear() + } + } + named("apiElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("nar")) + variants.clear() + } + } +} \ No newline at end of file diff --git a/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts new file mode 100644 index 0000000000..7a760e58e8 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Convention plugin for modules using the Shadow plugin. +// Applies the shadow plugin, disables the default jar task, and makes the +// shadow jar the primary artifact for both runtimeElements and apiElements, +// so plain project() dependencies resolve to the shadow jar. + +plugins { + id("com.gradleup.shadow") +} + +shadow { + addShadowVariantIntoJavaComponent.set(false) +} + +tasks.named("shadowJar") { + archiveClassifier.set("") + mergeServiceFiles() +} + +tasks.named("jar") { + enabled = false +} + +configurations { + named("runtimeElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("shadowJar")) + variants.clear() + } + } + named("apiElements") { + outgoing { + artifacts.clear() + artifact(tasks.named("shadowJar")) + variants.clear() + } + } +} diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts new file mode 100644 index 0000000000..ec81a012a5 --- /dev/null +++ b/build-logic/settings.gradle.kts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +dependencyResolutionManagement { + repositories { + gradlePluginPortal() + mavenCentral() + } + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} +rootProject.name = "build-logic" +include("conventions") diff --git a/build.gradle.kts b/build.gradle.kts index 9aefca6122..9f0827841c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,269 +17,66 @@ * under the License. */ +import com.github.vlsi.gradle.git.dsl.gitignore +import org.jetbrains.gradle.ext.copyright +import org.jetbrains.gradle.ext.settings + plugins { alias(libs.plugins.rat) + alias(libs.plugins.version.catalog.update) + alias(libs.plugins.versions) + alias(libs.plugins.crlf) apply false + alias(libs.plugins.idea.ext) + alias(libs.plugins.spotless) apply false // workaround for https://github.com/diffplug/spotless/issues/2877 } -tasks.named("rat").configure { - val excludesProp = this.javaClass.getMethod("getExcludes").invoke(this) - @Suppress("UNCHECKED_CAST") - val excludes = excludesProp as MutableCollection - excludes.addAll(listOf( - // Build artifacts - "**/build/**", - "**/target/**", - // Gradle files - ".gradle/**", - "gradle/wrapper/**", - "**/.gradle/**", - "**/gradle/wrapper/**", - // Generated Flatbuffer files (Kinesis) - "**/org/apache/pulsar/io/kinesis/fbs/*.java", - // Services files - "**/META-INF/services/*", - // Certificates and keys - "**/*.crt", - "**/*.key", - "**/*.csr", - "**/*.pem", - "**/*.srl", - "**/certificate-authority/serial", - "**/certificate-authority/index.txt", - "**/*.json", - "**/*.txt", - // Project/IDE files - "**/*.md", - ".github/**", - "**/*.nar", - "**/.gitignore", - "**/.gitattributes", - "**/*.iml", - "**/.classpath", - "**/.project", - "**/.settings", - "**/.idea/**", - "**/.vscode/**", - // Avro schemas - "**/*.avsc", - // Patch files - "**/*.patch", - // Hidden directories - ".*/**", - // Test output - "**/test-output/**", - // Log files - "**/*.log", - )) -} - -val catalog = the().named("libs") -val pulsarConnectorsVersion = catalog.findVersion("pulsar-connectors").get().requiredVersion - -allprojects { - group = "org.apache.pulsar" - version = pulsarConnectorsVersion -} - -subprojects { - // Platform modules use java-platform which is mutually exclusive with java-library. - if (project.name == "pulsar-dependencies") { - return@subprojects - } - - // Parent modules and non-Java modules that have no source code of their own - val skipModules = setOf("jdbc", "debezium", "distribution", "docker") - if (project.name in skipModules && project.childProjects.isNotEmpty()) { - return@subprojects - } - - apply(plugin = "java-library") - - // Add shared test resources (log4j2-test.xml) to the test classpath for all modules. - the()["test"].resources.srcDir(rootProject.file("gradle/test-resources")) - - tasks.withType { - options.encoding = "UTF-8" - options.release.set(17) - options.compilerArgs.addAll(listOf("-parameters")) - } - - configurations.all { - // Exclude the old SLF4J 1.x bridge - exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl") - - // Force Jackson version to match the version catalog - resolutionStrategy.eachDependency { - if (requested.group.startsWith("com.fasterxml.jackson")) { - useVersion(rootProject.libs.versions.jackson.get()) - } - } - } - - // Exclude bc-fips from modules that don't need it. - val modulesUsingBcFips = setOf("kafka-connect-adaptor") - if (project.name !in modulesUsingBcFips) { - configurations.all { - exclude(group = "org.bouncycastle", module = "bc-fips") - } - } - - dependencies { - // Enforced platform pins all dependency versions from the version catalog. - "implementation"(enforcedPlatform(project(":pulsar-dependencies"))) - - // Resolve lz4-java capability conflict - configurations.all { - resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") { - select("at.yawk.lz4:lz4-java:0") - } - } - - // Annotation processing for Lombok - "compileOnly"(rootProject.libs.lombok) - "annotationProcessor"(rootProject.libs.lombok) - "testCompileOnly"(rootProject.libs.lombok) - "testAnnotationProcessor"(rootProject.libs.lombok) - - // Common test dependencies - "testImplementation"(rootProject.libs.testng) - "testImplementation"(rootProject.libs.mockito.core) - "testImplementation"(rootProject.libs.assertj.core) - "testImplementation"(rootProject.libs.awaitility) - "testImplementation"(rootProject.libs.system.lambda) - "testImplementation"(rootProject.libs.slf4j.api) - - // Logging runtime for tests — provides Log4j2 as the SLF4J backend. - // Some connectors (Alluxio minicluster, Solr embedded) require a logging - // implementation to be present at test runtime. - "testRuntimeOnly"(rootProject.libs.log4j.api) - "testRuntimeOnly"(rootProject.libs.log4j.core) - "testRuntimeOnly"(rootProject.libs.log4j.slf4j2.impl) - "testRuntimeOnly"(rootProject.libs.jcl.over.slf4j) - } - - tasks.withType { - useTestNG { - // TestNG group filtering - providers.gradleProperty("testGroups").orNull?.let { groups -> - includeGroups(*groups.split(",").map { it.trim() }.toTypedArray()) - } - val excludedTestGroups = providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky") - excludeGroups(*(excludedTestGroups.split(",").map { it.trim() }.toTypedArray())) - } - maxHeapSize = "1300m" - maxParallelForks = 4 - systemProperty("testRetryCount", System.getProperty("testRetryCount", "1")) - systemProperty("testFailFast", System.getProperty("testFailFast", "true")) - jvmArgs( - "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", - "--add-opens", "java.base/java.lang=ALL-UNNAMED", - "--add-opens", "java.base/java.io=ALL-UNNAMED", - "--add-opens", "java.base/java.util=ALL-UNNAMED", - "--add-opens", "java.base/sun.net=ALL-UNNAMED", - "--add-opens", "java.management/sun.management=ALL-UNNAMED", - "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED", - "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED", - "--add-opens", "java.base/java.nio=ALL-UNNAMED", - "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", - "-XX:+EnableDynamicAgentLoading", - "-Xshare:off", - "-Dio.netty.tryReflectionSetAccessible=true", - "-Dpulsar.allocator.pooled=true", - "-Dpulsar.allocator.exit_on_oom=false", - "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", - "-Dpulsar.test.preventExit=true", - ) +versionCatalogUpdate { + sortByKey = false + keep { + keepUnusedVersions.set(true) } +} - // Shadow JAR modules: expose the shadow JAR as a consumable configuration so other - // projects can depend on it via project(path = "...", configuration = "shadowElements") - pluginManager.withPlugin("com.gradleup.shadow") { - val shadowElements by configurations.creating { - isCanBeConsumed = true - isCanBeResolved = false - attributes { - attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.JAVA_RUNTIME)) - attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling.SHADOWED)) - } - } - artifacts.add("shadowElements", tasks.named("shadowJar")) +tasks.named("dependencyUpdates") { + outputFormatter = "html" + rejectVersionIf { + val nonStable = candidate.version.contains("alpha") || candidate.version.contains("beta") || candidate.version.contains("rc") + // OpenTelemetry publishes stable releases with -alpha suffix for some modules + val isOpenTelemetry = candidate.group.startsWith("io.opentelemetry") + nonStable && !(isOpenTelemetry && candidate.version.contains("alpha")) } +} - // NAR modules should not bundle Pulsar platform dependencies — they are provided - // at runtime by Pulsar's classloader hierarchy. - pluginManager.withPlugin("io.github.merlimat.nar") { - val pulsarPlatformModules = setOf( - "pulsar-client-api", - "pulsar-client-admin-api", - "pulsar-client-original", - "pulsar-client", - "pulsar-common", - "pulsar-config-validation", - "bouncy-castle-bc", - "pulsar-functions-api", - "pulsar-functions-instance", - "pulsar-functions-proto", - "pulsar-functions-secrets", - "pulsar-functions-utils", - "pulsar-io-core", - "pulsar-metadata", - "pulsar-opentelemetry", - "managed-ledger", - "pulsar-package-core", - ) - configurations.named("runtimeClasspath") { - exclude(group = "org.apache.bookkeeper") - exclude(group = "com.google.protobuf") - pulsarPlatformModules.forEach { module -> - exclude(group = "org.apache.pulsar", module = module) - } - } +// ── Apache RAT (Release Audit Tool) ───────────────────────────────────────── +tasks.named("rat").configure { + // Honour .gitignore exclusions so RAT skips untracked/generated files. + // Register .gitignore files as inputs so the task re-runs when they change. + inputs.files(fileTree(rootDir) { + include("**/.gitignore") + exclude("**/build/**") + exclude("**/.gradle/**") + }) + // use crlf plugin's gitignore dsl + gitignore(rootDir) + // Apply additional RAT-specific exclusions from .ratignore. + val ratignoreFile = rootDir.resolve(".ratignore") + inputs.file(ratignoreFile) + exclude(ratignoreFile.readLines().map { it.trim() }.filter { it.isNotBlank() && !it.startsWith("#") }) +} - // The NAR plugin copies from runtimeClasspath which resolves project dependencies - // as class directories, not JARs. The NarClassLoader expects JARs in - // META-INF/bundled-dependencies/. Force the NAR task to use JAR artifacts. - val jarView = configurations.named("runtimeClasspath").get() - .incoming.artifactView { - attributes { - attribute( - LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, - objects.named(LibraryElements::class.java, LibraryElements.JAR) - ) +idea { + project { + settings { + // add ASL2 copyright profile to IntelliJ + copyright { + useDefault = "ASL2" + profiles { + create("ASL2") { + notice = rootProject.file("src/license-header.txt").readText().trimEnd() + keyword = "Copyright" + } } - }.files - tasks.named("nar", Jar::class.java) { - into("META-INF/bundled-dependencies") { - from(jarView) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE } } } - - // Set archive names to match Maven artifactId for nested modules. - val parentProject = project.parent - if (parentProject != null && parentProject != rootProject && parentProject.parent != rootProject - && !project.name.startsWith(parentProject.name)) { - val qualifiedName = "${parentProject.name}-${project.name}" - the().archivesName.set(qualifiedName) - pluginManager.withPlugin("io.github.merlimat.nar") { - @Suppress("UNCHECKED_CAST") - val narExt = extensions.getByName("nar") - val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as Property - narIdProp.set(qualifiedName) - } - } - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version, - ) - } - } } - -// Access version catalog from subprojects -val Project.libs: org.gradle.accessors.dm.LibrariesForLibs - get() = rootProject.extensions.getByType() diff --git a/canal/build.gradle.kts b/canal/build.gradle.kts index 2a20556272..29397beeb3 100644 --- a/canal/build.gradle.kts +++ b/canal/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/cassandra/build.gradle.kts b/cassandra/build.gradle.kts index 8c2fab2c5b..da76cc01b8 100644 --- a/cassandra/build.gradle.kts +++ b/cassandra/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/core/build.gradle.kts b/debezium/core/build.gradle.kts index df1cc24cb4..746dc05543 100644 --- a/debezium/core/build.gradle.kts +++ b/debezium/core/build.gradle.kts @@ -17,6 +17,9 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} dependencies { compileOnly(libs.pulsar.io.core) diff --git a/debezium/mongodb/build.gradle.kts b/debezium/mongodb/build.gradle.kts index 53c2670166..d4292a9876 100644 --- a/debezium/mongodb/build.gradle.kts +++ b/debezium/mongodb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/mssql/build.gradle.kts b/debezium/mssql/build.gradle.kts index 1cfcc2c88a..07aac7f6ca 100644 --- a/debezium/mssql/build.gradle.kts +++ b/debezium/mssql/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/mysql/build.gradle.kts b/debezium/mysql/build.gradle.kts index bc79ff6e2a..33e30485a1 100644 --- a/debezium/mysql/build.gradle.kts +++ b/debezium/mysql/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/oracle/build.gradle.kts b/debezium/oracle/build.gradle.kts index 35bf12bcb4..dd1ef02d83 100644 --- a/debezium/oracle/build.gradle.kts +++ b/debezium/oracle/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/debezium/postgres/build.gradle.kts b/debezium/postgres/build.gradle.kts index f49f8a81fa..8dc37a5f32 100644 --- a/debezium/postgres/build.gradle.kts +++ b/debezium/postgres/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/distribution/io/build.gradle.kts b/distribution/io/build.gradle.kts index 91c7721e9f..9aa7b81470 100644 --- a/distribution/io/build.gradle.kts +++ b/distribution/io/build.gradle.kts @@ -18,9 +18,9 @@ */ // Distribution module — no Java compilation needed -tasks.named("compileJava") { enabled = false } -tasks.named("compileTestJava") { enabled = false } -tasks.named("jar") { enabled = false } +plugins { + base +} val pulsarVersion = project.version.toString() diff --git a/docker/pulsar-all/build.gradle.kts b/docker/pulsar-all/build.gradle.kts index 5ea1aa5f7a..448d3d1f3c 100644 --- a/docker/pulsar-all/build.gradle.kts +++ b/docker/pulsar-all/build.gradle.kts @@ -18,9 +18,9 @@ */ // Docker image module — no Java compilation needed -tasks.named("compileJava") { enabled = false } -tasks.named("compileTestJava") { enabled = false } -tasks.named("jar") { enabled = false } +plugins { + base +} val catalog = extensions.getByType().named("libs") val pulsarConnectorsVersion = project.version.toString() diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 62cabf83a4..0f89397b3f 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { implementation(libs.pulsar.io.core) implementation(libs.guava) @@ -51,6 +55,34 @@ dependencies { implementation(project(":rabbitmq")) implementation(project(":redis")) implementation(project(":solr")) - implementation(project(":alluxio")) + implementation(project(":alluxio")) { + exclude("org.eclipse.jetty", "jetty-bom") + exclude("io.netty", "netty-bom") + exclude("io.grpc", "grpc-bom") + } implementation(project(":azure-data-explorer")) } + +val exportClasspath by tasks.registering { + dependsOn(tasks.classes) + val classpath = sourceSets.main.get().output + configurations.runtimeClasspath.get() + inputs.files(classpath) + val outputFile = layout.buildDirectory.file("classpath.txt") + outputs.file(outputFile) + doLast { + outputFile.get().asFile.apply { + parentFile.mkdirs() + writeText(classpath.asPath) + } + } +} + +tasks.register("generateConnectorDocs") { + dependsOn(tasks.classes) + mainClass.set("org.apache.pulsar.io.docs.ConnectorDocGenerator") + classpath = sourceSets.main.get().output + configurations.runtimeClasspath.get() + inputs.files(classpath) + val outputDir = layout.buildDirectory.dir("connector-docs") + outputs.dir(outputDir) + args("-o", outputDir.get().asFile.absolutePath) +} \ No newline at end of file diff --git a/docs/pulsar-io-gen.sh b/docs/pulsar-io-gen.sh new file mode 100755 index 0000000000..0f2416a7e3 --- /dev/null +++ b/docs/pulsar-io-gen.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# Check for the java to use +if [[ -z $JAVA_HOME ]]; then + JAVA=$(which java) + if [ $? != 0 ]; then + echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 + exit 1 + fi +else + JAVA=$JAVA_HOME/bin/java +fi +CLASSPATH_FILE="$SCRIPT_DIR/build/classpath.txt" +$SCRIPT_DIR/../gradlew exportClasspath --console=plain > /dev/null +$JAVA -cp "$(cat $CLASSPATH_FILE)" org.apache.pulsar.io.docs.ConnectorDocGenerator "$@" \ No newline at end of file diff --git a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java index 2e9d6a9f27..e159c889ac 100644 --- a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java +++ b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java @@ -28,6 +28,7 @@ import java.lang.reflect.Modifier; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -51,8 +52,11 @@ public class ConnectorDocGenerator implements Callable { private static Reflections newReflections() throws Exception { final String[] classpathList = System.getProperty("java.class.path").split(":"); final List urlList = new ArrayList<>(); - for (String file : classpathList) { - urlList.add(new File(file).toURI().toURL()); + for (String cp : classpathList) { + File file = new File(cp); + if (file.isFile() && cp.endsWith(".nar")) { + urlList.add(new URL("jar:" + file.toURI() + "!/")); + } } return new Reflections(new ConfigurationBuilder().setUrls(urlList)); } @@ -70,7 +74,7 @@ private void generateConnectorYamlFile(Class configClass, PrintWriter writer) Field[] fields = configClass.getDeclaredFields(); for (Field field : fields) { - if (Modifier.isStatic(field.getModifiers())) { + if (Modifier.isStatic(field.getModifiers()) || Modifier.isTransient(field.getModifiers())) { continue; } FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class); @@ -107,12 +111,16 @@ private void generatorConnectorYamlFiles(String outputDir) throws IOException { Set> connectorClasses = reflections.getTypesAnnotatedWith(Connector.class); log.info("Retrieve all `Connector` annotated classes : {}", connectorClasses); + Path outputDirPath = Path.of(outputDir); + if (!Files.exists(outputDirPath)) { + Files.createDirectories(outputDirPath); + } for (Class connectorClass : connectorClasses) { final Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); final String name = connectorDef.name().toLowerCase(); final String type = connectorDef.type().name().toLowerCase(); final String filename = "pulsar-io-%s-%s.yml".formatted(name, type); - final Path outputPath = Path.of(outputDir, filename); + final Path outputPath = outputDirPath.resolve(filename); try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); generateConnectorYamlFile(connectorClass, connectorDef, pw); @@ -125,7 +133,7 @@ private void generatorConnectorYamlFiles(String outputDir) throws IOException { names = {"-o", "--output-dir"}, description = "The output dir to dump connector docs", required = true) - String outputDir = null; + String outputDir; @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") boolean help = false; diff --git a/dynamodb/build.gradle.kts b/dynamodb/build.gradle.kts index 7697d16bae..980e18c0ba 100644 --- a/dynamodb/build.gradle.kts +++ b/dynamodb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/elastic-search/build.gradle.kts b/elastic-search/build.gradle.kts index 620cb7759f..1b2022cd8e 100644 --- a/elastic-search/build.gradle.kts +++ b/elastic-search/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { compileOnly(libs.pulsar.io.core) diff --git a/file/build.gradle.kts b/file/build.gradle.kts index a22c7ac9fb..3725b4f536 100644 --- a/file/build.gradle.kts +++ b/file/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/gradle.properties b/gradle.properties index 20a29b3a26..fc83eeeece 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,7 +17,11 @@ # under the License. # +group=org.apache.pulsar +version=4.3.0-SNAPSHOT + org.gradle.configuration-cache=true +org.gradle.configureondemand=true org.gradle.parallel=true org.gradle.caching=true org.gradle.jvmargs=-Xmx4g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0e87504f16..743d543fc1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,26 +16,21 @@ # specific language governing permissions and limitations # under the License. # - [versions] # Core -pulsar = "4.1.3" -pulsar-connectors = "4.3.0-SNAPSHOT" +pulsar = "4.2.0" java = "17" - -# Docker -docker-jdk = "21" -pulsar-client-python = "3.10.0" - # Code quality -checkstyle = "10.14.2" - +checkstyle = "13.3.0" +spotless = "8.4.0" +idea-ext = "1.4.1" # Major frameworks bookkeeper = "4.17.3" zookeeper = "3.9.5" netty = "4.1.132.Final" netty-iouring = "0.0.26.Final" jetty = "12.1.5" +jetty9 = "9.4.58.v20250814" jersey = "2.42" jackson = "2.18.6" protobuf = "3.25.5" @@ -43,16 +38,14 @@ grpc = "1.75.0" slf4j = "2.0.17" log4j2 = "2.25.3" lombok = "1.18.42" - # OpenTelemetry opentelemetry = "1.56.0" opentelemetry-alpha = "1.56.0-alpha" opentelemetry-instrumentation = "2.21.0" opentelemetry-instrumentation-alpha = "2.21.0-alpha" opentelemetry-semconv = "1.37.0" - # Apache Commons -commons-lang3 = "3.19.0" +commons-lang3 = "3.20.0" commons-io = "2.21.0" commons-codec = "1.20.0" commons-compress = "1.28.0" @@ -63,7 +56,6 @@ commons-math3 = "3.6.1" commons-logging = "1.3.5" commons-beanutils = "1.11.0" commons-configuration2 = "2.12.0" - # BouncyCastle bouncycastle-bcprov = "1.78.1" bouncycastle-bcpkix = "1.81" @@ -71,49 +63,40 @@ bouncycastle-bcutil = "1.81" bouncycastle-bcprov-ext = "1.78.1" bouncycastle-bcpkix-fips = "2.0.10" bouncycastle-bc-fips = "2.0.1" - # Serialization avro = "1.12.0" gson = "2.13.2" snakeyaml = "2.0" - # Vert.x vertx = "4.5.24" - # Networking / HTTP asynchttpclient = "2.12.4" conscrypt = "2.5.2" okhttp3 = "5.3.1" okio = "3.16.3" -netty-tcnative = "2.0.75.Final" httpcomponents-httpclient = "4.5.13" httpcomponents-httpcore = "4.4.15" - # Google libraries (transitive deps, versions managed to match Maven) google-auth = "1.24.1" google-http-client = "1.41.0" j2objc-annotations = "1.3" opencensus = "0.28.0" opentelemetry-gcp-resources = "1.48.0-alpha" - # Data structures / Utils guava = "33.4.8-jre" caffeine = "3.2.3" -fastutil = "8.5.16" jctools = "4.0.5" roaringbitmap = "1.6.9" hppc = "0.9.1" aircompressor = "2.0.3" completable-futures = "0.3.6" re2j = "1.8" - # Metrics / Observability prometheus = "0.16.0" prometheus-jmx = "0.16.1" dropwizardmetrics = "4.1.12.1" hdrHistogram = "2.1.9" perfmark = "0.26.0" - # Auth jsonwebtoken = "0.13.0" athenz = "1.10.62" @@ -121,18 +104,15 @@ jose4j = "0.9.6" nimbus-jose-jwt = "9.37.4" auth0-java-jwt = "4.3.0" auth0-jwks-rsa = "0.22.0" - # CLI picocli = "4.7.5" jline3 = "3.21.0" jline2 = "2.14.6" - javassist = "3.25.0-GA" rocksdb = "7.9.2" kotlin-stdlib = "1.8.20" audience-annotations = "0.12.0" jetbrains-annotations = "13.0" - # Misc curator = "5.7.1" reflections = "0.10.2" @@ -151,7 +131,6 @@ guice = "5.1.0" snappy = "1.1.10.8" ipaddress = "5.5.0" zt-zip = "1.17" - # Jakarta jakarta-ws-rs = "2.1.6" jakarta-annotation = "1.3.5" @@ -159,17 +138,14 @@ jakarta-activation = "1.2.2" jakarta-xml-bind = "2.3.3" jakarta-validation = "2.0.2" javax-servlet = "3.1.0" - # Oxia / etcd oxia = "0.7.4" - # Build plugins -lightproto = "0.6.2" +lightproto = "0.6.6" errorprone = "2.45.0" spotbugs = "4.9.6" checkerframework = "3.33.0" jsr305 = "3.0.2" - # Test testng = "7.7.1" mockito = "5.19.0" @@ -189,16 +165,13 @@ skyscreamer = "1.5.0" zstd-jni = "1.5.7-3" lz4java = "1.10.3" spring = "6.2.12" - # Connectors / IO kafka-client = "4.1.1" confluent = "8.1.1" opensearch = "2.19.4" elasticsearch-java = "8.15.3" debezium = "3.4.2.Final" -debezium-mysql-connector = "9.4.0" kubernetesclient = "23.0.0" - # IO connector specific aerospike-client = "4.5.0" aws-sdk = "1.12.788" @@ -214,67 +187,47 @@ solr = "9.8.0" hbase = "2.6.4-hadoop3" hadoop3 = "3.4.3" jclouds = "2.6.0" - +openmldb = "0.4.4-hotfix1" +docker-java = "3.7.1" # Shading -shadow = "9.3.2" +shadow = "9.4.1" [libraries] # SLF4J -slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } -jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } - +slf4j-bom = { module = "org.slf4j:slf4j-bom", version.ref = "slf4j" } +slf4j-api = { module = "org.slf4j:slf4j-api" } +jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j" } # Log4j2 -log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j2" } -log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j2" } -log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j2" } -log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json", version.ref = "log4j2" } -log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j2" } - +log4j-bom = { module = "org.apache.logging.log4j:log4j-bom", version.ref = "log4j2" } +log4j-api = { module = "org.apache.logging.log4j:log4j-api" } +log4j-core = { module = "org.apache.logging.log4j:log4j-core" } +log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl" } # Lombok lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } - # Jackson jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson" } -jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" } -jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson" } -jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } -jackson-module-parameter-names = { module = "com.fasterxml.jackson.module:jackson-module-parameter-names", version.ref = "jackson" } -jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } -jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", version.ref = "jackson" } -jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } -jackson-module-jsonSchema = { module = "com.fasterxml.jackson.module:jackson-module-jsonSchema", version.ref = "jackson" } -jackson-jaxrs-json-provider = { module = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", version.ref = "jackson" } - +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } +jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } +jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor" } # Netty netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } -netty-common = { module = "io.netty:netty-common", version.ref = "netty" } -netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty" } -netty-handler = { module = "io.netty:netty-handler", version.ref = "netty" } -netty-transport = { module = "io.netty:netty-transport", version.ref = "netty" } -netty-codec-http = { module = "io.netty:netty-codec-http", version.ref = "netty" } -netty-codec-http2 = { module = "io.netty:netty-codec-http2", version.ref = "netty" } -netty-codec-haproxy = { module = "io.netty:netty-codec-haproxy", version.ref = "netty" } -netty-handler-proxy = { module = "io.netty:netty-handler-proxy", version.ref = "netty" } -netty-codec-socks = { module = "io.netty:netty-codec-socks", version.ref = "netty" } -netty-resolver-dns = { module = "io.netty:netty-resolver-dns", version.ref = "netty" } -netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native-macos", version.ref = "netty" } -netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" } -netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common", version.ref = "netty" } -netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" } +netty-buffer = { module = "io.netty:netty-buffer" } +netty-handler = { module = "io.netty:netty-handler" } +netty-codec-http = { module = "io.netty:netty-codec-http" } netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" } netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" } netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" } - # Protobuf / gRPC -protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } -protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util", version.ref = "protobuf" } -grpc-all = { module = "io.grpc:grpc-all", version.ref = "grpc" } -grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref = "grpc" } -grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } - +protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref = "protobuf" } +protobuf-java = { module = "com.google.protobuf:protobuf-java" } +protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util" } +grpc-bom = { module = "io.grpc:grpc-bom", version.ref = "grpc" } +grpc-all = { module = "io.grpc:grpc-all" } +grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded" } +grpc-stub = { module = "io.grpc:grpc-stub" } # Guava guava = { module = "com.google.guava:guava", version.ref = "guava" } - # Apache Commons commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons-lang3" } commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } @@ -283,7 +236,6 @@ commons-compress = { module = "org.apache.commons:commons-compress", version.ref commons-collections4 = { module = "org.apache.commons:commons-collections4", version.ref = "commons-collections4" } commons-cli = { module = "commons-cli:commons-cli", version.ref = "commons-cli" } commons-math3 = { module = "org.apache.commons:commons-math3", version.ref = "commons-math3" } - # BookKeeper bookkeeper-server = { module = "org.apache.bookkeeper:bookkeeper-server", version.ref = "bookkeeper" } bookkeeper-common-allocator = { module = "org.apache.bookkeeper:bookkeeper-common-allocator", version.ref = "bookkeeper" } @@ -297,86 +249,74 @@ bookkeeper-stream-storage-service-impl = { module = "org.apache.bookkeeper:strea bookkeeper-tools-framework = { module = "org.apache.bookkeeper:bookkeeper-tools-framework", version.ref = "bookkeeper" } bookkeeper-http-vertx-server = { module = "org.apache.bookkeeper.http:vertx-http-server", version.ref = "bookkeeper" } distributedlog-core = { module = "org.apache.distributedlog:distributedlog-core", version.ref = "bookkeeper" } - # ZooKeeper zookeeper = { module = "org.apache.zookeeper:zookeeper", version.ref = "zookeeper" } zookeeper-tests = { module = "org.apache.zookeeper:zookeeper", version.ref = "zookeeper" } - # Curator curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "curator" } - # Conscrypt conscrypt-openjdk-uber = { module = "org.conscrypt:conscrypt-openjdk-uber", version.ref = "conscrypt" } - # Jetty -jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" } -jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "jetty" } -jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server", version.ref = "jetty" } -jetty-compression-server = { module = "org.eclipse.jetty.compression:jetty-compression-server", version.ref = "jetty" } -jetty-compression-gzip = { module = "org.eclipse.jetty.compression:jetty-compression-gzip", version.ref = "jetty" } -jetty-client = { module = "org.eclipse.jetty:jetty-client", version.ref = "jetty" } -jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet", version.ref = "jetty" } -jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets", version.ref = "jetty" } -jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy", version.ref = "jetty" } -jetty-websocket-jetty-api = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-api", version.ref = "jetty" } -jetty-websocket-jetty-client = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-client", version.ref = "jetty" } -jetty-ee8-websocket-jetty-server = { module = "org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server", version.ref = "jetty" } - +jetty-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty" } +jetty9-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty9" } +jetty-server = { module = "org.eclipse.jetty:jetty-server" } +jetty-util = { module = "org.eclipse.jetty:jetty-util" } +jetty-alpn-conscrypt-server = { module = "org.eclipse.jetty:jetty-alpn-conscrypt-server" } +jetty-compression-server = { module = "org.eclipse.jetty.compression:jetty-compression-server" } +jetty-compression-gzip = { module = "org.eclipse.jetty.compression:jetty-compression-gzip" } +jetty-client = { module = "org.eclipse.jetty:jetty-client" } +jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet" } +jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets" } +jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy" } +jetty-websocket-jetty-api = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-api" } +jetty-websocket-jetty-client = { module = "org.eclipse.jetty.websocket:jetty-websocket-jetty-client" } +jetty-ee8-websocket-jetty-server = { module = "org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server" } # Jersey -jersey-server = { module = "org.glassfish.jersey.core:jersey-server", version.ref = "jersey" } -jersey-container-servlet-core = { module = "org.glassfish.jersey.containers:jersey-container-servlet-core", version.ref = "jersey" } -jersey-container-servlet = { module = "org.glassfish.jersey.containers:jersey-container-servlet", version.ref = "jersey" } -jersey-media-json-jackson = { module = "org.glassfish.jersey.media:jersey-media-json-jackson", version.ref = "jersey" } -jersey-client = { module = "org.glassfish.jersey.core:jersey-client", version.ref = "jersey" } -jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2", version.ref = "jersey" } -jersey-media-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } -jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jersey-test-framework-core", version.ref = "jersey" } -jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2", version.ref = "jersey" } - +jersey-bom = { module = "org.glassfish.jersey:jersey-bom", version.ref = "jersey" } +jersey-server = { module = "org.glassfish.jersey.core:jersey-server" } +jersey-container-servlet-core = { module = "org.glassfish.jersey.containers:jersey-container-servlet-core" } +jersey-container-servlet = { module = "org.glassfish.jersey.containers:jersey-container-servlet" } +jersey-media-json-jackson = { module = "org.glassfish.jersey.media:jersey-media-json-jackson" } +jersey-client = { module = "org.glassfish.jersey.core:jersey-client" } +jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2" } +jersey-media-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart" } +jersey-test-framework-core = { module = "org.glassfish.jersey.test-framework:jersey-test-framework-core" } +jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2" } # Prometheus -simpleclient = { module = "io.prometheus:simpleclient", version.ref = "prometheus" } -simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot", version.ref = "prometheus" } -simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine", version.ref = "prometheus" } -simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver", version.ref = "prometheus" } -simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet", version.ref = "prometheus" } -simpleclient-common = { module = "io.prometheus:simpleclient_common", version.ref = "prometheus" } -simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2", version.ref = "prometheus" } +simpleclient-bom = { module = "io.prometheus:simpleclient_bom", version.ref = "prometheus" } prometheus-jmx-collector = { module = "io.prometheus.jmx:collector", version.ref = "prometheus-jmx" } - # OpenTelemetry -opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } -opentelemetry-api-incubator = { module = "io.opentelemetry:opentelemetry-api-incubator", version.ref = "opentelemetry-alpha" } -opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } -opentelemetry-sdk-extension-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure", version.ref = "opentelemetry" } -opentelemetry-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing", version.ref = "opentelemetry" } -opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version.ref = "opentelemetry" } -opentelemetry-exporter-prometheus = { module = "io.opentelemetry:opentelemetry-exporter-prometheus", version.ref = "opentelemetry-alpha" } -opentelemetry-instrumentation-resources = { module = "io.opentelemetry.instrumentation:opentelemetry-resources", version.ref = "opentelemetry-instrumentation-alpha" } -opentelemetry-instrumentation-runtime-telemetry-java17 = { module = "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17", version.ref = "opentelemetry-instrumentation-alpha" } +opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version.ref = "opentelemetry" } +opentelemetry-bom-alpha = { module = "io.opentelemetry:opentelemetry-bom-alpha", version.ref = "opentelemetry-alpha" } +opentelemetry-instrumentation-bom = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom", version.ref = "opentelemetry-instrumentation" } +opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version.ref = "opentelemetry-instrumentation-alpha" } +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api" } +opentelemetry-api-incubator = { module = "io.opentelemetry:opentelemetry-api-incubator" } +opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk" } +opentelemetry-sdk-extension-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure" } +opentelemetry-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp" } +opentelemetry-exporter-prometheus = { module = "io.opentelemetry:opentelemetry-exporter-prometheus" } +opentelemetry-instrumentation-resources = { module = "io.opentelemetry.instrumentation:opentelemetry-resources" } +opentelemetry-instrumentation-runtime-telemetry-java17 = { module = "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17" } opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version.ref = "opentelemetry-semconv" } opentelemetry-gcp-resources = { module = "io.opentelemetry.contrib:opentelemetry-gcp-resources", version.ref = "opentelemetry-gcp-resources" } - # BouncyCastle bcpkix-jdk18on = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle-bcpkix" } bcprov-ext-jdk18on = { module = "org.bouncycastle:bcprov-ext-jdk18on", version.ref = "bouncycastle-bcprov-ext" } bcpkix-fips = { module = "org.bouncycastle:bcpkix-fips", version.ref = "bouncycastle-bcpkix-fips" } bc-fips = { module = "org.bouncycastle:bc-fips", version.ref = "bouncycastle-bc-fips" } -bcutil-fips = { module = "org.bouncycastle:bcutil-fips", version = "2.0.5" } - +bcutil-fips = "org.bouncycastle:bcutil-fips:2.0.5" # RocksDB rocksdbjni = { module = "org.rocksdb:rocksdbjni", version.ref = "rocksdb" } - # Error Prone error-prone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "errorprone" } - # Data structures caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } -fastutil = { module = "it.unimi.dsi:fastutil", version.ref = "fastutil" } jctools-core = { module = "org.jctools:jctools-core", version.ref = "jctools" } roaringbitmap = { module = "org.roaringbitmap:RoaringBitmap", version.ref = "roaringbitmap" } hppc = { module = "com.carrotsearch:hppc", version.ref = "hppc" } aircompressor = { module = "io.airlift:aircompressor", version.ref = "aircompressor" } - # Misc libs gson = { module = "com.google.code.gson:gson", version.ref = "gson" } re2j = { module = "com.google.re2j:re2j", version.ref = "re2j" } @@ -391,10 +331,12 @@ javassist = { module = "org.javassist:javassist", version.ref = "javassist" } commons-text = { module = "org.apache.commons:commons-text", version.ref = "commons-text" } typetools = { module = "net.jodah:typetools", version.ref = "typetools" } perfmark-api = { module = "io.perfmark:perfmark-api", version.ref = "perfmark" } -okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm", version.ref = "okhttp3" } -okhttp3-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp3" } -okio = { module = "com.squareup.okio:okio", version.ref = "okio" } -# Transitive dep version pins (enforced by pulsar-dependencies platform) +okhttp3-bom = { module = "com.squareup.okhttp3:okhttp-bom", version.ref = "okhttp3" } +okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm" } +okhttp3-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor" } +okio-bom = { module = "com.squareup.okio:okio-bom", version.ref = "okio" } +okio = { module = "com.squareup.okio:okio" } +# Transitive dep version pins (enforced by pulsar-connectors-dependencies platform) google-auth-library-credentials = { module = "com.google.auth:google-auth-library-credentials", version.ref = "google-auth" } google-auth-library-oauth2-http = { module = "com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth" } google-http-client = { module = "com.google.http-client:google-http-client", version.ref = "google-http-client" } @@ -406,7 +348,6 @@ httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", v httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "httpcomponents-httpcore" } jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version.ref = "jakarta-annotation" } kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin-stdlib" } - snakeyaml = { module = "org.yaml:snakeyaml", version.ref = "snakeyaml" } ant = { module = "org.apache.ant:ant", version.ref = "ant" } guice = { module = "com.google.inject:guice", version.ref = "guice" } @@ -416,7 +357,7 @@ vertx-core = { module = "io.vertx:vertx-core", version.ref = "vertx" } vertx-web = { module = "io.vertx:vertx-web", version.ref = "vertx" } avro = { module = "org.apache.avro:avro", version.ref = "avro" } avro-protobuf = { module = "org.apache.avro:avro-protobuf", version.ref = "avro" } -joda-time = { module = "joda-time:joda-time", version = "2.10.10" } +joda-time = "joda-time:joda-time:2.10.10" sketches-core = { module = "com.yahoo.datasketches:sketches-core", version.ref = "sketches" } java-semver = { module = "com.github.zafarkhaja:java-semver", version.ref = "java-semver" } oshi-core = { module = "com.github.oshi:oshi-core-java11", version.ref = "oshi" } @@ -426,16 +367,15 @@ dropwizardmetrics-core = { module = "io.dropwizard.metrics:metrics-core", versio dropwizardmetrics-graphite = { module = "io.dropwizard.metrics:metrics-graphite", version.ref = "dropwizardmetrics" } dropwizardmetrics-jvm = { module = "io.dropwizard.metrics:metrics-jvm", version.ref = "dropwizardmetrics" } snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy" } -jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" } +jspecify = "org.jspecify:jspecify:1.0.0" reflections = { module = "org.reflections:reflections", version.ref = "reflections" } - # Auth / Security -jjwt-api = { module = "io.jsonwebtoken:jjwt-api", version.ref = "jsonwebtoken" } -jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl", version.ref = "jsonwebtoken" } -jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson", version.ref = "jsonwebtoken" } +jjwt-bom = { module = "io.jsonwebtoken:jjwt-bom", version.ref = "jsonwebtoken" } +jjwt-api = { module = "io.jsonwebtoken:jjwt-api" } +jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl" } +jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson" } auth0-java-jwt = { module = "com.auth0:java-jwt", version.ref = "auth0-java-jwt" } auth0-jwks-rsa = { module = "com.auth0:jwks-rsa", version.ref = "auth0-jwks-rsa" } - # Jakarta jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "jakarta-ws-rs" } jakarta-activation-api = { module = "jakarta.activation:jakarta.activation-api", version.ref = "jakarta-activation" } @@ -444,18 +384,16 @@ jakarta-xml-bind-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", versi javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version.ref = "javax-servlet" } zt-zip = { module = "org.zeroturnaround:zt-zip", version.ref = "zt-zip" } ipaddress = { module = "com.github.seancfoley:ipaddress", version.ref = "ipaddress" } - # Oxia / etcd oxia-client = { module = "io.github.oxia-db:oxia-client", version.ref = "oxia" } oxia-testcontainers = { module = "io.github.oxia-db:oxia-testcontainers", version.ref = "oxia" } - # Static analysis spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version.ref = "spotbugs" } jsr305 = { module = "com.google.code.findbugs:jsr305", version.ref = "jsr305" } - # Test testng = { module = "org.testng:testng", version.ref = "testng" } -mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } +mockito-bom = { module = "org.mockito:mockito-bom", version.ref = "mockito" } +mockito-core = { module = "org.mockito:mockito-core" } awaitility = { module = "org.awaitility:awaitility", version.ref = "awaitility" } assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" } @@ -475,33 +413,35 @@ spring-jdbc = { module = "org.springframework:spring-jdbc", version.ref = "sprin spring-orm = { module = "org.springframework:spring-orm", version.ref = "spring" } kubernetes-client-java = { module = "io.kubernetes:client-java", version.ref = "kubernetesclient" } kubernetes-client-java-api-fluent = { module = "io.kubernetes:client-java-api-fluent", version.ref = "kubernetesclient" } -testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } -testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "testcontainers" } -testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy", version.ref = "testcontainers" } -testcontainers-localstack = { module = "org.testcontainers:localstack", version.ref = "testcontainers" } -testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq", version.ref = "testcontainers" } -testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } -testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref = "testcontainers" } -testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } -testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "testcontainers" } -testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "testcontainers" } -testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "testcontainers" } -testcontainers-k3s = { module = "org.testcontainers:k3s", version.ref = "testcontainers" } +docker-java-bom = { module = "com.github.docker-java:docker-java-bom", version.ref = "docker-java" } +testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version.ref = "testcontainers" } +testcontainers = { module = "org.testcontainers:testcontainers" } +testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch" } +testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy" } +testcontainers-localstack = { module = "org.testcontainers:localstack" } +testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq" } +testcontainers-kafka = { module = "org.testcontainers:kafka" } +testcontainers-mysql = { module = "org.testcontainers:mysql" } +testcontainers-postgresql = { module = "org.testcontainers:postgresql" } +testcontainers-mongodb = { module = "org.testcontainers:mongodb" } +testcontainers-pulsar = { module = "org.testcontainers:pulsar" } +testcontainers-cassandra = { module = "org.testcontainers:cassandra" } +testcontainers-k3s = { module = "org.testcontainers:k3s" } kerby-simplekdc = { module = "org.apache.kerby:kerb-simplekdc", version.ref = "kerby" } json = { module = "org.json:json", version.ref = "json" } - # AWS SDKs -aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core", version.ref = "aws-sdk" } -aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts", version.ref = "aws-sdk" } -aws-sdk2-regions = { module = "software.amazon.awssdk:regions", version.ref = "aws-sdk2" } -aws-sdk2-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws-sdk2" } -aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis", version.ref = "aws-sdk2" } -aws-sdk2-utils = { module = "software.amazon.awssdk:utils", version.ref = "aws-sdk2" } -dynamodb-streams-kinesis-adapter = { module = "com.amazonaws:dynamodb-streams-kinesis-adapter", version = "1.6.0" } -amazon-kinesis-client = { module = "software.amazon.kinesis:amazon-kinesis-client", version = "2.6.0" } -amazon-kinesis-client-v3 = { module = "software.amazon.kinesis:amazon-kinesis-client", version = "3.1.2" } -amazon-kinesis-producer = { module = "software.amazon.kinesis:amazon-kinesis-producer", version = "1.0.4" } - +aws-java-sdk-bom = { module = "com.amazonaws:aws-java-sdk-bom", version.ref = "aws-sdk" } +aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core" } +aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts" } +aws-sdk2-bom = { module = "software.amazon.awssdk:bom", version.ref = "aws-sdk2" } +aws-sdk2-regions = { module = "software.amazon.awssdk:regions" } +aws-sdk2-sts = { module = "software.amazon.awssdk:sts" } +aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis" } +aws-sdk2-utils = { module = "software.amazon.awssdk:utils" } +dynamodb-streams-kinesis-adapter = "com.amazonaws:dynamodb-streams-kinesis-adapter:1.6.0" +amazon-kinesis-client = "software.amazon.kinesis:amazon-kinesis-client:2.6.0" +amazon-kinesis-client-v3 = "software.amazon.kinesis:amazon-kinesis-client:3.1.2" +amazon-kinesis-producer = "software.amazon.kinesis:amazon-kinesis-producer:1.0.4" # Kafka / Confluent kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka-client" } kafka-connect-runtime = { module = "org.apache.kafka:connect-runtime", version.ref = "kafka-client" } @@ -512,104 +452,95 @@ kafka-connect-file = { module = "org.apache.kafka:connect-file", version.ref = " kafka-schema-registry-client = { module = "io.confluent:kafka-schema-registry-client", version.ref = "confluent" } kafka-avro-serializer = { module = "io.confluent:kafka-avro-serializer", version.ref = "confluent" } kafka-connect-avro-converter = { module = "io.confluent:kafka-connect-avro-converter", version.ref = "confluent" } - # ElasticSearch / OpenSearch opensearch-rest-high-level-client = { module = "org.opensearch.client:opensearch-rest-high-level-client", version.ref = "opensearch" } elasticsearch-java = { module = "co.elastic.clients:elasticsearch-java", version.ref = "elasticsearch-java" } - # Debezium -debezium-core = { module = "io.debezium:debezium-core", version.ref = "debezium" } -debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium" } -debezium-connector-mongodb = { module = "io.debezium:debezium-connector-mongodb", version.ref = "debezium" } -debezium-connector-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium" } -debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle", version.ref = "debezium" } -debezium-connector-sqlserver = { module = "io.debezium:debezium-connector-sqlserver", version.ref = "debezium" } - +debezium-bom = { module = "io.debezium:debezium-bom", version.ref = "debezium" } +debezium-core = { module = "io.debezium:debezium-core" } +debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql" } +debezium-connector-mongodb = { module = "io.debezium:debezium-connector-mongodb" } +debezium-connector-postgres = { module = "io.debezium:debezium-connector-postgres" } +debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle" } +debezium-connector-sqlserver = { module = "io.debezium:debezium-connector-sqlserver" } # Database drivers -postgresql-jdbc = { module = "org.postgresql:postgresql", version = "42.7.10" } -sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version = "3.47.1.0" } -clickhouse-jdbc = { module = "com.clickhouse:clickhouse-jdbc", version = "0.4.6" } -mariadb-jdbc = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.5.5" } -openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version = "0.4.4-hotfix1" } -openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version = "0.4.4-hotfix1" } - +postgresql-jdbc = "org.postgresql:postgresql:42.7.10" +sqlite-jdbc = "org.xerial:sqlite-jdbc:3.47.1.0" +clickhouse-jdbc = "com.clickhouse:clickhouse-jdbc:0.4.6" +mariadb-jdbc = "org.mariadb.jdbc:mariadb-java-client:3.5.5" +mysql-connector-j = "com.mysql:mysql-connector-j:9.4.0" +openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref = "openmldb" } +openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version.ref = "openmldb" } # JClouds jclouds-allblobstore = { module = "org.apache.jclouds:jclouds-allblobstore", version.ref = "jclouds" } jclouds-blobstore = { module = "org.apache.jclouds:jclouds-blobstore", version.ref = "jclouds" } - # Hadoop hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" } hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version.ref = "hadoop3" } hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" } hadoop-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop3" } - # HBase hbase-client = { module = "org.apache.hbase:hbase-client", version.ref = "hbase" } hbase-common = { module = "org.apache.hbase:hbase-common", version.ref = "hbase" } - # NoSQL / Search aerospike-client = { module = "com.aerospike:aerospike-client-bc", version.ref = "aerospike-client" } cassandra-driver = { module = "com.datastax.cassandra:cassandra-driver-core", version.ref = "cassandra-driver" } failsafe = { module = "dev.failsafe:failsafe", version.ref = "failsafe" } -docker-java-core = { module = "com.github.docker-java:docker-java-core", version = "3.4.1" } mongodb-driver-reactivestreams = { module = "org.mongodb:mongodb-driver-reactivestreams", version.ref = "mongodb-driver" } mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version.ref = "mongodb-driver" } lettuce-core = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" } solr-solrj = { module = "org.apache.solr:solr-solrj", version.ref = "solr" } solr-test-framework = { module = "org.apache.solr:solr-test-framework", version.ref = "solr" } solr-core = { module = "org.apache.solr:solr-core", version.ref = "solr" } - # Messaging rabbitmq-amqp-client = { module = "com.rabbitmq:amqp-client", version.ref = "rabbitmq-client" } nsq-j = { module = "com.sproutsocial:nsq-j", version.ref = "nsq-client" } - # Time series influxdb-client-java = { module = "com.influxdb:influxdb-client-java", version.ref = "influxdb-client" } influxdb-java = { module = "org.influxdb:influxdb-java", version.ref = "influxdb-java" } - # IO specific -jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref = "jackson" } -json-smart = { module = "net.minidev:json-smart", version = "2.5.2" } -json-flattener = { module = "com.github.wnameless.json:json-flattener", version = "0.16.4" } -flatbuffers-java = { module = "com.google.flatbuffers:flatbuffers-java", version = "1.9.0" } -jfairy = { module = "io.codearte.jfairy:jfairy", version = "0.5.9" } -embedded-redis = { module = "com.github.kstyrc:embedded-redis", version = "0.6" } - +json-smart = "net.minidev:json-smart:2.5.2" +json-flattener = "com.github.wnameless.json:json-flattener:0.16.4" +flatbuffers-java = "com.google.flatbuffers:flatbuffers-java:1.9.0" +jfairy = "io.codearte.jfairy:jfairy:0.5.9" +embedded-redis = "com.github.kstyrc:embedded-redis:0.6" # Auth athenz-zts-java-client = { module = "com.yahoo.athenz:athenz-zts-java-client", version.ref = "athenz" } athenz-cert-refresher = { module = "com.yahoo.athenz:athenz-cert-refresher", version.ref = "athenz" } athenz-auth-core = { module = "com.yahoo.athenz:athenz-auth-core", version.ref = "athenz" } athenz-zpe-java-client = { module = "com.yahoo.athenz:athenz-zpe-java-client", version.ref = "athenz" } - # Misc bcprov-jdk18on = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bouncycastle-bcprov" } commons-logging = { module = "commons-logging:commons-logging", version.ref = "commons-logging" } commons-beanutils = { module = "commons-beanutils:commons-beanutils", version.ref = "commons-beanutils" } commons-configuration2 = { module = "org.apache.commons:commons-configuration2", version.ref = "commons-configuration2" } bookkeeper-stats-api = { module = "org.apache.bookkeeper.stats:bookkeeper-stats-api", version.ref = "bookkeeper" } -datasketches-memory = { module = "org.apache.datasketches:datasketches-memory", version = "2.2.0" } -datasketches-java = { module = "org.apache.datasketches:datasketches-java", version = "6.1.1" } - +datasketches-memory = "org.apache.datasketches:datasketches-memory:2.2.0" +datasketches-java = "org.apache.datasketches:datasketches-java:6.1.1" # Pulsar core modules (published Maven artifacts, used by connectors) -pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core", version.ref = "pulsar" } -pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common", version.ref = "pulsar" } -pulsar-common = { module = "org.apache.pulsar:pulsar-common", version.ref = "pulsar" } -pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api", version.ref = "pulsar" } -pulsar-client = { module = "org.apache.pulsar:pulsar-client-original", version.ref = "pulsar" } -pulsar-client-admin = { module = "org.apache.pulsar:pulsar-client-admin-original", version.ref = "pulsar" } -pulsar-config-validation = { module = "org.apache.pulsar:pulsar-config-validation", version.ref = "pulsar" } -pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api", version.ref = "pulsar" } -pulsar-functions-instance = { module = "org.apache.pulsar:pulsar-functions-instance", version.ref = "pulsar" } -pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils", version.ref = "pulsar" } -pulsar-broker = { module = "org.apache.pulsar:pulsar-broker", version.ref = "pulsar" } -pulsar-broker-test = { module = "org.apache.pulsar:pulsar-broker", version.ref = "pulsar" } -pulsar-testmocks = { module = "org.apache.pulsar:testmocks", version.ref = "pulsar" } -pulsar-buildtools = { module = "org.apache.pulsar:buildtools", version.ref = "pulsar" } +pulsar-bom = { module = "org.apache.pulsar:pulsar-bom", version.ref = "pulsar" } +pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core" } +pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common" } +pulsar-common = { module = "org.apache.pulsar:pulsar-common" } +pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api" } +pulsar-client = { module = "org.apache.pulsar:pulsar-client-original" } +pulsar-client-admin = { module = "org.apache.pulsar:pulsar-client-admin-original" } +pulsar-config-validation = { module = "org.apache.pulsar:pulsar-config-validation" } +pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api" } +pulsar-functions-instance = { module = "org.apache.pulsar:pulsar-functions-instance" } +pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils" } +pulsar-broker = { module = "org.apache.pulsar:pulsar-broker" } +pulsar-testmocks = { module = "org.apache.pulsar:testmocks" } +pulsar-buildtools = { module = "org.apache.pulsar:buildtools" } [plugins] lightproto = { id = "io.streamnative.lightproto", version.ref = "lightproto" } -nar = { id = "io.github.merlimat.nar", version = "0.1.3" } -protobuf = { id = "com.google.protobuf", version = "0.9.6" } +nar = "io.github.merlimat.nar:0.1.3" +protobuf = "com.google.protobuf:0.9.6" shadow = { id = "com.gradleup.shadow", version.ref = "shadow" } -rat = { id = "org.nosphere.apache.rat", version = "0.8.1" } -license = { id = "com.github.hierynomus.license", version = "0.16.1" } +rat = "org.nosphere.apache.rat:0.8.1" +spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } +idea-ext = { id = "org.jetbrains.gradle.plugin.idea-ext", version.ref = "idea-ext" } +version-catalog-update = "nl.littlerobots.version-catalog-update:1.1.0" +versions = "com.github.ben-manes.versions:0.53.0" +crlf = "com.github.vlsi.crlf:3.0.1" diff --git a/hbase/build.gradle.kts b/hbase/build.gradle.kts index caa37adf66..be1f1e5c5b 100644 --- a/hbase/build.gradle.kts +++ b/hbase/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/hdfs3/build.gradle.kts b/hdfs3/build.gradle.kts index 1f20430559..82e22541ca 100644 --- a/hdfs3/build.gradle.kts +++ b/hdfs3/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/http/build.gradle.kts b/http/build.gradle.kts index f4008b947f..9b101afd8d 100644 --- a/http/build.gradle.kts +++ b/http/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/influxdb/build.gradle.kts b/influxdb/build.gradle.kts index a29d19b6ec..d3a589ec8d 100644 --- a/influxdb/build.gradle.kts +++ b/influxdb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/jdbc/clickhouse/build.gradle.kts b/jdbc/clickhouse/build.gradle.kts index da66bc4802..7a36945ed9 100644 --- a/jdbc/clickhouse/build.gradle.kts +++ b/jdbc/clickhouse/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/core/build.gradle.kts b/jdbc/core/build.gradle.kts index 7d38cdfde2..5d40fba464 100644 --- a/jdbc/core/build.gradle.kts +++ b/jdbc/core/build.gradle.kts @@ -17,6 +17,10 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} + dependencies { api(libs.pulsar.io.common) api(libs.pulsar.io.core) diff --git a/jdbc/mariadb/build.gradle.kts b/jdbc/mariadb/build.gradle.kts index 1691bff406..ce5fe962dd 100644 --- a/jdbc/mariadb/build.gradle.kts +++ b/jdbc/mariadb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/openmldb/build.gradle.kts b/jdbc/openmldb/build.gradle.kts index a9894a8ee8..7eae514afe 100644 --- a/jdbc/openmldb/build.gradle.kts +++ b/jdbc/openmldb/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/postgres/build.gradle.kts b/jdbc/postgres/build.gradle.kts index 752366f894..3aa3da17c2 100644 --- a/jdbc/postgres/build.gradle.kts +++ b/jdbc/postgres/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/jdbc/sqlite/build.gradle.kts b/jdbc/sqlite/build.gradle.kts index 4e1495a838..66c92cface 100644 --- a/jdbc/sqlite/build.gradle.kts +++ b/jdbc/sqlite/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(project(":jdbc:pulsar-io-jdbc-core")) diff --git a/kafka-connect-adaptor-nar/build.gradle.kts b/kafka-connect-adaptor-nar/build.gradle.kts index 4ee233d59d..05ebf6307a 100644 --- a/kafka-connect-adaptor-nar/build.gradle.kts +++ b/kafka-connect-adaptor-nar/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } nar { narId.set("pulsar-io-kafka-connect-adaptor") diff --git a/kafka-connect-adaptor/build.gradle.kts b/kafka-connect-adaptor/build.gradle.kts index 4809b8269e..1349adb1a4 100644 --- a/kafka-connect-adaptor/build.gradle.kts +++ b/kafka-connect-adaptor/build.gradle.kts @@ -17,6 +17,9 @@ * under the License. */ +plugins { + id("pulsar-connectors.java-conventions") +} dependencies { compileOnly(libs.pulsar.io.core) diff --git a/kafka/build.gradle.kts b/kafka/build.gradle.kts index 8046aa6af7..6d219d0460 100644 --- a/kafka/build.gradle.kts +++ b/kafka/build.gradle.kts @@ -18,16 +18,14 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } - // KafkaBytesSource uses SchemaInfoImpl from pulsar-common, which is excluded from -// NAR runtimeClasspath by the global exclusion. Bundle it via a separate configuration -// since the NAR classloader's parent (rootClassLoader) only has java-instance.jar. -val narExtraDeps by configurations.creating { - isCanBeResolved = true - isCanBeConsumed = false +// NAR runtimeClasspath by default. Include it in the NAR bundle. +pulsarConnectorsNar { + includePulsarModule("pulsar-common") } dependencies { @@ -35,7 +33,6 @@ dependencies { implementation(libs.pulsar.io.core) implementation(libs.pulsar.common) implementation(libs.pulsar.client) - narExtraDeps(libs.pulsar.common) implementation(libs.jackson.databind) implementation(libs.jackson.dataformat.yaml) implementation(libs.guava) @@ -52,9 +49,3 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.bcpkix.jdk18on) } - -tasks.named("nar") { - from(narExtraDeps) { into("META-INF/bundled-dependencies") } - bundledDependencies.from(narExtraDeps) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} diff --git a/kinesis-kpl-shaded/build.gradle.kts b/kinesis-kpl-shaded/build.gradle.kts index 4197e5d5de..bf6ecce7cf 100644 --- a/kinesis-kpl-shaded/build.gradle.kts +++ b/kinesis-kpl-shaded/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.shadow) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.shadow-conventions") } dependencies { @@ -34,16 +35,7 @@ configurations.all { } } -// Disable the default jar task so the shadow JAR is the only artifact. -// This avoids Gradle's implicit dependency validation errors when consumers -// use project() to depend on this module. -tasks.jar { - enabled = false -} - tasks.shadowJar { - archiveClassifier.set("") - mergeServiceFiles() dependencies { include(dependency("software.amazon.kinesis:amazon-kinesis-producer")) include(dependency("com.google.protobuf:protobuf-java")) diff --git a/kinesis/build.gradle.kts b/kinesis/build.gradle.kts index 53f4995e27..2a801334bc 100644 --- a/kinesis/build.gradle.kts +++ b/kinesis/build.gradle.kts @@ -18,13 +18,14 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { // The shaded KPL project bundles amazon-kinesis-producer with relocated protobuf. - // Use shadowElements to get the shadow JAR (which has relocated protobuf). - implementation(project(path = ":kinesis-kpl-shaded", configuration = "shadowElements")) + // The shadow convention exposes the shadow JAR as the primary artifact. + implementation(project(":kinesis-kpl-shaded")) // CompileOnly: needed for compilation against KPL classes but NOT bundled in NAR. // At runtime, KPL classes come from the shaded project's shadow JAR. compileOnly(libs.amazon.kinesis.producer) diff --git a/mongo/build.gradle.kts b/mongo/build.gradle.kts index b760c25863..d06d6c8bc6 100644 --- a/mongo/build.gradle.kts +++ b/mongo/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/netty/build.gradle.kts b/netty/build.gradle.kts index be70778709..43e3e5a685 100644 --- a/netty/build.gradle.kts +++ b/netty/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/nsq/build.gradle.kts b/nsq/build.gradle.kts index 732c22c695..be75a00087 100644 --- a/nsq/build.gradle.kts +++ b/nsq/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.core) diff --git a/pulsar-dependencies/build.gradle.kts b/pulsar-connectors-dependencies/build.gradle.kts similarity index 55% rename from pulsar-dependencies/build.gradle.kts rename to pulsar-connectors-dependencies/build.gradle.kts index 7bead9941f..0be83b0dad 100644 --- a/pulsar-dependencies/build.gradle.kts +++ b/pulsar-connectors-dependencies/build.gradle.kts @@ -19,7 +19,7 @@ // Enforced platform module that declares version constraints for all dependencies. // This is the Gradle equivalent of Maven's dependencyManagement section. -// All subprojects consume this via: implementation(enforcedPlatform(project(":pulsar-dependencies"))) +// All subprojects consume this via: implementation(enforcedPlatform(project(":pulsar-connectors-dependencies"))) plugins { `java-platform` } @@ -30,14 +30,20 @@ javaPlatform { } dependencies { - constraints { - // Iterate over all library declarations in the version catalog and add them as constraints. - // This ensures that any transitive dependency matching a catalog entry gets pinned to - // the version we specify, regardless of what version a transitive dependency requests. - val catalog = project.extensions.getByType().named("libs") - catalog.libraryAliases.forEach { alias -> - catalog.findLibrary(alias).ifPresent { provider -> - api(provider) + val catalog = project.extensions.getByType().named("libs") + // Iterate over all library declarations in the version catalog and add them as constraints. + // This ensures that any transitive dependency matching a catalog entry gets pinned to + // the version we specify, regardless of what version a transitive dependency requests. + // BOM entries (detected by alias name) are imported as platforms rather than constraints. + catalog.libraryAliases.forEach { alias -> + catalog.findLibrary(alias).ifPresent { provider -> + val dep = provider.get() + if (alias.endsWith(".bom")) { + api(platform(provider)) + } else if (dep.versionConstraint.requiredVersion.isNotEmpty()) { + // Only add constraints for entries with explicit versions. + // Versionless entries are managed by BOMs imported above. + constraints.api(provider) } } } diff --git a/rabbitmq/build.gradle.kts b/rabbitmq/build.gradle.kts index d7613cef2d..b15ee747ac 100644 --- a/rabbitmq/build.gradle.kts +++ b/rabbitmq/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/redis/build.gradle.kts b/redis/build.gradle.kts index eb57bee916..bfd82a9a68 100644 --- a/redis/build.gradle.kts +++ b/redis/build.gradle.kts @@ -18,7 +18,8 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } dependencies { implementation(libs.pulsar.io.common) diff --git a/settings.gradle.kts b/settings.gradle.kts index 9194ce066e..d186f01b9c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,6 +22,7 @@ pluginManagement { gradlePluginPortal() mavenCentral() } + includeBuild("build-logic") } dependencyResolutionManagement { @@ -49,8 +50,16 @@ dependencyResolutionManagement { rootProject.name = "pulsar-connectors" +// This build requires Java 17 or later. Version check can be skipped with -PskipJavaVersionCheck parameter. +val javaVersion = providers.provider { JavaVersion.current() } +require(providers.gradleProperty("skipJavaVersionCheck").isPresent + || javaVersion.get() >= JavaVersion.VERSION_17) { + "This build requires Java 17 or later, but is running on Java ${javaVersion.get()}. " + + "Pass -PskipJavaVersionCheck to skip this check." +} + // Enforced platform for dependency version management -include("pulsar-dependencies") +include("pulsar-connectors-dependencies") // Simple connectors (flat layout, top-level directories) include("aerospike") diff --git a/solr/build.gradle.kts b/solr/build.gradle.kts index d15cedc516..d338d67bef 100644 --- a/solr/build.gradle.kts +++ b/solr/build.gradle.kts @@ -18,10 +18,11 @@ */ plugins { - alias(libs.plugins.nar) + id("pulsar-connectors.java-conventions") + id("pulsar-connectors.nar-conventions") } // Solr 9.x embeds Jetty 10.x, which is incompatible with Pulsar's Jetty 12. -// The pulsar-dependencies platform enforces Jetty 12 strict versions, which override +// The pulsar-connectors-dependencies platform enforces Jetty 12 strict versions, which override // enforcedPlatform("jetty-bom:10.0.24") because Gradle picks the highest strict version. // Use resolutionStrategy.force to downgrade Jetty to 10.0.24 for test configurations. val jetty10Version = "10.0.24" diff --git a/src/license-header.txt b/src/license-header.txt new file mode 100644 index 0000000000..60b675e310 --- /dev/null +++ b/src/license-header.txt @@ -0,0 +1,16 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License.