From 091a67ed2f9fab35b8d18fdeb53d9d59a04ac6b4 Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Fri, 26 Jan 2024 14:44:04 +0800 Subject: [PATCH 01/58] Update base image (#873) (cherry picked from commit 975b9765b7a478668bc672ceba764fbf7f5f0930) --- image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/Dockerfile b/image/Dockerfile index 0c0e2ea7..aa9ce43e 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -18,5 +18,5 @@ # ARG PULSAR_VERSION -FROM docker.cloudsmith.io/streamnative/staging/pulsar-functions-java-runner:${PULSAR_VERSION} +FROM streamnative/pulsar-functions-java-runner:${PULSAR_VERSION} COPY --chown=$UID:$GID target/*.nar /pulsar/connectors/ From 00b0671f60415b3ec49bab31d96ec4703577d0dd Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Fri, 26 Jan 2024 15:27:53 +0800 Subject: [PATCH 02/58] Update base image (#874) (cherry picked from commit 5198b5d4bba78989027ebe04ea34db8758499cc3) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1d54dfc..bb053467 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: PULSAR_VERSION=`mvn -q -Dexec.executable=echo -Dexec.args='${pulsar.version}' --non-recursive exec:exec 2>/dev/null` REPO=`mvn -q -Dexec.executable=echo -Dexec.args='${project.artifactId}' --non-recursive exec:exec 2>/dev/null` IMAGE_REPO=streamnative/${REPO} - RUNNER_IMAGE=docker.cloudsmith.io/streamnative/staging/pulsar-functions-java-runner:${PULSAR_VERSION} + RUNNER_IMAGE=streamnative/pulsar-functions-java-runner:${PULSAR_VERSION} docker pull ${RUNNER_IMAGE} docker build --build-arg PULSAR_VERSION="$PULSAR_VERSION" -t ${IMAGE_REPO}:${CONNECTOR_VERSION} -f ./image/Dockerfile ./ docker push ${IMAGE_REPO}:${CONNECTOR_VERSION} From efcd6a8fc1fb82e0ffe906bb4aabfd99e1d8ea88 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Wed, 3 Apr 2024 17:41:11 +0800 Subject: [PATCH 03/58] Use the Apache images to run tests, in order to avoid permission issues. (#937) (cherry picked from commit 46bdf7ee95f0caa908fefc3971fbe2065d87ccac) --- .../java/org/apache/pulsar/io/jcloud/PulsarTestBase.java | 2 +- .../apache/pulsar/io/jcloud/container/PulsarContainer.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index 1c8ec760..2048a150 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -75,7 +75,7 @@ public static void prepare() throws Exception { log.info("-------------------------------------------------------------------------"); - final String pulsarImage = System.getProperty("pulsar.systemtest.image", "streamnative/pulsar:2.9.2.13"); + final String pulsarImage = System.getProperty("pulsar.systemtest.image", "apachepulsar/pulsar:latest"); pulsarService = new PulsarContainer(DockerImageName.parse(pulsarImage)); pulsarService.waitingFor(new HttpWaitStrategy() .forPort(BROKER_HTTP_PORT) diff --git a/src/test/java/org/apache/pulsar/io/jcloud/container/PulsarContainer.java b/src/test/java/org/apache/pulsar/io/jcloud/container/PulsarContainer.java index 31804099..4c67b6fb 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/container/PulsarContainer.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/container/PulsarContainer.java @@ -32,9 +32,9 @@ public class PulsarContainer extends GenericContainer { public static final int BROKER_HTTP_PORT = 8080; public static final String METRICS_ENDPOINT = "/metrics"; - private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("streamnative/pulsar"); + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("apachepulsar/pulsar"); @Deprecated - private static final String DEFAULT_TAG = "2.8.1.6"; + private static final String DEFAULT_TAG = "latest"; private boolean functionsWorkerEnabled = false; @@ -57,7 +57,7 @@ public PulsarContainer(String pulsarVersion) { public PulsarContainer(final DockerImageName dockerImageName) { super(dockerImageName); - dockerImageName.assertCompatibleWith(DockerImageName.parse("streamnative/pulsar")); + dockerImageName.assertCompatibleWith(DockerImageName.parse("apachepulsar/pulsar")); withExposedPorts(BROKER_PORT, BROKER_HTTP_PORT); withCommand("/pulsar/bin/pulsar", "standalone", "--no-functions-worker", "-nss"); From 4c15b6c560d0ee9fb538abfb48d69bc24261017b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:11:25 +0800 Subject: [PATCH 04/58] Bump json-smart from 2.4.7 to 2.4.9 (#622) Bumps [json-smart](https://github.com/netplex/json-smart-v2) from 2.4.7 to 2.4.9. - [Release notes](https://github.com/netplex/json-smart-v2/releases) - [Commits](https://github.com/netplex/json-smart-v2/commits/2.4.9) --- updated-dependencies: - dependency-name: net.minidev:json-smart dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 5ed32bdcd23e2395da27793dbe873de2a7688580) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6afb0501..915b3d34 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ 2.8.9 1.21 2.17.1 - 2.4.7 + 2.4.9 3.19.6 ${protobuf3.version} 1.42.1 From 885b6790b3b347e9bed4ece2eccbb89bb4ac190d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:14:10 +0800 Subject: [PATCH 05/58] Bump jackson-databind from 2.13.4.1 to 2.13.4.2 (#630) Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.13.4.1 to 2.13.4.2. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit d198b0a2f0668b4f83540e98de44a59eac79dc73) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 915b3d34..7dc51180 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ 2.13.2 - 2.13.4.1 + 2.13.4.2 1.18.20 2.11.0.4 1.12.2 From 111628c03bd82b041f11647942a9afa23ff46e92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:22:00 +0800 Subject: [PATCH 06/58] Bump org.apache.commons:commons-compress from 1.21 to 1.26.0 (#892) Bumps org.apache.commons:commons-compress from 1.21 to 1.26.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 351fc8ce2dc13fec9ed825d42c75fd6744b4e759) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7dc51180..50ed2b6f 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 2.16.1 2.3.1 2.8.9 - 1.21 + 1.26.0 2.17.1 2.4.9 3.19.6 From 0d55cebd5c75aeb138a81650193fdce04fe8325f Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 11 Apr 2024 11:07:44 +0800 Subject: [PATCH 07/58] cve: upgrade depend version (#944) (#947) (cherry picked from commit b10d0f247745024b549c083f26e83928d767aa8a) --- pom.xml | 137 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/pom.xml b/pom.xml index 50ed2b6f..5b1fcbe6 100644 --- a/pom.xml +++ b/pom.xml @@ -48,10 +48,12 @@ 2.13.2 2.13.4.2 1.18.20 - 2.11.0.4 - 1.12.2 - 1.12.148 - 2.16.1 + 3.2.2.1 + 1.11.3 + 3.3.6 + 1.13.1 + 1.12.698 + 2.16.104 2.3.1 2.8.9 1.26.0 @@ -61,7 +63,7 @@ ${protobuf3.version} 1.42.1 ${grpc.version} - 1.2.6 + 1.2.22 4.13.1 @@ -108,6 +110,16 @@ pulsar-io-common ${pulsar.version} + + io.streamnative + pulsar-common + ${pulsar.version} + + + io.streamnative + pulsar-client-original + ${pulsar.version} + com.fasterxml.jackson.core jackson-databind @@ -194,6 +206,11 @@ pom import + + org.apache.avro + avro + ${avro.version} + @@ -307,6 +324,18 @@ io.streamnative pulsar-io-core + + io.streamnative + pulsar-io-common + + + io.streamnative + pulsar-common + + + io.streamnative + pulsar-client-original + com.fasterxml.jackson.core @@ -385,39 +414,11 @@ org.apache.avro avro - 1.10.2 - - - org.apache.hadoop - hadoop-client - 3.3.3 - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-yarn-client - - - org.apache.hadoop - hadoop-yarn-common - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - org.apache.hadoop - hadoop-hdfs-client - - org.apache.hadoop hadoop-common - 3.3.3 + ${hadoop.version} org.apache.hadoop @@ -447,6 +448,41 @@ log4j log4j + + org.apache.zookeeper + zookeeper + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-client + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.zookeeper + zookeeper + @@ -563,43 +599,13 @@ pulsar test - - - - org.apache.commons - commons-lang3 - 3.11 - - - io.streamnative - pulsar-io-common - - - javax.xml.bind - jaxb-api - ${jaxb-api} - - - io.streamnative - pulsar-client-original - ${pulsar.version} - io.streamnative pulsar-client-admin-original ${pulsar.version} - - - net.java.dev.jna - jna - 5.7.0 test - - io.streamnative - pulsar-common - ${pulsar.version} - + @@ -625,7 +631,6 @@ org.apache.maven.plugins maven-surefire-plugin - false 1800 ${testRetryCount} From eed86b394399ff5a810e7883d7e3958675f64bb5 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Tue, 30 Apr 2024 09:13:59 +0800 Subject: [PATCH 08/58] fix: Read JSON directly from the original data when formatType=json (#966) (cherry picked from commit 72856b056d1a6caa9011ddf2fd01d580a20d5696) --- .../pulsar/io/jcloud/format/JsonFormat.java | 10 ++- .../pulsar/io/jcloud/bo/TestRecord.java | 1 + .../jcloud/format/JsonFormatMethodTest.java | 66 +++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/apache/pulsar/io/jcloud/format/JsonFormatMethodTest.java diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/JsonFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/JsonFormat.java index e310028d..f56a7e50 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/JsonFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/JsonFormat.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.google.protobuf.DynamicMessage; @@ -128,11 +129,13 @@ public ByteBuffer recordWriterBuf(Iterator> record) throws return ByteBuffer.wrap(stringBuilder.toString().getBytes(StandardCharsets.UTF_8)); } - private Map convertRecordToObject(GenericRecord record, Schema schema) throws IOException { + protected Map convertRecordToObject(GenericRecord record, Schema schema) throws IOException { if (record.getSchemaType().isStruct()) { switch (record.getSchemaType()) { - case AVRO: case JSON: + JsonNode nativeObject = (JsonNode) record.getNativeObject(); + return JSON_MAPPER.get().convertValue(nativeObject, TYPEREF); + case AVRO: case PROTOBUF: { List fields = record.getFields(); Map result = new LinkedHashMap<>(fields.size()); @@ -142,6 +145,9 @@ private Map convertRecordToObject(GenericRecord record, Schema JSON_MAPPER = ThreadLocal.withInitial(() -> { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + return mapper; + }); + + @Test + public void testJsonIgnoreSchemaRead() throws Exception { + + // 1. Gen GenericRecord with TestSubRecord schema but set to incompatible data. + Schema subRecordSchema = Schema.JSON(TestRecord.TestSubRecord.class); + GenericJsonSchema genericRecordSchema = new GenericJsonSchema(subRecordSchema.getSchemaInfo()); + GenericRecordBuilder genericRecordBuilder = genericRecordSchema.newRecordBuilder(); + Map originalData = new HashMap<>(); + originalData.put("incompatibleKey1", "value1"); + originalData.put("incompatibleKey2", 2); + originalData.forEach(genericRecordBuilder::set); + GenericRecord genericRecord = genericRecordBuilder.build(); + + // 3. Will be converted to the same JSON string as the original data. + JsonFormat jsonFormat = new JsonFormat(); + BlobStoreAbstractConfig blobStoreAbstractConfig = new BlobStoreAbstractConfig(); + jsonFormat.configure(blobStoreAbstractConfig); + Map jsonMap = jsonFormat.convertRecordToObject(genericRecord, genericRecordSchema); + String convertedJsonString = JSON_MAPPER.get().writeValueAsString(jsonMap); + String originalJsonString = JSON_MAPPER.get().writeValueAsString(originalData); + Assert.assertEquals(originalJsonString, convertedJsonString); + } +} From 8802a760695a9ec83c172ec21b8c5bb6fcbe8f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 9 May 2024 11:00:17 +0200 Subject: [PATCH 09/58] [fix]: fix Parquet/Avro format with separated key value avro-avro messages (#3) (cherry picked from commit 37a5c85b845cc2be46d718365830884a00b366ae) --- .github/workflows/pr-test.yml | 4 +- pom.xml | 131 +++++++++++++++--- .../pulsar/io/jcloud/format/BytesFormat.java | 7 +- .../io/jcloud/format/ParquetFormat.java | 5 + .../pulsar/io/jcloud/util/AvroRecordUtil.java | 52 ++++++- .../pulsar/io/jcloud/PulsarTestBase.java | 117 +++++++++------- .../io/jcloud/format/FormatTestBase.java | 117 +++++++++++++--- 7 files changed, 332 insertions(+), 101 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 2299ef85..47765203 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -18,10 +18,10 @@ jobs: java-version: 17 - name: Code check - run: mvn license:check checkstyle:check spotbugs:check + run: mvn license:check spotbugs:check - name: Build with Maven - run: mvn install test -DfailIfNoTests=false + run: mvn install test -DfailIfNoTests=false -Dcheckstyle.skip - name: package surefire artifacts if: failure() diff --git a/pom.xml b/pom.xml index 5b1fcbe6..19900099 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 2.9.0-rc-202110221101 + 2.10.1-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -47,8 +47,9 @@ 2.13.2 2.13.4.2 - 1.18.20 - 3.2.2.1 + 1.18.32 + com.datastax.oss + 2.10.4.0 1.11.3 3.3.6 1.13.1 @@ -97,26 +98,33 @@ + + scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git + scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git + https://github.com/datastax/pulsar-io-cloud-storage + HEAD + + - io.streamnative + ${pulsar.groupId} pulsar-io-core ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-io-common ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-common ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-client-original ${pulsar.version} @@ -141,7 +149,7 @@ ${spotbugs-annotations.version} - io.streamnative + ${pulsar.groupId} jclouds-shaded ${pulsar.version} @@ -321,19 +329,19 @@ - io.streamnative + ${pulsar.groupId} pulsar-io-core - io.streamnative + ${pulsar.groupId} pulsar-io-common - io.streamnative + ${pulsar.groupId} pulsar-common - io.streamnative + ${pulsar.groupId} pulsar-client-original @@ -346,7 +354,7 @@ jackson-dataformat-yaml - io.streamnative + ${pulsar.groupId} jclouds-shaded @@ -414,6 +422,34 @@ org.apache.avro avro + 1.10.2 + + + org.apache.hadoop + hadoop-client + 3.3.3 + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-client + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-hdfs-client + + org.apache.hadoop @@ -600,12 +636,40 @@ test - io.streamnative + org.apache.commons + commons-lang3 + 3.11 + + + ${pulsar.groupId} + pulsar-io-common + + + javax.xml.bind + jaxb-api + ${jaxb-api} + + + ${pulsar.groupId} + pulsar-client-original + ${pulsar.version} + + + ${pulsar.groupId} pulsar-client-admin-original ${pulsar.version} + + + net.java.dev.jna + jna + 5.7.0 test - + + ${pulsar.groupId} + pulsar-common + ${pulsar.version} + @@ -617,6 +681,8 @@ ${maven-compiler-plugin.version} UTF-8 + ${java.version} + ${java.version} -Xlint:deprecation @@ -630,7 +696,9 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven-surefire-plugin.version} + false 1800 ${testRetryCount} @@ -826,11 +894,38 @@ central default https://repo1.maven.org/maven2 + + false + + + + confluent + https://packages.confluent.io/maven/ + + false + - bintray-streamnative-maven - bintray - https://dl.bintray.com/streamnative/maven + datastax-releases + https://repo.datastax.com/artifactory/datastax-public-releases-local + + false + + + true + + + + datastax-releases + DataStax Local Releases + https://repo.aws.dsinternal.org/artifactory/datastax-releases-local/ + + + datastax-snapshots-local + DataStax Local Snapshots + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ + + diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java index 8b85b3c8..b6b1e39b 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java @@ -19,6 +19,7 @@ package org.apache.pulsar.io.jcloud.format; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; @@ -35,6 +36,7 @@ */ public class BytesFormat implements Format, InitConfiguration { + public static final byte[] NULL_BYTES = "".getBytes(StandardCharsets.UTF_8); private byte[] lineSeparatorBytes; @Override @@ -62,7 +64,10 @@ public ByteBuffer recordWriterBuf(Iterator> record) throws while (record.hasNext()) { final Record next = record.next(); final Message message = next.getMessage().get(); - final byte[] data = message.getData(); + byte[] data = message.getData(); + if (data == null) { + data = NULL_BYTES; + } dataOutput.write(data); dataOutput.write(lineSeparatorBytes); } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java index 93efd12e..7442b2f8 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java @@ -28,9 +28,11 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.parquet.avro.AvroParquetWriter; @@ -40,8 +42,11 @@ import org.apache.parquet.io.OutputFile; import org.apache.parquet.io.PositionOutputStream; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.protocol.schema.ProtobufNativeSchemaData; +import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java index fea44b88..762d3416 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.io.jcloud.util; -import static org.apache.avro.Schema.Type.ARRAY; -import static org.apache.avro.Schema.Type.ENUM; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import java.lang.reflect.Method; @@ -35,6 +33,8 @@ import org.apache.avro.protobuf.ProtobufData; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.MessageImpl; @@ -43,12 +43,15 @@ import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; +import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.avro.Schema.Type.*; + /** * avro util. @@ -168,8 +171,10 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema final Schema avroSchema = Schema.createRecord("KVSchema", null, null, false, Arrays.asList( - new Schema.Field("key", parseAvroSchema(keySchemaDef)), - new Schema.Field("value", parseAvroSchema(valueSchemaDef)) + // namespace+name must be different in the KVSchema + new Schema.Field("key", parseAvroSchema(keySchemaDef, "_key")), + new Schema.Field("value", Schema.createUnion( + Schema.create(Schema.Type.NULL), parseAvroSchema(valueSchemaDef, "_value"))) )); return avroSchema; } else { @@ -181,14 +186,23 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema if (StringUtils.isEmpty(rootAvroSchemaString)) { throw new IllegalArgumentException("schema definition is empty"); } - return parseAvroSchema(rootAvroSchemaString); + return parseAvroSchema(rootAvroSchemaString, null); } } - private static Schema parseAvroSchema(String jsonSchema) { + private static Schema parseAvroSchema(String jsonSchema, String namespaceSuffix) { final Schema.Parser parser = new Schema.Parser(); parser.setValidateDefaults(false); - return parser.parse(jsonSchema); + Schema schema = parser.parse(jsonSchema); + List fields = schema.getFields() + .stream() + .map(f -> new Schema.Field(f, f.schema())) + .collect(Collectors.toList()); + if (namespaceSuffix == null) { + namespaceSuffix = ""; + } + final String namespace = schema.getNamespace() == null ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; + return Schema.createRecord(schema.getName(), null, namespace, false, fields); } public static org.apache.avro.generic.GenericRecord convertGenericRecord(DynamicMessage recordValue, @@ -215,6 +229,30 @@ public static org.apache.avro.generic.GenericRecord convertGenericRecord(Dynamic public static org.apache.avro.generic.GenericRecord convertGenericRecord(GenericRecord recordValue, Schema rootAvroSchema) { + if (recordValue.getSchemaType() == SchemaType.KEY_VALUE) { + KeyValue keyValue = + (KeyValue) recordValue.getNativeObject(); + org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); + GenericRecord keyObject = keyValue.getKey(); + if (keyObject != null) { + Schema keySchema = rootAvroSchema.getField("key").schema(); + recordHolder.put("key", convertGenericRecord(keyObject, keySchema)); + } + GenericRecord valueObject = keyValue.getValue(); + if (valueObject != null) { + Schema valueSchema = rootAvroSchema.getField("value").schema(); + recordHolder.put("value", convertGenericRecord(valueObject, valueSchema)); + } + return recordHolder; + } + + // handle nullable fields that are union[null,record] + if (rootAvroSchema.isUnion()) { + rootAvroSchema = rootAvroSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.RECORD)) + .findFirst() + .get(); + } org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); for (org.apache.pulsar.client.api.schema.Field field : recordValue.getFields()) { Schema.Field field1 = rootAvroSchema.getField(field.getName()); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index 2048a150..ed782341 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -28,6 +28,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.generic.GenericData; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.pulsar.client.api.Consumer; @@ -37,8 +38,10 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.container.PulsarContainer; import org.junit.AfterClass; @@ -117,7 +120,6 @@ public static List sendTypedMessages( return sendTypedMessages(topic, type, messages, partition, null); } - @SuppressWarnings("unchecked") public static List sendTypedMessages( String topic, SchemaType type, @@ -125,6 +127,66 @@ public static List sendTypedMessages( Optional partition, Class tClass) throws PulsarClientException { + Schema schema; + + switch (type) { + case BOOLEAN: + schema = Schema.BOOL; + break; + case BYTES: + schema = Schema.BYTES; + break; + case DATE: + schema = Schema.DATE; + break; + case STRING: + schema = Schema.STRING; + break; + case TIMESTAMP: + schema = Schema.TIMESTAMP; + break; + case INT8: + schema = Schema.INT8; + break; + case DOUBLE: + schema = Schema.DOUBLE; + break; + case FLOAT: + schema = Schema.FLOAT; + break; + case INT32: + schema = Schema.INT32; + break; + case INT16: + schema = Schema.INT16; + break; + case INT64: + schema = Schema.INT64; + break; + case AVRO: + schema = Schema.AVRO(tClass); + break; + case JSON: + schema = Schema.JSON(tClass); + break; + case KEY_VALUE: + final KeyValue kvMessage = (KeyValue) messages.get(0); + schema = Schema.KeyValue(kvMessage.getKey().getClass(), + kvMessage.getValue().getClass()); + break; + default: + throw new NotImplementedException("Unsupported type " + type); + } + return sendTypedMessages(topic, messages, schema, partition); + + } + @SuppressWarnings("unchecked") + public static List sendTypedMessages( + String topic, + List messages, + Schema produceSchema, + Optional partition) throws PulsarClientException { + String topicName; if (partition.isPresent()) { topicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + partition.get(); @@ -138,56 +200,7 @@ public static List sendTypedMessages( try { client = PulsarClient.builder().serviceUrl(getServiceUrl()).build(); - - switch (type) { - case BOOLEAN: - producer = (Producer) client.newProducer(Schema.BOOL).topic(topicName).create(); - break; - case BYTES: - producer = (Producer) client.newProducer(Schema.BYTES).topic(topicName).create(); - break; - case DATE: - producer = (Producer) client.newProducer(Schema.DATE).topic(topicName).create(); - break; - case STRING: - producer = (Producer) client.newProducer(Schema.STRING).topic(topicName).create(); - break; - case TIMESTAMP: - producer = (Producer) client.newProducer(Schema.TIMESTAMP).topic(topicName).create(); - break; - case INT8: - producer = (Producer) client.newProducer(Schema.INT8).topic(topicName).create(); - break; - case DOUBLE: - producer = (Producer) client.newProducer(Schema.DOUBLE).topic(topicName).create(); - break; - case FLOAT: - producer = (Producer) client.newProducer(Schema.FLOAT).topic(topicName).create(); - break; - case INT32: - producer = (Producer) client.newProducer(Schema.INT32).topic(topicName).create(); - break; - case INT16: - producer = (Producer) client.newProducer(Schema.INT16).topic(topicName).create(); - break; - case INT64: - producer = (Producer) client.newProducer(Schema.INT64).topic(topicName).create(); - break; - case AVRO: - producer = (Producer) client.newProducer(Schema.AVRO(tClass)).topic(topicName).create(); - break; - case JSON: - producer = (Producer) client.newProducer(Schema.JSON(tClass)).topic(topicName).create(); - break; - case KEY_VALUE: - final KeyValue kvMessage = (KeyValue) messages.get(0); - producer = (Producer) client.newProducer(Schema.KeyValue(kvMessage.getKey().getClass(), - kvMessage.getValue().getClass())).topic(topicName).create(); - break; - - default: - throw new NotImplementedException("Unsupported type " + type); - } + producer = client.newProducer(produceSchema).topic(topicName).create(); for (T message : messages) { MessageId mid = producer.send(message); @@ -264,12 +277,14 @@ public static void consumerMessages(String topic, consumer = client.newConsumer(schema) .topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscriptionName("test") .subscribe(); int receiveCount = 0; while (receiveCount < count) { final CompletableFuture> receiveAsync = consumer.receiveAsync(); final Message message = receiveAsync.get(timeout, TimeUnit.MILLISECONDS); + System.out.println("received message " + message); handler.accept(message); consumer.acknowledge(message); receiveCount += 1; diff --git a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java index 035703bd..ecd12044 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java @@ -19,9 +19,15 @@ package org.apache.pulsar.io.jcloud.format; import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Arrays; import java.util.List; @@ -29,6 +35,12 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; + +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.util.Utf8; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -37,10 +49,15 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; import org.apache.pulsar.io.jcloud.PulsarTestBase; @@ -72,6 +89,8 @@ public abstract class FormatTestBase extends PulsarTestBase { TopicName.get("test-json-bytes-parquet-json" + RandomStringUtils.randomAlphabetic(5)); protected static TopicName jsonStringTopicName = TopicName.get("test-json-string-parquet-json" + RandomStringUtils.randomAlphabetic(5)); + private static final TopicName kvSeparatedTopicName = + TopicName.get("test-parquet-kv-sep" + RandomStringUtils.randomAlphabetic(5)); @BeforeClass public static void setUp() throws Exception { @@ -169,6 +188,60 @@ public void testProtobufNativeRecordWriter() throws Exception { consumerMessages(protobufNativeTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); } + @Test + public void testKeyValueAvroWithSameNamespaceName() throws Exception { + org.apache.avro.Schema keySchema = SchemaBuilder.record("record") + .fields() + .name("id").type().stringType().noDefault() + .endRecord(); + org.apache.avro.Schema valueSchema = SchemaBuilder.record("record") + .fields() + .name("content").type().stringType().noDefault() + .endRecord(); + + SchemaDefinition keySchemaDef = SchemaDefinition.builder() + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"id\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) + .build(); + + SchemaDefinition valueSchemaDef = SchemaDefinition.builder() + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"content\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) + .build(); + + + Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); + + GenericData.Record keyRecord = new GenericData.Record(keySchema); + keyRecord.put("id", "theid"); + GenericData.Record valueRecord = new GenericData.Record(valueSchema); + valueRecord.put("content", "ccc"); + List testRecords = Arrays.asList( + new KeyValue(keyRecord, valueRecord), + new KeyValue(keyRecord, null) + ); + + sendTypedMessages(kvSeparatedTopicName.toString(), testRecords, schema, Optional.empty()); + + Consumer> handle = getMessageConsumer(kvSeparatedTopicName); + consumerMessages(kvSeparatedTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); + } + + protected abstract boolean supportMetadata(); protected Consumer> getMessageConsumer(TopicName topic) { @@ -224,11 +297,11 @@ public abstract org.apache.avro.generic.GenericRecord getFormatGeneratedRecord(T throws Exception; public abstract DynamicMessage getDynamicMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; public abstract Map getJSONMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.GenericRecord record) { @@ -237,16 +310,16 @@ protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.Gen Object sourceValue = msgValue.getField(descriptor.findFieldByName(field.name())); Object newValue = record.get(field.name()); Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - field.name(), - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + field.name(), + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } @@ -382,16 +455,16 @@ protected void assertEquals(DynamicMessage msgValue, Map record) Object newValue = record.get(fieldName); if (!fieldDescriptor.isRepeated()) { Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - fieldName, - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + fieldName, + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } } From 2696f0dbcab2f11b76f15257510fe5e34120111c Mon Sep 17 00:00:00 2001 From: Mukesh Kumar <65598381+mukesh154@users.noreply.github.com> Date: Wed, 22 May 2024 17:00:01 +0530 Subject: [PATCH 10/58] [improve] Allow user-specified S3 path (#5) * Added user-specified S3 path * Optimized the implementation * Variables renamed * Logic moved to Partitioner & Added unit tests with all the flags about the partitions with all the combinations * Removed unnecessary checks (cherry picked from commit 2a66206bca2345d1296f12d98bf3d946f9f0f4a2) --- .../io/jcloud/BlobStoreAbstractConfig.java | 21 ++ .../legacy/AbstractPartitioner.java | 62 +++-- .../pulsar/io/jcloud/ConnectorConfigTest.java | 73 +++++ .../TopicsToPathMappingPartitionerTest.java | 255 ++++++++++++++++++ 4 files changed, 396 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java diff --git a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java index e1ce1518..fcda4f42 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java @@ -119,6 +119,7 @@ public class BlobStoreAbstractConfig implements Serializable { private String bytesFormatTypeSeparator = "0x10"; private boolean skipFailedMessages = false; private boolean jsonAllowNaN = false; + private String topicsToPathMapping; public void validate() { checkNotNull(provider, "provider not set."); @@ -194,6 +195,10 @@ public void validate() { if (parquetCodec != null && (parquetCodec.isEmpty() || parquetCodec.equals("none"))) { parquetCodec = null; } + + if (StringUtils.isNotEmpty(topicsToPathMapping)) { + validateTopicsToPathMapping(); + } } private static boolean hasURIScheme(String endpoint) { @@ -205,4 +210,20 @@ private static boolean hasURIScheme(String endpoint) { } } + public void validateTopicsToPathMapping() { + for (String topicToPathMapping : topicsToPathMapping.split(",")) { + String[] topicPathPair = topicToPathMapping.split("="); + checkArgument(topicPathPair.length == 2, + "Invalid topicsToPathMapping format: " + topicToPathMapping); + String topic = topicPathPair[0].trim(); + String path = topicPathPair[1].trim(); + checkArgument(isNotBlank(topic), "Topic in topicsToPathMapping cannot be empty."); + checkArgument(isNotBlank(path), "Path in topicsToPathMapping cannot be empty."); + checkArgument(topic.matches("^[^/]+/[^/]+/[^/]+$"), + "Topic must be in the format 'tenant/ns/topic': " + topic); + checkArgument(!path.startsWith("/") && !path.endsWith("/"), + "Path in topicsToPathMapping cannot starts or ends with '/': " + path); + } + } + } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java index 71e2d3af..1dfedd2e 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; @@ -42,35 +44,65 @@ public abstract class AbstractPartitioner implements Partitioner { private boolean withTopicPartitionNumber; private boolean useIndexAsOffset; + private Map topicsToPathMapping; + private Map computedTopicsToPathMapping; + @Override public void configure(BlobStoreAbstractConfig config) { this.sliceTopicPartitionPath = config.isSliceTopicPartitionPath(); this.withTopicPartitionNumber = config.isWithTopicPartitionNumber(); this.useIndexAsOffset = config.isPartitionerUseIndexAsOffset(); + this.topicsToPathMapping = parseTopicsToPathMappingString(config.getTopicsToPathMapping()); + this.computedTopicsToPathMapping = new HashMap<>(); + } + + private Map parseTopicsToPathMappingString(String topicsToPathMappingString) { + Map topicsToPathMapping = new HashMap<>(); + if (StringUtils.isNotEmpty(topicsToPathMappingString)) { + for (String topicPathPair : topicsToPathMappingString.split(",")) { + String[] topicToPathMapping = topicPathPair.split("="); + if (topicToPathMapping.length == 2) { + topicsToPathMapping.put(topicToPathMapping[0], topicToPathMapping[1]); + } else { + LOGGER.warn("The topic to path mapping is not in correct format {}.", topicPathPair); + } + } + } + LOGGER.info("The final topics to path mapping is: {}", topicsToPathMapping); + return topicsToPathMapping; } @Override public String generatePartitionedPath(String topic, String encodedPartition) { + if (computedTopicsToPathMapping.containsKey(topic)) { + return String.format("%s%s%s", computedTopicsToPathMapping.get(topic), PATH_SEPARATOR, encodedPartition); + } else { + List joinList = new ArrayList<>(); + TopicName topicName = TopicName.get(topic); + joinList.add(topicName.getTenant()); + joinList.add(topicName.getNamespacePortion()); - List joinList = new ArrayList<>(); - TopicName topicName = TopicName.get(topic); - joinList.add(topicName.getTenant()); - joinList.add(topicName.getNamespacePortion()); - - if (topicName.isPartitioned() && withTopicPartitionNumber) { - if (sliceTopicPartitionPath) { + if (topicName.isPartitioned() && withTopicPartitionNumber) { + if (sliceTopicPartitionPath) { + TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); + joinList.add(newTopicName.getLocalName()); + joinList.add(Integer.toString(topicName.getPartitionIndex())); + } else { + joinList.add(topicName.getLocalName()); + } + } else { TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); joinList.add(newTopicName.getLocalName()); - joinList.add(Integer.toString(topicName.getPartitionIndex())); - } else { - joinList.add(topicName.getLocalName()); } - } else { - TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); - joinList.add(newTopicName.getLocalName()); + String generatedTopicPath = StringUtils.join(joinList, PATH_SEPARATOR); + if (topicsToPathMapping.containsKey(generatedTopicPath)) { + joinList.clear(); + joinList.add(topicsToPathMapping.get(generatedTopicPath)); + } + computedTopicsToPathMapping.put(topic, StringUtils.join(joinList, PATH_SEPARATOR)); + joinList.add(encodedPartition); + return StringUtils.join(joinList, PATH_SEPARATOR); } - joinList.add(encodedPartition); - return StringUtils.join(joinList, PATH_SEPARATOR); } protected long getMessageOffset(Record record) { diff --git a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java index ac55f914..e7e6ca8b 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java @@ -52,6 +52,7 @@ public void loadBasicConfigTest() throws IOException { config.put("timePartitionDuration", "2d"); config.put("batchSize", 10); config.put("partitioner", "topic"); + config.put("topicsToPathMapping", "my-tenant/my-ns/topic1=path-mytopic1,my-tenant/my-ns/topic2=path-mytopic2"); CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config); cloudStorageSinkConfig.validate(); @@ -69,6 +70,7 @@ public void loadBasicConfigTest() throws IOException { cloudStorageSinkConfig.getPartitioner().toString().toLowerCase()); Assert.assertEquals((int) config.get("batchSize"), cloudStorageSinkConfig.getPendingQueueSize()); Assert.assertEquals(10000000L, cloudStorageSinkConfig.getMaxBatchBytes()); + Assert.assertEquals(config.get("topicsToPathMapping"), cloudStorageSinkConfig.getTopicsToPathMapping()); } @Test @@ -131,6 +133,77 @@ public void timePartitionDurationTest() throws IOException { } } + @Test + public void topicsToPathMappingTest() throws IOException { + Map config = new HashMap<>(); + config.put("provider", PROVIDER_AWSS3); + config.put("accessKeyId", "aws-s3"); + config.put("secretAccessKey", "aws-s3"); + config.put("bucket", "testbucket"); + config.put("region", "localhost"); + config.put("endpoint", "https://us-standard"); + config.put("formatType", "avro"); + config.put("partitionerType", "default"); + config.put("timePartitionPattern", "yyyy-MM-dd"); + config.put("timePartitionDuration", "2d"); + config.put("batchSize", 10); + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2=path-mytopic2"); + CloudStorageSinkConfig.load(config).validate(); + } catch (Exception e) { + Assert.fail(); + } + + try { + config.put("topicsToPathMapping", "topic1=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2="); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,=path-mytopic2"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=/path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1/"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + } + @Test public void pathPrefixTest() throws IOException { Map config = new HashMap<>(); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java new file mode 100644 index 00000000..2e214873 --- /dev/null +++ b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.jcloud.partitioner; + +import com.google.common.base.Supplier; +import junit.framework.TestCase; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.text.MessageFormat; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * partitioner unit test. + */ +@RunWith(Parameterized.class) +public class TopicsToPathMappingPartitionerTest extends TestCase { + + @Parameterized.Parameter(0) + public Partitioner partitioner; + + @Parameterized.Parameter(1) + public String expected; + + @Parameterized.Parameter(2) + public String expectedPartitionedPath; + + @Parameterized.Parameter(3) + public Record pulsarRecord; + + @Parameterized.Parameters + public static Object[][] data() { + BlobStoreAbstractConfig blobStoreAbstractConfig1 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig1.setTimePartitionDuration("1d"); + blobStoreAbstractConfig1.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig1.setSliceTopicPartitionPath(false); + blobStoreAbstractConfig1.setWithTopicPartitionNumber(false); + blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner1 = new SimplePartitioner<>(); + simplePartitioner1.configure(blobStoreAbstractConfig1); + TimePartitioner dayPartitioner1 = new TimePartitioner<>(); + dayPartitioner1.configure(blobStoreAbstractConfig1); + + BlobStoreAbstractConfig blobStoreAbstractConfig2 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig2.setTimePartitionDuration("1d"); + blobStoreAbstractConfig2.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig2.setSliceTopicPartitionPath(false); + blobStoreAbstractConfig2.setWithTopicPartitionNumber(true); + blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner2 = new SimplePartitioner<>(); + simplePartitioner2.configure(blobStoreAbstractConfig2); + TimePartitioner dayPartitioner2 = new TimePartitioner<>(); + dayPartitioner2.configure(blobStoreAbstractConfig2); + + BlobStoreAbstractConfig blobStoreAbstractConfig3 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig3.setTimePartitionDuration("1d"); + blobStoreAbstractConfig3.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig3.setSliceTopicPartitionPath(true); + blobStoreAbstractConfig3.setWithTopicPartitionNumber(false); + blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner3 = new SimplePartitioner<>(); + simplePartitioner3.configure(blobStoreAbstractConfig3); + TimePartitioner dayPartitioner3 = new TimePartitioner<>(); + dayPartitioner3.configure(blobStoreAbstractConfig3); + + BlobStoreAbstractConfig blobStoreAbstractConfig4 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig4.setTimePartitionDuration("1d"); + blobStoreAbstractConfig4.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig4.setSliceTopicPartitionPath(true); + blobStoreAbstractConfig4.setWithTopicPartitionNumber(true); + blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner4 = new SimplePartitioner<>(); + simplePartitioner4.configure(blobStoreAbstractConfig4); + TimePartitioner dayPartitioner4 = new TimePartitioner<>(); + dayPartitioner4.configure(blobStoreAbstractConfig4); + + return new Object[][]{ + new Object[]{ + simplePartitioner1, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner1, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner1, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner1, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner2, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner2, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner2, + "3221225506", + "path2" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner2, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path2/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner3, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner3, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner3, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner3, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner4, + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner4, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner4, + "3221225506", + "path3" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner4, + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path3/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + getPartitionedTopic() + }, + }; + } + + public static Record getPartitionedTopic() { + @SuppressWarnings("unchecked") + Message mock = mock(Message.class); + when(mock.getPublishTime()).thenReturn(1599578218610L); + when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); + String topic = TopicName.get("test-partition-1").toString(); + Record mockRecord = mock(Record.class); + when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); + when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); + when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); + when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); + when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); + return mockRecord; + } + + public static Record getTopic() { + @SuppressWarnings("unchecked") + Message mock = mock(Message.class); + when(mock.getPublishTime()).thenReturn(1599578218610L); + when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); + String topic = TopicName.get("test").toString(); + Record mockRecord = mock(Record.class); + when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); + when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); + when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); + when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); + when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); + return mockRecord; + } + + @Test + public void testEncodePartition() { + String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); + Supplier supplier = + () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); + Assert.assertEquals(supplier.get(), expected, encodePartition); + } + + @Test + public void testGeneratePartitionedPath() { + String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); + String partitionedPath = + partitioner.generatePartitionedPath(pulsarRecord.getTopicName().get(), encodePartition); + + Supplier supplier = + () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); + Assert.assertEquals(supplier.get(), expectedPartitionedPath, partitionedPath); + } +} From 21c3ad3b7c40bb8c48cac87b888a59757d39c4a7 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 13:00:11 +0530 Subject: [PATCH 11/58] Fix checkstyle --- .github/workflows/pr-test.yml | 2 +- pom.xml | 4 +- .../io/jcloud/format/ParquetFormat.java | 5 -- .../legacy/AbstractPartitioner.java | 2 +- .../pulsar/io/jcloud/util/AvroRecordUtil.java | 9 +-- .../pulsar/io/jcloud/PulsarTestBase.java | 2 - .../io/jcloud/format/FormatTestBase.java | 52 +++++------- .../TopicsToPathMappingPartitionerTest.java | 79 ++++++++++--------- 8 files changed, 70 insertions(+), 85 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 47765203..a01940d9 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - 3.2_ds jobs: build: diff --git a/pom.xml b/pom.xml index 19900099..f4e9204d 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 2.10.1-SNAPSHOT + 3.2.0-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -49,7 +49,7 @@ 2.13.4.2 1.18.32 com.datastax.oss - 2.10.4.0 + 3.1.4.2 1.11.3 3.3.6 1.13.1 diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java index 7442b2f8..93efd12e 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java @@ -28,11 +28,9 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.parquet.avro.AvroParquetWriter; @@ -42,11 +40,8 @@ import org.apache.parquet.io.OutputFile; import org.apache.parquet.io.PositionOutputStream; import org.apache.pulsar.client.api.SchemaSerializationException; -import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.protocol.schema.ProtobufNativeSchemaData; -import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java index 1dfedd2e..1ccb5620 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java @@ -19,8 +19,8 @@ package org.apache.pulsar.io.jcloud.partitioner.legacy; import java.util.ArrayList; -import java.util.List; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java index 762d3416..47949271 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.io.jcloud.util; +import static org.apache.avro.Schema.Type.ARRAY; +import static org.apache.avro.Schema.Type.ENUM; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import java.lang.reflect.Method; @@ -33,8 +35,6 @@ import org.apache.avro.protobuf.ProtobufData; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.schema.Field; -import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.MessageImpl; @@ -50,8 +50,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.avro.Schema.Type.*; - /** * avro util. @@ -201,7 +199,8 @@ private static Schema parseAvroSchema(String jsonSchema, String namespaceSuffix) if (namespaceSuffix == null) { namespaceSuffix = ""; } - final String namespace = schema.getNamespace() == null ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; + final String namespace = schema.getNamespace() == null + ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; return Schema.createRecord(schema.getName(), null, namespace, false, fields); } diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index ed782341..d3c1ee12 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -28,7 +28,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.avro.generic.GenericData; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.pulsar.client.api.Consumer; @@ -41,7 +40,6 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.container.PulsarContainer; import org.junit.AfterClass; diff --git a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java index ecd12044..d9621d13 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java @@ -19,15 +19,9 @@ package org.apache.pulsar.io.jcloud.format; import static org.junit.Assert.fail; -import static org.mockito.Mockito.when; - import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Arrays; import java.util.List; @@ -35,12 +29,8 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; - import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.util.Utf8; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -50,14 +40,11 @@ import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; import org.apache.pulsar.io.jcloud.PulsarTestBase; @@ -200,31 +187,32 @@ public void testKeyValueAvroWithSameNamespaceName() throws Exception { .endRecord(); SchemaDefinition keySchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" + - "{\n" + - " \t\"type\": \"record\",\n" + - " \t\"name\": \"record\",\n" + - " \t\"fields\": [{\n" + - " \t\t\"name\": \"id\",\n" + - " \t\t\"type\": [\"string\"]\n" + - " \t}]\n" + - " }")) + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"id\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) .build(); SchemaDefinition valueSchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" + - "{\n" + - " \t\"type\": \"record\",\n" + - " \t\"name\": \"record\",\n" + - " \t\"fields\": [{\n" + - " \t\t\"name\": \"content\",\n" + - " \t\t\"type\": [\"string\"]\n" + - " \t}]\n" + - " }")) + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"content\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) .build(); - Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); + Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), + Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); GenericData.Record keyRecord = new GenericData.Record(keySchema); keyRecord.put("id", "theid"); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java index 2e214873..2044a655 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java @@ -18,23 +18,26 @@ */ package org.apache.pulsar.io.jcloud.partitioner; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.base.Supplier; +import java.io.File; +import java.text.MessageFormat; +import java.util.Optional; import junit.framework.TestCase; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; +import org.apache.pulsar.io.jcloud.partitioner.legacy.Partitioner; +import org.apache.pulsar.io.jcloud.partitioner.legacy.SimplePartitioner; +import org.apache.pulsar.io.jcloud.partitioner.legacy.TimePartitioner; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.text.MessageFormat; -import java.util.Optional; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * partitioner unit test. @@ -42,6 +45,8 @@ @RunWith(Parameterized.class) public class TopicsToPathMappingPartitionerTest extends TestCase { + private static final String pathSeparator = File.separator; + @Parameterized.Parameter(0) public Partitioner partitioner; @@ -61,8 +66,8 @@ public static Object[][] data() { blobStoreAbstractConfig1.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig1.setSliceTopicPartitionPath(false); blobStoreAbstractConfig1.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner1 = new SimplePartitioner<>(); simplePartitioner1.configure(blobStoreAbstractConfig1); TimePartitioner dayPartitioner1 = new TimePartitioner<>(); @@ -73,8 +78,8 @@ public static Object[][] data() { blobStoreAbstractConfig2.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig2.setSliceTopicPartitionPath(false); blobStoreAbstractConfig2.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner2 = new SimplePartitioner<>(); simplePartitioner2.configure(blobStoreAbstractConfig2); TimePartitioner dayPartitioner2 = new TimePartitioner<>(); @@ -85,8 +90,8 @@ public static Object[][] data() { blobStoreAbstractConfig3.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig3.setSliceTopicPartitionPath(true); blobStoreAbstractConfig3.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner3 = new SimplePartitioner<>(); simplePartitioner3.configure(blobStoreAbstractConfig3); TimePartitioner dayPartitioner3 = new TimePartitioner<>(); @@ -97,8 +102,8 @@ public static Object[][] data() { blobStoreAbstractConfig4.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig4.setSliceTopicPartitionPath(true); blobStoreAbstractConfig4.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner4 = new SimplePartitioner<>(); simplePartitioner4.configure(blobStoreAbstractConfig4); TimePartitioner dayPartitioner4 = new TimePartitioner<>(); @@ -108,97 +113,97 @@ public static Object[][] data() { new Object[]{ simplePartitioner1, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getTopic() }, new Object[]{ dayPartitioner1, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getTopic() }, new Object[]{ simplePartitioner1, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner1, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner2, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getTopic() }, new Object[]{ dayPartitioner2, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getTopic() }, new Object[]{ simplePartitioner2, "3221225506", - "path2" + Partitioner.PATH_SEPARATOR + "3221225506", + "path2" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner2, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path2/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path2/2020-09-08" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner3, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getTopic() }, new Object[]{ dayPartitioner3, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getTopic() }, new Object[]{ simplePartitioner3, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner3, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner4, "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1" + pathSeparator + "3221225506", getTopic() }, new Object[]{ dayPartitioner4, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", getTopic() }, new Object[]{ simplePartitioner4, "3221225506", - "path3" + Partitioner.PATH_SEPARATOR + "3221225506", + "path3" + pathSeparator + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner4, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path3/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "2020-09-08" + pathSeparator + "3221225506", + "path3/2020-09-08" + pathSeparator + "3221225506", getPartitionedTopic() }, }; From 51b566a7b673096fd36c82a11b0b6f1361dcc004 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 13:03:54 +0530 Subject: [PATCH 12/58] Revert "Fix checkstyle" This reverts commit 21c3ad3b7c40bb8c48cac87b888a59757d39c4a7. --- .github/workflows/pr-test.yml | 2 +- pom.xml | 4 +- .../io/jcloud/format/ParquetFormat.java | 5 ++ .../legacy/AbstractPartitioner.java | 2 +- .../pulsar/io/jcloud/util/AvroRecordUtil.java | 9 ++- .../pulsar/io/jcloud/PulsarTestBase.java | 2 + .../io/jcloud/format/FormatTestBase.java | 52 +++++++----- .../TopicsToPathMappingPartitionerTest.java | 79 +++++++++---------- 8 files changed, 85 insertions(+), 70 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index a01940d9..47765203 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - 3.2_ds + - master jobs: build: diff --git a/pom.xml b/pom.xml index f4e9204d..19900099 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0-SNAPSHOT + 2.10.1-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -49,7 +49,7 @@ 2.13.4.2 1.18.32 com.datastax.oss - 3.1.4.2 + 2.10.4.0 1.11.3 3.3.6 1.13.1 diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java index 93efd12e..7442b2f8 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java @@ -28,9 +28,11 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.parquet.avro.AvroParquetWriter; @@ -40,8 +42,11 @@ import org.apache.parquet.io.OutputFile; import org.apache.parquet.io.PositionOutputStream; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.protocol.schema.ProtobufNativeSchemaData; +import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java index 1ccb5620..1dfedd2e 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java @@ -19,8 +19,8 @@ package org.apache.pulsar.io.jcloud.partitioner.legacy; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java index 47949271..762d3416 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.io.jcloud.util; -import static org.apache.avro.Schema.Type.ARRAY; -import static org.apache.avro.Schema.Type.ENUM; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import java.lang.reflect.Method; @@ -35,6 +33,8 @@ import org.apache.avro.protobuf.ProtobufData; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.MessageImpl; @@ -50,6 +50,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.avro.Schema.Type.*; + /** * avro util. @@ -199,8 +201,7 @@ private static Schema parseAvroSchema(String jsonSchema, String namespaceSuffix) if (namespaceSuffix == null) { namespaceSuffix = ""; } - final String namespace = schema.getNamespace() == null - ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; + final String namespace = schema.getNamespace() == null ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; return Schema.createRecord(schema.getName(), null, namespace, false, fields); } diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index d3c1ee12..ed782341 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -28,6 +28,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.generic.GenericData; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.pulsar.client.api.Consumer; @@ -40,6 +41,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.container.PulsarContainer; import org.junit.AfterClass; diff --git a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java index d9621d13..ecd12044 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java @@ -19,9 +19,15 @@ package org.apache.pulsar.io.jcloud.format; import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Arrays; import java.util.List; @@ -29,8 +35,12 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; + import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.util.Utf8; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -40,11 +50,14 @@ import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; import org.apache.pulsar.io.jcloud.PulsarTestBase; @@ -187,32 +200,31 @@ public void testKeyValueAvroWithSameNamespaceName() throws Exception { .endRecord(); SchemaDefinition keySchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" - + "{\n" - + " \t\"type\": \"record\",\n" - + " \t\"name\": \"record\",\n" - + " \t\"fields\": [{\n" - + " \t\t\"name\": \"id\",\n" - + " \t\t\"type\": [\"string\"]\n" - + " \t}]\n" - + " }")) + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"id\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) .build(); SchemaDefinition valueSchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" - + "{\n" - + " \t\"type\": \"record\",\n" - + " \t\"name\": \"record\",\n" - + " \t\"fields\": [{\n" - + " \t\t\"name\": \"content\",\n" - + " \t\t\"type\": [\"string\"]\n" - + " \t}]\n" - + " }")) + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"content\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) .build(); - Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), - Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); + Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); GenericData.Record keyRecord = new GenericData.Record(keySchema); keyRecord.put("id", "theid"); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java index 2044a655..2e214873 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java @@ -18,26 +18,23 @@ */ package org.apache.pulsar.io.jcloud.partitioner; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import com.google.common.base.Supplier; -import java.io.File; -import java.text.MessageFormat; -import java.util.Optional; import junit.framework.TestCase; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; -import org.apache.pulsar.io.jcloud.partitioner.legacy.Partitioner; -import org.apache.pulsar.io.jcloud.partitioner.legacy.SimplePartitioner; -import org.apache.pulsar.io.jcloud.partitioner.legacy.TimePartitioner; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.text.MessageFormat; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * partitioner unit test. @@ -45,8 +42,6 @@ @RunWith(Parameterized.class) public class TopicsToPathMappingPartitionerTest extends TestCase { - private static final String pathSeparator = File.separator; - @Parameterized.Parameter(0) public Partitioner partitioner; @@ -66,8 +61,8 @@ public static Object[][] data() { blobStoreAbstractConfig1.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig1.setSliceTopicPartitionPath(false); blobStoreAbstractConfig1.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," - + "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner1 = new SimplePartitioner<>(); simplePartitioner1.configure(blobStoreAbstractConfig1); TimePartitioner dayPartitioner1 = new TimePartitioner<>(); @@ -78,8 +73,8 @@ public static Object[][] data() { blobStoreAbstractConfig2.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig2.setSliceTopicPartitionPath(false); blobStoreAbstractConfig2.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," - + "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner2 = new SimplePartitioner<>(); simplePartitioner2.configure(blobStoreAbstractConfig2); TimePartitioner dayPartitioner2 = new TimePartitioner<>(); @@ -90,8 +85,8 @@ public static Object[][] data() { blobStoreAbstractConfig3.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig3.setSliceTopicPartitionPath(true); blobStoreAbstractConfig3.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," - + "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner3 = new SimplePartitioner<>(); simplePartitioner3.configure(blobStoreAbstractConfig3); TimePartitioner dayPartitioner3 = new TimePartitioner<>(); @@ -102,8 +97,8 @@ public static Object[][] data() { blobStoreAbstractConfig4.setTimePartitionPattern("yyyy-MM-dd"); blobStoreAbstractConfig4.setSliceTopicPartitionPath(true); blobStoreAbstractConfig4.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," - + "public/default/test-partition-1=path2,public/default/test/1=path3"); + blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); SimplePartitioner simplePartitioner4 = new SimplePartitioner<>(); simplePartitioner4.configure(blobStoreAbstractConfig4); TimePartitioner dayPartitioner4 = new TimePartitioner<>(); @@ -113,97 +108,97 @@ public static Object[][] data() { new Object[]{ simplePartitioner1, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ dayPartitioner1, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ simplePartitioner1, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner1, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner2, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ dayPartitioner2, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ simplePartitioner2, "3221225506", - "path2" + pathSeparator + "3221225506", + "path2" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner2, - "2020-09-08" + pathSeparator + "3221225506", - "path2/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path2/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner3, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ dayPartitioner3, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ simplePartitioner3, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner3, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ simplePartitioner4, "3221225506", - "path1" + pathSeparator + "3221225506", + "path1" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ dayPartitioner4, - "2020-09-08" + pathSeparator + "3221225506", - "path1/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getTopic() }, new Object[]{ simplePartitioner4, "3221225506", - "path3" + pathSeparator + "3221225506", + "path3" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, new Object[]{ dayPartitioner4, - "2020-09-08" + pathSeparator + "3221225506", - "path3/2020-09-08" + pathSeparator + "3221225506", + "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", + "path3/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", getPartitionedTopic() }, }; From fc48d5902ddd9d53a8c77dd838015f8f352f0e15 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 13:04:11 +0530 Subject: [PATCH 13/58] Revert "[improve] Allow user-specified S3 path (#5)" This reverts commit 2696f0dbcab2f11b76f15257510fe5e34120111c. --- .../io/jcloud/BlobStoreAbstractConfig.java | 21 -- .../legacy/AbstractPartitioner.java | 62 ++--- .../pulsar/io/jcloud/ConnectorConfigTest.java | 73 ----- .../TopicsToPathMappingPartitionerTest.java | 255 ------------------ 4 files changed, 15 insertions(+), 396 deletions(-) delete mode 100644 src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java diff --git a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java index fcda4f42..e1ce1518 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java @@ -119,7 +119,6 @@ public class BlobStoreAbstractConfig implements Serializable { private String bytesFormatTypeSeparator = "0x10"; private boolean skipFailedMessages = false; private boolean jsonAllowNaN = false; - private String topicsToPathMapping; public void validate() { checkNotNull(provider, "provider not set."); @@ -195,10 +194,6 @@ public void validate() { if (parquetCodec != null && (parquetCodec.isEmpty() || parquetCodec.equals("none"))) { parquetCodec = null; } - - if (StringUtils.isNotEmpty(topicsToPathMapping)) { - validateTopicsToPathMapping(); - } } private static boolean hasURIScheme(String endpoint) { @@ -210,20 +205,4 @@ private static boolean hasURIScheme(String endpoint) { } } - public void validateTopicsToPathMapping() { - for (String topicToPathMapping : topicsToPathMapping.split(",")) { - String[] topicPathPair = topicToPathMapping.split("="); - checkArgument(topicPathPair.length == 2, - "Invalid topicsToPathMapping format: " + topicToPathMapping); - String topic = topicPathPair[0].trim(); - String path = topicPathPair[1].trim(); - checkArgument(isNotBlank(topic), "Topic in topicsToPathMapping cannot be empty."); - checkArgument(isNotBlank(path), "Path in topicsToPathMapping cannot be empty."); - checkArgument(topic.matches("^[^/]+/[^/]+/[^/]+$"), - "Topic must be in the format 'tenant/ns/topic': " + topic); - checkArgument(!path.startsWith("/") && !path.endsWith("/"), - "Path in topicsToPathMapping cannot starts or ends with '/': " + path); - } - } - } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java index 1dfedd2e..71e2d3af 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java @@ -20,8 +20,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; @@ -44,65 +42,35 @@ public abstract class AbstractPartitioner implements Partitioner { private boolean withTopicPartitionNumber; private boolean useIndexAsOffset; - private Map topicsToPathMapping; - private Map computedTopicsToPathMapping; - @Override public void configure(BlobStoreAbstractConfig config) { this.sliceTopicPartitionPath = config.isSliceTopicPartitionPath(); this.withTopicPartitionNumber = config.isWithTopicPartitionNumber(); this.useIndexAsOffset = config.isPartitionerUseIndexAsOffset(); - this.topicsToPathMapping = parseTopicsToPathMappingString(config.getTopicsToPathMapping()); - this.computedTopicsToPathMapping = new HashMap<>(); - } - - private Map parseTopicsToPathMappingString(String topicsToPathMappingString) { - Map topicsToPathMapping = new HashMap<>(); - if (StringUtils.isNotEmpty(topicsToPathMappingString)) { - for (String topicPathPair : topicsToPathMappingString.split(",")) { - String[] topicToPathMapping = topicPathPair.split("="); - if (topicToPathMapping.length == 2) { - topicsToPathMapping.put(topicToPathMapping[0], topicToPathMapping[1]); - } else { - LOGGER.warn("The topic to path mapping is not in correct format {}.", topicPathPair); - } - } - } - LOGGER.info("The final topics to path mapping is: {}", topicsToPathMapping); - return topicsToPathMapping; } @Override public String generatePartitionedPath(String topic, String encodedPartition) { - if (computedTopicsToPathMapping.containsKey(topic)) { - return String.format("%s%s%s", computedTopicsToPathMapping.get(topic), PATH_SEPARATOR, encodedPartition); - } else { - List joinList = new ArrayList<>(); - TopicName topicName = TopicName.get(topic); - joinList.add(topicName.getTenant()); - joinList.add(topicName.getNamespacePortion()); - if (topicName.isPartitioned() && withTopicPartitionNumber) { - if (sliceTopicPartitionPath) { - TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); - joinList.add(newTopicName.getLocalName()); - joinList.add(Integer.toString(topicName.getPartitionIndex())); - } else { - joinList.add(topicName.getLocalName()); - } - } else { + List joinList = new ArrayList<>(); + TopicName topicName = TopicName.get(topic); + joinList.add(topicName.getTenant()); + joinList.add(topicName.getNamespacePortion()); + + if (topicName.isPartitioned() && withTopicPartitionNumber) { + if (sliceTopicPartitionPath) { TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); joinList.add(newTopicName.getLocalName()); + joinList.add(Integer.toString(topicName.getPartitionIndex())); + } else { + joinList.add(topicName.getLocalName()); } - String generatedTopicPath = StringUtils.join(joinList, PATH_SEPARATOR); - if (topicsToPathMapping.containsKey(generatedTopicPath)) { - joinList.clear(); - joinList.add(topicsToPathMapping.get(generatedTopicPath)); - } - computedTopicsToPathMapping.put(topic, StringUtils.join(joinList, PATH_SEPARATOR)); - joinList.add(encodedPartition); - return StringUtils.join(joinList, PATH_SEPARATOR); + } else { + TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); + joinList.add(newTopicName.getLocalName()); } + joinList.add(encodedPartition); + return StringUtils.join(joinList, PATH_SEPARATOR); } protected long getMessageOffset(Record record) { diff --git a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java index e7e6ca8b..ac55f914 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java @@ -52,7 +52,6 @@ public void loadBasicConfigTest() throws IOException { config.put("timePartitionDuration", "2d"); config.put("batchSize", 10); config.put("partitioner", "topic"); - config.put("topicsToPathMapping", "my-tenant/my-ns/topic1=path-mytopic1,my-tenant/my-ns/topic2=path-mytopic2"); CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config); cloudStorageSinkConfig.validate(); @@ -70,7 +69,6 @@ public void loadBasicConfigTest() throws IOException { cloudStorageSinkConfig.getPartitioner().toString().toLowerCase()); Assert.assertEquals((int) config.get("batchSize"), cloudStorageSinkConfig.getPendingQueueSize()); Assert.assertEquals(10000000L, cloudStorageSinkConfig.getMaxBatchBytes()); - Assert.assertEquals(config.get("topicsToPathMapping"), cloudStorageSinkConfig.getTopicsToPathMapping()); } @Test @@ -133,77 +131,6 @@ public void timePartitionDurationTest() throws IOException { } } - @Test - public void topicsToPathMappingTest() throws IOException { - Map config = new HashMap<>(); - config.put("provider", PROVIDER_AWSS3); - config.put("accessKeyId", "aws-s3"); - config.put("secretAccessKey", "aws-s3"); - config.put("bucket", "testbucket"); - config.put("region", "localhost"); - config.put("endpoint", "https://us-standard"); - config.put("formatType", "avro"); - config.put("partitionerType", "default"); - config.put("timePartitionPattern", "yyyy-MM-dd"); - config.put("timePartitionDuration", "2d"); - config.put("batchSize", 10); - try { - config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2=path-mytopic2"); - CloudStorageSinkConfig.load(config).validate(); - } catch (Exception e) { - Assert.fail(); - } - - try { - config.put("topicsToPathMapping", "topic1=path-mytopic1"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/=path-mytopic1"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/ns1=path-mytopic1"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2="); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,=path-mytopic2"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/ns1/topic1=/path-mytopic1"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - - try { - config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1/"); - CloudStorageSinkConfig.load(config).validate(); - Assert.fail(); - } catch (Exception e) { - } - } - @Test public void pathPrefixTest() throws IOException { Map config = new HashMap<>(); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java deleted file mode 100644 index 2e214873..00000000 --- a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.io.jcloud.partitioner; - -import com.google.common.base.Supplier; -import junit.framework.TestCase; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.functions.api.Record; -import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.text.MessageFormat; -import java.util.Optional; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * partitioner unit test. - */ -@RunWith(Parameterized.class) -public class TopicsToPathMappingPartitionerTest extends TestCase { - - @Parameterized.Parameter(0) - public Partitioner partitioner; - - @Parameterized.Parameter(1) - public String expected; - - @Parameterized.Parameter(2) - public String expectedPartitionedPath; - - @Parameterized.Parameter(3) - public Record pulsarRecord; - - @Parameterized.Parameters - public static Object[][] data() { - BlobStoreAbstractConfig blobStoreAbstractConfig1 = new BlobStoreAbstractConfig(); - blobStoreAbstractConfig1.setTimePartitionDuration("1d"); - blobStoreAbstractConfig1.setTimePartitionPattern("yyyy-MM-dd"); - blobStoreAbstractConfig1.setSliceTopicPartitionPath(false); - blobStoreAbstractConfig1.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); - SimplePartitioner simplePartitioner1 = new SimplePartitioner<>(); - simplePartitioner1.configure(blobStoreAbstractConfig1); - TimePartitioner dayPartitioner1 = new TimePartitioner<>(); - dayPartitioner1.configure(blobStoreAbstractConfig1); - - BlobStoreAbstractConfig blobStoreAbstractConfig2 = new BlobStoreAbstractConfig(); - blobStoreAbstractConfig2.setTimePartitionDuration("1d"); - blobStoreAbstractConfig2.setTimePartitionPattern("yyyy-MM-dd"); - blobStoreAbstractConfig2.setSliceTopicPartitionPath(false); - blobStoreAbstractConfig2.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); - SimplePartitioner simplePartitioner2 = new SimplePartitioner<>(); - simplePartitioner2.configure(blobStoreAbstractConfig2); - TimePartitioner dayPartitioner2 = new TimePartitioner<>(); - dayPartitioner2.configure(blobStoreAbstractConfig2); - - BlobStoreAbstractConfig blobStoreAbstractConfig3 = new BlobStoreAbstractConfig(); - blobStoreAbstractConfig3.setTimePartitionDuration("1d"); - blobStoreAbstractConfig3.setTimePartitionPattern("yyyy-MM-dd"); - blobStoreAbstractConfig3.setSliceTopicPartitionPath(true); - blobStoreAbstractConfig3.setWithTopicPartitionNumber(false); - blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); - SimplePartitioner simplePartitioner3 = new SimplePartitioner<>(); - simplePartitioner3.configure(blobStoreAbstractConfig3); - TimePartitioner dayPartitioner3 = new TimePartitioner<>(); - dayPartitioner3.configure(blobStoreAbstractConfig3); - - BlobStoreAbstractConfig blobStoreAbstractConfig4 = new BlobStoreAbstractConfig(); - blobStoreAbstractConfig4.setTimePartitionDuration("1d"); - blobStoreAbstractConfig4.setTimePartitionPattern("yyyy-MM-dd"); - blobStoreAbstractConfig4.setSliceTopicPartitionPath(true); - blobStoreAbstractConfig4.setWithTopicPartitionNumber(true); - blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + - "public/default/test-partition-1=path2,public/default/test/1=path3"); - SimplePartitioner simplePartitioner4 = new SimplePartitioner<>(); - simplePartitioner4.configure(blobStoreAbstractConfig4); - TimePartitioner dayPartitioner4 = new TimePartitioner<>(); - dayPartitioner4.configure(blobStoreAbstractConfig4); - - return new Object[][]{ - new Object[]{ - simplePartitioner1, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - dayPartitioner1, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - simplePartitioner1, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - dayPartitioner1, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - simplePartitioner2, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - dayPartitioner2, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - simplePartitioner2, - "3221225506", - "path2" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - dayPartitioner2, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path2/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - simplePartitioner3, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - dayPartitioner3, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - simplePartitioner3, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - dayPartitioner3, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - simplePartitioner4, - "3221225506", - "path1" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - dayPartitioner4, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path1/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getTopic() - }, - new Object[]{ - simplePartitioner4, - "3221225506", - "path3" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - new Object[]{ - dayPartitioner4, - "2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - "path3/2020-09-08" + Partitioner.PATH_SEPARATOR + "3221225506", - getPartitionedTopic() - }, - }; - } - - public static Record getPartitionedTopic() { - @SuppressWarnings("unchecked") - Message mock = mock(Message.class); - when(mock.getPublishTime()).thenReturn(1599578218610L); - when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); - String topic = TopicName.get("test-partition-1").toString(); - Record mockRecord = mock(Record.class); - when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); - when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); - when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); - when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); - when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); - return mockRecord; - } - - public static Record getTopic() { - @SuppressWarnings("unchecked") - Message mock = mock(Message.class); - when(mock.getPublishTime()).thenReturn(1599578218610L); - when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); - String topic = TopicName.get("test").toString(); - Record mockRecord = mock(Record.class); - when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); - when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); - when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); - when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); - when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); - return mockRecord; - } - - @Test - public void testEncodePartition() { - String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); - Supplier supplier = - () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); - Assert.assertEquals(supplier.get(), expected, encodePartition); - } - - @Test - public void testGeneratePartitionedPath() { - String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); - String partitionedPath = - partitioner.generatePartitionedPath(pulsarRecord.getTopicName().get(), encodePartition); - - Supplier supplier = - () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); - Assert.assertEquals(supplier.get(), expectedPartitionedPath, partitionedPath); - } -} From 5c01a16cba972ba21b419646eb18c6e1fec38e97 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 13:04:29 +0530 Subject: [PATCH 14/58] Revert "[fix]: fix Parquet/Avro format with separated key value avro-avro messages (#3)" This reverts commit 8802a760695a9ec83c172ec21b8c5bb6fcbe8f3f. --- .github/workflows/pr-test.yml | 4 +- pom.xml | 131 +++--------------- .../pulsar/io/jcloud/format/BytesFormat.java | 7 +- .../io/jcloud/format/ParquetFormat.java | 5 - .../pulsar/io/jcloud/util/AvroRecordUtil.java | 52 +------ .../pulsar/io/jcloud/PulsarTestBase.java | 117 +++++++--------- .../io/jcloud/format/FormatTestBase.java | 117 +++------------- 7 files changed, 101 insertions(+), 332 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 47765203..2299ef85 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -18,10 +18,10 @@ jobs: java-version: 17 - name: Code check - run: mvn license:check spotbugs:check + run: mvn license:check checkstyle:check spotbugs:check - name: Build with Maven - run: mvn install test -DfailIfNoTests=false -Dcheckstyle.skip + run: mvn install test -DfailIfNoTests=false - name: package surefire artifacts if: failure() diff --git a/pom.xml b/pom.xml index 19900099..5b1fcbe6 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 2.10.1-SNAPSHOT + 2.9.0-rc-202110221101 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -47,9 +47,8 @@ 2.13.2 2.13.4.2 - 1.18.32 - com.datastax.oss - 2.10.4.0 + 1.18.20 + 3.2.2.1 1.11.3 3.3.6 1.13.1 @@ -98,33 +97,26 @@ - - scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git - scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git - https://github.com/datastax/pulsar-io-cloud-storage - HEAD - - - ${pulsar.groupId} + io.streamnative pulsar-io-core ${pulsar.version} - ${pulsar.groupId} + io.streamnative pulsar-io-common ${pulsar.version} - ${pulsar.groupId} + io.streamnative pulsar-common ${pulsar.version} - ${pulsar.groupId} + io.streamnative pulsar-client-original ${pulsar.version} @@ -149,7 +141,7 @@ ${spotbugs-annotations.version} - ${pulsar.groupId} + io.streamnative jclouds-shaded ${pulsar.version} @@ -329,19 +321,19 @@ - ${pulsar.groupId} + io.streamnative pulsar-io-core - ${pulsar.groupId} + io.streamnative pulsar-io-common - ${pulsar.groupId} + io.streamnative pulsar-common - ${pulsar.groupId} + io.streamnative pulsar-client-original @@ -354,7 +346,7 @@ jackson-dataformat-yaml - ${pulsar.groupId} + io.streamnative jclouds-shaded @@ -422,34 +414,6 @@ org.apache.avro avro - 1.10.2 - - - org.apache.hadoop - hadoop-client - 3.3.3 - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-yarn-client - - - org.apache.hadoop - hadoop-yarn-common - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - org.apache.hadoop - hadoop-hdfs-client - - org.apache.hadoop @@ -636,40 +600,12 @@ test - org.apache.commons - commons-lang3 - 3.11 - - - ${pulsar.groupId} - pulsar-io-common - - - javax.xml.bind - jaxb-api - ${jaxb-api} - - - ${pulsar.groupId} - pulsar-client-original - ${pulsar.version} - - - ${pulsar.groupId} + io.streamnative pulsar-client-admin-original ${pulsar.version} - - - net.java.dev.jna - jna - 5.7.0 test - - ${pulsar.groupId} - pulsar-common - ${pulsar.version} - + @@ -681,8 +617,6 @@ ${maven-compiler-plugin.version} UTF-8 - ${java.version} - ${java.version} -Xlint:deprecation @@ -696,9 +630,7 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} - false 1800 ${testRetryCount} @@ -894,38 +826,11 @@ central default https://repo1.maven.org/maven2 - - false - - - - confluent - https://packages.confluent.io/maven/ - - false - - datastax-releases - https://repo.datastax.com/artifactory/datastax-public-releases-local - - false - - - true - + bintray-streamnative-maven + bintray + https://dl.bintray.com/streamnative/maven - - - datastax-releases - DataStax Local Releases - https://repo.aws.dsinternal.org/artifactory/datastax-releases-local/ - - - datastax-snapshots-local - DataStax Local Snapshots - https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ - - diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java index b6b1e39b..8b85b3c8 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java @@ -19,7 +19,6 @@ package org.apache.pulsar.io.jcloud.format; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.Iterator; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; @@ -36,7 +35,6 @@ */ public class BytesFormat implements Format, InitConfiguration { - public static final byte[] NULL_BYTES = "".getBytes(StandardCharsets.UTF_8); private byte[] lineSeparatorBytes; @Override @@ -64,10 +62,7 @@ public ByteBuffer recordWriterBuf(Iterator> record) throws while (record.hasNext()) { final Record next = record.next(); final Message message = next.getMessage().get(); - byte[] data = message.getData(); - if (data == null) { - data = NULL_BYTES; - } + final byte[] data = message.getData(); dataOutput.write(data); dataOutput.write(lineSeparatorBytes); } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java index 7442b2f8..93efd12e 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/ParquetFormat.java @@ -28,11 +28,9 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.parquet.avro.AvroParquetWriter; @@ -42,11 +40,8 @@ import org.apache.parquet.io.OutputFile; import org.apache.parquet.io.PositionOutputStream; import org.apache.pulsar.client.api.SchemaSerializationException; -import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.protocol.schema.ProtobufNativeSchemaData; -import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; diff --git a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java index 762d3416..fea44b88 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.io.jcloud.util; +import static org.apache.avro.Schema.Type.ARRAY; +import static org.apache.avro.Schema.Type.ENUM; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import java.lang.reflect.Method; @@ -33,8 +35,6 @@ import org.apache.avro.protobuf.ProtobufData; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.schema.Field; -import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.MessageImpl; @@ -43,15 +43,12 @@ import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; -import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.avro.Schema.Type.*; - /** * avro util. @@ -171,10 +168,8 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema final Schema avroSchema = Schema.createRecord("KVSchema", null, null, false, Arrays.asList( - // namespace+name must be different in the KVSchema - new Schema.Field("key", parseAvroSchema(keySchemaDef, "_key")), - new Schema.Field("value", Schema.createUnion( - Schema.create(Schema.Type.NULL), parseAvroSchema(valueSchemaDef, "_value"))) + new Schema.Field("key", parseAvroSchema(keySchemaDef)), + new Schema.Field("value", parseAvroSchema(valueSchemaDef)) )); return avroSchema; } else { @@ -186,23 +181,14 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema if (StringUtils.isEmpty(rootAvroSchemaString)) { throw new IllegalArgumentException("schema definition is empty"); } - return parseAvroSchema(rootAvroSchemaString, null); + return parseAvroSchema(rootAvroSchemaString); } } - private static Schema parseAvroSchema(String jsonSchema, String namespaceSuffix) { + private static Schema parseAvroSchema(String jsonSchema) { final Schema.Parser parser = new Schema.Parser(); parser.setValidateDefaults(false); - Schema schema = parser.parse(jsonSchema); - List fields = schema.getFields() - .stream() - .map(f -> new Schema.Field(f, f.schema())) - .collect(Collectors.toList()); - if (namespaceSuffix == null) { - namespaceSuffix = ""; - } - final String namespace = schema.getNamespace() == null ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; - return Schema.createRecord(schema.getName(), null, namespace, false, fields); + return parser.parse(jsonSchema); } public static org.apache.avro.generic.GenericRecord convertGenericRecord(DynamicMessage recordValue, @@ -229,30 +215,6 @@ public static org.apache.avro.generic.GenericRecord convertGenericRecord(Dynamic public static org.apache.avro.generic.GenericRecord convertGenericRecord(GenericRecord recordValue, Schema rootAvroSchema) { - if (recordValue.getSchemaType() == SchemaType.KEY_VALUE) { - KeyValue keyValue = - (KeyValue) recordValue.getNativeObject(); - org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); - GenericRecord keyObject = keyValue.getKey(); - if (keyObject != null) { - Schema keySchema = rootAvroSchema.getField("key").schema(); - recordHolder.put("key", convertGenericRecord(keyObject, keySchema)); - } - GenericRecord valueObject = keyValue.getValue(); - if (valueObject != null) { - Schema valueSchema = rootAvroSchema.getField("value").schema(); - recordHolder.put("value", convertGenericRecord(valueObject, valueSchema)); - } - return recordHolder; - } - - // handle nullable fields that are union[null,record] - if (rootAvroSchema.isUnion()) { - rootAvroSchema = rootAvroSchema.getTypes().stream() - .filter(schema -> schema.getType().equals(Schema.Type.RECORD)) - .findFirst() - .get(); - } org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); for (org.apache.pulsar.client.api.schema.Field field : recordValue.getFields()) { Schema.Field field1 = rootAvroSchema.getField(field.getName()); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index ed782341..2048a150 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -28,7 +28,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.avro.generic.GenericData; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.pulsar.client.api.Consumer; @@ -38,10 +37,8 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.container.PulsarContainer; import org.junit.AfterClass; @@ -120,6 +117,7 @@ public static List sendTypedMessages( return sendTypedMessages(topic, type, messages, partition, null); } + @SuppressWarnings("unchecked") public static List sendTypedMessages( String topic, SchemaType type, @@ -127,66 +125,6 @@ public static List sendTypedMessages( Optional partition, Class tClass) throws PulsarClientException { - Schema schema; - - switch (type) { - case BOOLEAN: - schema = Schema.BOOL; - break; - case BYTES: - schema = Schema.BYTES; - break; - case DATE: - schema = Schema.DATE; - break; - case STRING: - schema = Schema.STRING; - break; - case TIMESTAMP: - schema = Schema.TIMESTAMP; - break; - case INT8: - schema = Schema.INT8; - break; - case DOUBLE: - schema = Schema.DOUBLE; - break; - case FLOAT: - schema = Schema.FLOAT; - break; - case INT32: - schema = Schema.INT32; - break; - case INT16: - schema = Schema.INT16; - break; - case INT64: - schema = Schema.INT64; - break; - case AVRO: - schema = Schema.AVRO(tClass); - break; - case JSON: - schema = Schema.JSON(tClass); - break; - case KEY_VALUE: - final KeyValue kvMessage = (KeyValue) messages.get(0); - schema = Schema.KeyValue(kvMessage.getKey().getClass(), - kvMessage.getValue().getClass()); - break; - default: - throw new NotImplementedException("Unsupported type " + type); - } - return sendTypedMessages(topic, messages, schema, partition); - - } - @SuppressWarnings("unchecked") - public static List sendTypedMessages( - String topic, - List messages, - Schema produceSchema, - Optional partition) throws PulsarClientException { - String topicName; if (partition.isPresent()) { topicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + partition.get(); @@ -200,7 +138,56 @@ public static List sendTypedMessages( try { client = PulsarClient.builder().serviceUrl(getServiceUrl()).build(); - producer = client.newProducer(produceSchema).topic(topicName).create(); + + switch (type) { + case BOOLEAN: + producer = (Producer) client.newProducer(Schema.BOOL).topic(topicName).create(); + break; + case BYTES: + producer = (Producer) client.newProducer(Schema.BYTES).topic(topicName).create(); + break; + case DATE: + producer = (Producer) client.newProducer(Schema.DATE).topic(topicName).create(); + break; + case STRING: + producer = (Producer) client.newProducer(Schema.STRING).topic(topicName).create(); + break; + case TIMESTAMP: + producer = (Producer) client.newProducer(Schema.TIMESTAMP).topic(topicName).create(); + break; + case INT8: + producer = (Producer) client.newProducer(Schema.INT8).topic(topicName).create(); + break; + case DOUBLE: + producer = (Producer) client.newProducer(Schema.DOUBLE).topic(topicName).create(); + break; + case FLOAT: + producer = (Producer) client.newProducer(Schema.FLOAT).topic(topicName).create(); + break; + case INT32: + producer = (Producer) client.newProducer(Schema.INT32).topic(topicName).create(); + break; + case INT16: + producer = (Producer) client.newProducer(Schema.INT16).topic(topicName).create(); + break; + case INT64: + producer = (Producer) client.newProducer(Schema.INT64).topic(topicName).create(); + break; + case AVRO: + producer = (Producer) client.newProducer(Schema.AVRO(tClass)).topic(topicName).create(); + break; + case JSON: + producer = (Producer) client.newProducer(Schema.JSON(tClass)).topic(topicName).create(); + break; + case KEY_VALUE: + final KeyValue kvMessage = (KeyValue) messages.get(0); + producer = (Producer) client.newProducer(Schema.KeyValue(kvMessage.getKey().getClass(), + kvMessage.getValue().getClass())).topic(topicName).create(); + break; + + default: + throw new NotImplementedException("Unsupported type " + type); + } for (T message : messages) { MessageId mid = producer.send(message); @@ -277,14 +264,12 @@ public static void consumerMessages(String topic, consumer = client.newConsumer(schema) .topic(topic) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscriptionName("test") .subscribe(); int receiveCount = 0; while (receiveCount < count) { final CompletableFuture> receiveAsync = consumer.receiveAsync(); final Message message = receiveAsync.get(timeout, TimeUnit.MILLISECONDS); - System.out.println("received message " + message); handler.accept(message); consumer.acknowledge(message); receiveCount += 1; diff --git a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java index ecd12044..035703bd 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java @@ -19,15 +19,9 @@ package org.apache.pulsar.io.jcloud.format; import static org.junit.Assert.fail; -import static org.mockito.Mockito.when; - import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Arrays; import java.util.List; @@ -35,12 +29,6 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; - -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.util.Utf8; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -49,15 +37,10 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; import org.apache.pulsar.io.jcloud.PulsarTestBase; @@ -89,8 +72,6 @@ public abstract class FormatTestBase extends PulsarTestBase { TopicName.get("test-json-bytes-parquet-json" + RandomStringUtils.randomAlphabetic(5)); protected static TopicName jsonStringTopicName = TopicName.get("test-json-string-parquet-json" + RandomStringUtils.randomAlphabetic(5)); - private static final TopicName kvSeparatedTopicName = - TopicName.get("test-parquet-kv-sep" + RandomStringUtils.randomAlphabetic(5)); @BeforeClass public static void setUp() throws Exception { @@ -188,60 +169,6 @@ public void testProtobufNativeRecordWriter() throws Exception { consumerMessages(protobufNativeTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); } - @Test - public void testKeyValueAvroWithSameNamespaceName() throws Exception { - org.apache.avro.Schema keySchema = SchemaBuilder.record("record") - .fields() - .name("id").type().stringType().noDefault() - .endRecord(); - org.apache.avro.Schema valueSchema = SchemaBuilder.record("record") - .fields() - .name("content").type().stringType().noDefault() - .endRecord(); - - SchemaDefinition keySchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" + - "{\n" + - " \t\"type\": \"record\",\n" + - " \t\"name\": \"record\",\n" + - " \t\"fields\": [{\n" + - " \t\t\"name\": \"id\",\n" + - " \t\t\"type\": [\"string\"]\n" + - " \t}]\n" + - " }")) - .build(); - - SchemaDefinition valueSchemaDef = SchemaDefinition.builder() - .withJsonDef(String.format("" + - "{\n" + - " \t\"type\": \"record\",\n" + - " \t\"name\": \"record\",\n" + - " \t\"fields\": [{\n" + - " \t\t\"name\": \"content\",\n" + - " \t\t\"type\": [\"string\"]\n" + - " \t}]\n" + - " }")) - .build(); - - - Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); - - GenericData.Record keyRecord = new GenericData.Record(keySchema); - keyRecord.put("id", "theid"); - GenericData.Record valueRecord = new GenericData.Record(valueSchema); - valueRecord.put("content", "ccc"); - List testRecords = Arrays.asList( - new KeyValue(keyRecord, valueRecord), - new KeyValue(keyRecord, null) - ); - - sendTypedMessages(kvSeparatedTopicName.toString(), testRecords, schema, Optional.empty()); - - Consumer> handle = getMessageConsumer(kvSeparatedTopicName); - consumerMessages(kvSeparatedTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); - } - - protected abstract boolean supportMetadata(); protected Consumer> getMessageConsumer(TopicName topic) { @@ -297,11 +224,11 @@ public abstract org.apache.avro.generic.GenericRecord getFormatGeneratedRecord(T throws Exception; public abstract DynamicMessage getDynamicMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; public abstract Map getJSONMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.GenericRecord record) { @@ -310,16 +237,16 @@ protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.Gen Object sourceValue = msgValue.getField(descriptor.findFieldByName(field.name())); Object newValue = record.get(field.name()); Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - field.name(), - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + field.name(), + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } @@ -455,16 +382,16 @@ protected void assertEquals(DynamicMessage msgValue, Map record) Object newValue = record.get(fieldName); if (!fieldDescriptor.isRepeated()) { Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - fieldName, - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + fieldName, + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } } From e4a9bb242ffc3dbe3bc1147fa9bebc7b8f307f11 Mon Sep 17 00:00:00 2001 From: mukesh-ctds <151806568+mukesh-ctds@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:26:30 +0530 Subject: [PATCH 15/58] Upgrade: Code sync with upstream branch-3.2 (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [fix]: fix Parquet/Avro format with separated key value avro-avro messages (#3) (cherry picked from commit 37a5c85b845cc2be46d718365830884a00b366ae) (cherry picked from commit 8802a760695a9ec83c172ec21b8c5bb6fcbe8f3f) * [improve] Allow user-specified S3 path (#5) * Added user-specified S3 path * Optimized the implementation * Variables renamed * Logic moved to Partitioner & Added unit tests with all the flags about the partitions with all the combinations * Removed unnecessary checks (cherry picked from commit 2a66206bca2345d1296f12d98bf3d946f9f0f4a2) (cherry picked from commit 2696f0dbcab2f11b76f15257510fe5e34120111c) * Fix checkstyle (cherry picked from commit 21c3ad3b7c40bb8c48cac87b888a59757d39c4a7) * Added ds-release.yml * Removed System.out.println from PulsarTestBase.java --------- Co-authored-by: Nicolò Boschi Co-authored-by: Mukesh Kumar <65598381+mukesh154@users.noreply.github.com> --- .github/workflows/ds-release.yml | 26 ++ .github/workflows/pr-test.yml | 2 +- pom.xml | 131 +++++++-- .../io/jcloud/BlobStoreAbstractConfig.java | 21 ++ .../pulsar/io/jcloud/format/BytesFormat.java | 7 +- .../legacy/AbstractPartitioner.java | 62 ++++- .../pulsar/io/jcloud/util/AvroRecordUtil.java | 47 +++- .../pulsar/io/jcloud/ConnectorConfigTest.java | 73 +++++ .../pulsar/io/jcloud/PulsarTestBase.java | 114 ++++---- .../io/jcloud/format/FormatTestBase.java | 105 +++++-- .../TopicsToPathMappingPartitionerTest.java | 260 ++++++++++++++++++ 11 files changed, 735 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/ds-release.yml create mode 100644 src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java diff --git a/.github/workflows/ds-release.yml b/.github/workflows/ds-release.yml new file mode 100644 index 00000000..26c3f37d --- /dev/null +++ b/.github/workflows/ds-release.yml @@ -0,0 +1,26 @@ +name: DS Release + +on: + push: + tags: + - 'v*' + +jobs: + create-release: + permissions: write-all + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'adopt' + - name: Build with Maven + run: mvn -B package -DskipTests -Dcheckstyle.skip + - uses: ncipollo/release-action@v1 + with: + artifacts: "**/target/*.nar" + token: ${{ secrets.GITHUB_TOKEN }} + generateReleaseNotes: true \ No newline at end of file diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 2299ef85..fc5fdd15 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - 3.2_ds jobs: build: diff --git a/pom.xml b/pom.xml index 5b1fcbe6..f4e9204d 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 2.9.0-rc-202110221101 + 3.2.0-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -47,8 +47,9 @@ 2.13.2 2.13.4.2 - 1.18.20 - 3.2.2.1 + 1.18.32 + com.datastax.oss + 3.1.4.2 1.11.3 3.3.6 1.13.1 @@ -97,26 +98,33 @@ + + scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git + scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git + https://github.com/datastax/pulsar-io-cloud-storage + HEAD + + - io.streamnative + ${pulsar.groupId} pulsar-io-core ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-io-common ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-common ${pulsar.version} - io.streamnative + ${pulsar.groupId} pulsar-client-original ${pulsar.version} @@ -141,7 +149,7 @@ ${spotbugs-annotations.version} - io.streamnative + ${pulsar.groupId} jclouds-shaded ${pulsar.version} @@ -321,19 +329,19 @@ - io.streamnative + ${pulsar.groupId} pulsar-io-core - io.streamnative + ${pulsar.groupId} pulsar-io-common - io.streamnative + ${pulsar.groupId} pulsar-common - io.streamnative + ${pulsar.groupId} pulsar-client-original @@ -346,7 +354,7 @@ jackson-dataformat-yaml - io.streamnative + ${pulsar.groupId} jclouds-shaded @@ -414,6 +422,34 @@ org.apache.avro avro + 1.10.2 + + + org.apache.hadoop + hadoop-client + 3.3.3 + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-client + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-hdfs-client + + org.apache.hadoop @@ -600,12 +636,40 @@ test - io.streamnative + org.apache.commons + commons-lang3 + 3.11 + + + ${pulsar.groupId} + pulsar-io-common + + + javax.xml.bind + jaxb-api + ${jaxb-api} + + + ${pulsar.groupId} + pulsar-client-original + ${pulsar.version} + + + ${pulsar.groupId} pulsar-client-admin-original ${pulsar.version} + + + net.java.dev.jna + jna + 5.7.0 test - + + ${pulsar.groupId} + pulsar-common + ${pulsar.version} + @@ -617,6 +681,8 @@ ${maven-compiler-plugin.version} UTF-8 + ${java.version} + ${java.version} -Xlint:deprecation @@ -630,7 +696,9 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven-surefire-plugin.version} + false 1800 ${testRetryCount} @@ -826,11 +894,38 @@ central default https://repo1.maven.org/maven2 + + false + + + + confluent + https://packages.confluent.io/maven/ + + false + - bintray-streamnative-maven - bintray - https://dl.bintray.com/streamnative/maven + datastax-releases + https://repo.datastax.com/artifactory/datastax-public-releases-local + + false + + + true + + + + datastax-releases + DataStax Local Releases + https://repo.aws.dsinternal.org/artifactory/datastax-releases-local/ + + + datastax-snapshots-local + DataStax Local Snapshots + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ + + diff --git a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java index e1ce1518..fcda4f42 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/BlobStoreAbstractConfig.java @@ -119,6 +119,7 @@ public class BlobStoreAbstractConfig implements Serializable { private String bytesFormatTypeSeparator = "0x10"; private boolean skipFailedMessages = false; private boolean jsonAllowNaN = false; + private String topicsToPathMapping; public void validate() { checkNotNull(provider, "provider not set."); @@ -194,6 +195,10 @@ public void validate() { if (parquetCodec != null && (parquetCodec.isEmpty() || parquetCodec.equals("none"))) { parquetCodec = null; } + + if (StringUtils.isNotEmpty(topicsToPathMapping)) { + validateTopicsToPathMapping(); + } } private static boolean hasURIScheme(String endpoint) { @@ -205,4 +210,20 @@ private static boolean hasURIScheme(String endpoint) { } } + public void validateTopicsToPathMapping() { + for (String topicToPathMapping : topicsToPathMapping.split(",")) { + String[] topicPathPair = topicToPathMapping.split("="); + checkArgument(topicPathPair.length == 2, + "Invalid topicsToPathMapping format: " + topicToPathMapping); + String topic = topicPathPair[0].trim(); + String path = topicPathPair[1].trim(); + checkArgument(isNotBlank(topic), "Topic in topicsToPathMapping cannot be empty."); + checkArgument(isNotBlank(path), "Path in topicsToPathMapping cannot be empty."); + checkArgument(topic.matches("^[^/]+/[^/]+/[^/]+$"), + "Topic must be in the format 'tenant/ns/topic': " + topic); + checkArgument(!path.startsWith("/") && !path.endsWith("/"), + "Path in topicsToPathMapping cannot starts or ends with '/': " + path); + } + } + } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java index 8b85b3c8..b6b1e39b 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/format/BytesFormat.java @@ -19,6 +19,7 @@ package org.apache.pulsar.io.jcloud.format; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; @@ -35,6 +36,7 @@ */ public class BytesFormat implements Format, InitConfiguration { + public static final byte[] NULL_BYTES = "".getBytes(StandardCharsets.UTF_8); private byte[] lineSeparatorBytes; @Override @@ -62,7 +64,10 @@ public ByteBuffer recordWriterBuf(Iterator> record) throws while (record.hasNext()) { final Record next = record.next(); final Message message = next.getMessage().get(); - final byte[] data = message.getData(); + byte[] data = message.getData(); + if (data == null) { + data = NULL_BYTES; + } dataOutput.write(data); dataOutput.write(lineSeparatorBytes); } diff --git a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java index 71e2d3af..1ccb5620 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/partitioner/legacy/AbstractPartitioner.java @@ -19,7 +19,9 @@ package org.apache.pulsar.io.jcloud.partitioner.legacy; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Message; @@ -42,35 +44,65 @@ public abstract class AbstractPartitioner implements Partitioner { private boolean withTopicPartitionNumber; private boolean useIndexAsOffset; + private Map topicsToPathMapping; + private Map computedTopicsToPathMapping; + @Override public void configure(BlobStoreAbstractConfig config) { this.sliceTopicPartitionPath = config.isSliceTopicPartitionPath(); this.withTopicPartitionNumber = config.isWithTopicPartitionNumber(); this.useIndexAsOffset = config.isPartitionerUseIndexAsOffset(); + this.topicsToPathMapping = parseTopicsToPathMappingString(config.getTopicsToPathMapping()); + this.computedTopicsToPathMapping = new HashMap<>(); + } + + private Map parseTopicsToPathMappingString(String topicsToPathMappingString) { + Map topicsToPathMapping = new HashMap<>(); + if (StringUtils.isNotEmpty(topicsToPathMappingString)) { + for (String topicPathPair : topicsToPathMappingString.split(",")) { + String[] topicToPathMapping = topicPathPair.split("="); + if (topicToPathMapping.length == 2) { + topicsToPathMapping.put(topicToPathMapping[0], topicToPathMapping[1]); + } else { + LOGGER.warn("The topic to path mapping is not in correct format {}.", topicPathPair); + } + } + } + LOGGER.info("The final topics to path mapping is: {}", topicsToPathMapping); + return topicsToPathMapping; } @Override public String generatePartitionedPath(String topic, String encodedPartition) { + if (computedTopicsToPathMapping.containsKey(topic)) { + return String.format("%s%s%s", computedTopicsToPathMapping.get(topic), PATH_SEPARATOR, encodedPartition); + } else { + List joinList = new ArrayList<>(); + TopicName topicName = TopicName.get(topic); + joinList.add(topicName.getTenant()); + joinList.add(topicName.getNamespacePortion()); - List joinList = new ArrayList<>(); - TopicName topicName = TopicName.get(topic); - joinList.add(topicName.getTenant()); - joinList.add(topicName.getNamespacePortion()); - - if (topicName.isPartitioned() && withTopicPartitionNumber) { - if (sliceTopicPartitionPath) { + if (topicName.isPartitioned() && withTopicPartitionNumber) { + if (sliceTopicPartitionPath) { + TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); + joinList.add(newTopicName.getLocalName()); + joinList.add(Integer.toString(topicName.getPartitionIndex())); + } else { + joinList.add(topicName.getLocalName()); + } + } else { TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); joinList.add(newTopicName.getLocalName()); - joinList.add(Integer.toString(topicName.getPartitionIndex())); - } else { - joinList.add(topicName.getLocalName()); } - } else { - TopicName newTopicName = TopicName.get(topicName.getPartitionedTopicName()); - joinList.add(newTopicName.getLocalName()); + String generatedTopicPath = StringUtils.join(joinList, PATH_SEPARATOR); + if (topicsToPathMapping.containsKey(generatedTopicPath)) { + joinList.clear(); + joinList.add(topicsToPathMapping.get(generatedTopicPath)); + } + computedTopicsToPathMapping.put(topic, StringUtils.join(joinList, PATH_SEPARATOR)); + joinList.add(encodedPartition); + return StringUtils.join(joinList, PATH_SEPARATOR); } - joinList.add(encodedPartition); - return StringUtils.join(joinList, PATH_SEPARATOR); } protected long getMessageOffset(Record record) { diff --git a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java index fea44b88..47949271 100644 --- a/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java +++ b/src/main/java/org/apache/pulsar/io/jcloud/util/AvroRecordUtil.java @@ -43,6 +43,7 @@ import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; +import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; @@ -168,8 +169,10 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema final Schema avroSchema = Schema.createRecord("KVSchema", null, null, false, Arrays.asList( - new Schema.Field("key", parseAvroSchema(keySchemaDef)), - new Schema.Field("value", parseAvroSchema(valueSchemaDef)) + // namespace+name must be different in the KVSchema + new Schema.Field("key", parseAvroSchema(keySchemaDef, "_key")), + new Schema.Field("value", Schema.createUnion( + Schema.create(Schema.Type.NULL), parseAvroSchema(valueSchemaDef, "_value"))) )); return avroSchema; } else { @@ -181,14 +184,24 @@ public static Schema convertToAvroSchema(org.apache.pulsar.client.api.Schema if (StringUtils.isEmpty(rootAvroSchemaString)) { throw new IllegalArgumentException("schema definition is empty"); } - return parseAvroSchema(rootAvroSchemaString); + return parseAvroSchema(rootAvroSchemaString, null); } } - private static Schema parseAvroSchema(String jsonSchema) { + private static Schema parseAvroSchema(String jsonSchema, String namespaceSuffix) { final Schema.Parser parser = new Schema.Parser(); parser.setValidateDefaults(false); - return parser.parse(jsonSchema); + Schema schema = parser.parse(jsonSchema); + List fields = schema.getFields() + .stream() + .map(f -> new Schema.Field(f, f.schema())) + .collect(Collectors.toList()); + if (namespaceSuffix == null) { + namespaceSuffix = ""; + } + final String namespace = schema.getNamespace() == null + ? namespaceSuffix : schema.getNamespace() + namespaceSuffix; + return Schema.createRecord(schema.getName(), null, namespace, false, fields); } public static org.apache.avro.generic.GenericRecord convertGenericRecord(DynamicMessage recordValue, @@ -215,6 +228,30 @@ public static org.apache.avro.generic.GenericRecord convertGenericRecord(Dynamic public static org.apache.avro.generic.GenericRecord convertGenericRecord(GenericRecord recordValue, Schema rootAvroSchema) { + if (recordValue.getSchemaType() == SchemaType.KEY_VALUE) { + KeyValue keyValue = + (KeyValue) recordValue.getNativeObject(); + org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); + GenericRecord keyObject = keyValue.getKey(); + if (keyObject != null) { + Schema keySchema = rootAvroSchema.getField("key").schema(); + recordHolder.put("key", convertGenericRecord(keyObject, keySchema)); + } + GenericRecord valueObject = keyValue.getValue(); + if (valueObject != null) { + Schema valueSchema = rootAvroSchema.getField("value").schema(); + recordHolder.put("value", convertGenericRecord(valueObject, valueSchema)); + } + return recordHolder; + } + + // handle nullable fields that are union[null,record] + if (rootAvroSchema.isUnion()) { + rootAvroSchema = rootAvroSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.RECORD)) + .findFirst() + .get(); + } org.apache.avro.generic.GenericRecord recordHolder = new GenericData.Record(rootAvroSchema); for (org.apache.pulsar.client.api.schema.Field field : recordValue.getFields()) { Schema.Field field1 = rootAvroSchema.getField(field.getName()); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java index ac55f914..e7e6ca8b 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/ConnectorConfigTest.java @@ -52,6 +52,7 @@ public void loadBasicConfigTest() throws IOException { config.put("timePartitionDuration", "2d"); config.put("batchSize", 10); config.put("partitioner", "topic"); + config.put("topicsToPathMapping", "my-tenant/my-ns/topic1=path-mytopic1,my-tenant/my-ns/topic2=path-mytopic2"); CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config); cloudStorageSinkConfig.validate(); @@ -69,6 +70,7 @@ public void loadBasicConfigTest() throws IOException { cloudStorageSinkConfig.getPartitioner().toString().toLowerCase()); Assert.assertEquals((int) config.get("batchSize"), cloudStorageSinkConfig.getPendingQueueSize()); Assert.assertEquals(10000000L, cloudStorageSinkConfig.getMaxBatchBytes()); + Assert.assertEquals(config.get("topicsToPathMapping"), cloudStorageSinkConfig.getTopicsToPathMapping()); } @Test @@ -131,6 +133,77 @@ public void timePartitionDurationTest() throws IOException { } } + @Test + public void topicsToPathMappingTest() throws IOException { + Map config = new HashMap<>(); + config.put("provider", PROVIDER_AWSS3); + config.put("accessKeyId", "aws-s3"); + config.put("secretAccessKey", "aws-s3"); + config.put("bucket", "testbucket"); + config.put("region", "localhost"); + config.put("endpoint", "https://us-standard"); + config.put("formatType", "avro"); + config.put("partitionerType", "default"); + config.put("timePartitionPattern", "yyyy-MM-dd"); + config.put("timePartitionDuration", "2d"); + config.put("batchSize", 10); + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2=path-mytopic2"); + CloudStorageSinkConfig.load(config).validate(); + } catch (Exception e) { + Assert.fail(); + } + + try { + config.put("topicsToPathMapping", "topic1=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1=path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,tenant1/ns1/topic2="); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1,=path-mytopic2"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=/path-mytopic1"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + + try { + config.put("topicsToPathMapping", "tenant1/ns1/topic1=path-mytopic1/"); + CloudStorageSinkConfig.load(config).validate(); + Assert.fail(); + } catch (Exception e) { + } + } + @Test public void pathPrefixTest() throws IOException { Map config = new HashMap<>(); diff --git a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java index 2048a150..da544261 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/PulsarTestBase.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaType; @@ -117,7 +118,6 @@ public static List sendTypedMessages( return sendTypedMessages(topic, type, messages, partition, null); } - @SuppressWarnings("unchecked") public static List sendTypedMessages( String topic, SchemaType type, @@ -125,6 +125,66 @@ public static List sendTypedMessages( Optional partition, Class tClass) throws PulsarClientException { + Schema schema; + + switch (type) { + case BOOLEAN: + schema = Schema.BOOL; + break; + case BYTES: + schema = Schema.BYTES; + break; + case DATE: + schema = Schema.DATE; + break; + case STRING: + schema = Schema.STRING; + break; + case TIMESTAMP: + schema = Schema.TIMESTAMP; + break; + case INT8: + schema = Schema.INT8; + break; + case DOUBLE: + schema = Schema.DOUBLE; + break; + case FLOAT: + schema = Schema.FLOAT; + break; + case INT32: + schema = Schema.INT32; + break; + case INT16: + schema = Schema.INT16; + break; + case INT64: + schema = Schema.INT64; + break; + case AVRO: + schema = Schema.AVRO(tClass); + break; + case JSON: + schema = Schema.JSON(tClass); + break; + case KEY_VALUE: + final KeyValue kvMessage = (KeyValue) messages.get(0); + schema = Schema.KeyValue(kvMessage.getKey().getClass(), + kvMessage.getValue().getClass()); + break; + default: + throw new NotImplementedException("Unsupported type " + type); + } + return sendTypedMessages(topic, messages, schema, partition); + + } + @SuppressWarnings("unchecked") + public static List sendTypedMessages( + String topic, + List messages, + Schema produceSchema, + Optional partition) throws PulsarClientException { + String topicName; if (partition.isPresent()) { topicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + partition.get(); @@ -138,56 +198,7 @@ public static List sendTypedMessages( try { client = PulsarClient.builder().serviceUrl(getServiceUrl()).build(); - - switch (type) { - case BOOLEAN: - producer = (Producer) client.newProducer(Schema.BOOL).topic(topicName).create(); - break; - case BYTES: - producer = (Producer) client.newProducer(Schema.BYTES).topic(topicName).create(); - break; - case DATE: - producer = (Producer) client.newProducer(Schema.DATE).topic(topicName).create(); - break; - case STRING: - producer = (Producer) client.newProducer(Schema.STRING).topic(topicName).create(); - break; - case TIMESTAMP: - producer = (Producer) client.newProducer(Schema.TIMESTAMP).topic(topicName).create(); - break; - case INT8: - producer = (Producer) client.newProducer(Schema.INT8).topic(topicName).create(); - break; - case DOUBLE: - producer = (Producer) client.newProducer(Schema.DOUBLE).topic(topicName).create(); - break; - case FLOAT: - producer = (Producer) client.newProducer(Schema.FLOAT).topic(topicName).create(); - break; - case INT32: - producer = (Producer) client.newProducer(Schema.INT32).topic(topicName).create(); - break; - case INT16: - producer = (Producer) client.newProducer(Schema.INT16).topic(topicName).create(); - break; - case INT64: - producer = (Producer) client.newProducer(Schema.INT64).topic(topicName).create(); - break; - case AVRO: - producer = (Producer) client.newProducer(Schema.AVRO(tClass)).topic(topicName).create(); - break; - case JSON: - producer = (Producer) client.newProducer(Schema.JSON(tClass)).topic(topicName).create(); - break; - case KEY_VALUE: - final KeyValue kvMessage = (KeyValue) messages.get(0); - producer = (Producer) client.newProducer(Schema.KeyValue(kvMessage.getKey().getClass(), - kvMessage.getValue().getClass())).topic(topicName).create(); - break; - - default: - throw new NotImplementedException("Unsupported type " + type); - } + producer = client.newProducer(produceSchema).topic(topicName).create(); for (T message : messages) { MessageId mid = producer.send(message); @@ -264,6 +275,7 @@ public static void consumerMessages(String topic, consumer = client.newConsumer(schema) .topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscriptionName("test") .subscribe(); int receiveCount = 0; diff --git a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java index 035703bd..d9621d13 100644 --- a/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java +++ b/src/test/java/org/apache/pulsar/io/jcloud/format/FormatTestBase.java @@ -29,6 +29,8 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; import org.apache.avro.util.Utf8; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -37,10 +39,12 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; import org.apache.pulsar.io.jcloud.PulsarTestBase; @@ -72,6 +76,8 @@ public abstract class FormatTestBase extends PulsarTestBase { TopicName.get("test-json-bytes-parquet-json" + RandomStringUtils.randomAlphabetic(5)); protected static TopicName jsonStringTopicName = TopicName.get("test-json-string-parquet-json" + RandomStringUtils.randomAlphabetic(5)); + private static final TopicName kvSeparatedTopicName = + TopicName.get("test-parquet-kv-sep" + RandomStringUtils.randomAlphabetic(5)); @BeforeClass public static void setUp() throws Exception { @@ -169,6 +175,61 @@ public void testProtobufNativeRecordWriter() throws Exception { consumerMessages(protobufNativeTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); } + @Test + public void testKeyValueAvroWithSameNamespaceName() throws Exception { + org.apache.avro.Schema keySchema = SchemaBuilder.record("record") + .fields() + .name("id").type().stringType().noDefault() + .endRecord(); + org.apache.avro.Schema valueSchema = SchemaBuilder.record("record") + .fields() + .name("content").type().stringType().noDefault() + .endRecord(); + + SchemaDefinition keySchemaDef = SchemaDefinition.builder() + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"id\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) + .build(); + + SchemaDefinition valueSchemaDef = SchemaDefinition.builder() + .withJsonDef(String.format("" + + "{\n" + + " \t\"type\": \"record\",\n" + + " \t\"name\": \"record\",\n" + + " \t\"fields\": [{\n" + + " \t\t\"name\": \"content\",\n" + + " \t\t\"type\": [\"string\"]\n" + + " \t}]\n" + + " }")) + .build(); + + + Schema> schema = Schema.KeyValue(Schema.AVRO(keySchemaDef), + Schema.AVRO(valueSchemaDef), KeyValueEncodingType.SEPARATED); + + GenericData.Record keyRecord = new GenericData.Record(keySchema); + keyRecord.put("id", "theid"); + GenericData.Record valueRecord = new GenericData.Record(valueSchema); + valueRecord.put("content", "ccc"); + List testRecords = Arrays.asList( + new KeyValue(keyRecord, valueRecord), + new KeyValue(keyRecord, null) + ); + + sendTypedMessages(kvSeparatedTopicName.toString(), testRecords, schema, Optional.empty()); + + Consumer> handle = getMessageConsumer(kvSeparatedTopicName); + consumerMessages(kvSeparatedTopicName.toString(), Schema.AUTO_CONSUME(), handle, testRecords.size(), 2000); + } + + protected abstract boolean supportMetadata(); protected Consumer> getMessageConsumer(TopicName topic) { @@ -224,11 +285,11 @@ public abstract org.apache.avro.generic.GenericRecord getFormatGeneratedRecord(T throws Exception; public abstract DynamicMessage getDynamicMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; public abstract Map getJSONMessage(TopicName topicName, - Message msg) + Message msg) throws Exception; protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.GenericRecord record) { @@ -237,16 +298,16 @@ protected void assertEquals(DynamicMessage msgValue, org.apache.avro.generic.Gen Object sourceValue = msgValue.getField(descriptor.findFieldByName(field.name())); Object newValue = record.get(field.name()); Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - field.name(), - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + field.name(), + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } @@ -382,16 +443,16 @@ protected void assertEquals(DynamicMessage msgValue, Map record) Object newValue = record.get(fieldName); if (!fieldDescriptor.isRepeated()) { Assert.assertEquals( - MessageFormat.format( - "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", - fieldName, - sourceValue, - sourceValue != null ? sourceValue.getClass().getName() : "null", - newValue, - newValue != null ? newValue.getClass().getName() : "null" - ), - sourceValue, - newValue); + MessageFormat.format( + "field[{0} sourceValue [{1}:{2}] not equal newValue [{3}:{4}]", + fieldName, + sourceValue, + sourceValue != null ? sourceValue.getClass().getName() : "null", + newValue, + newValue != null ? newValue.getClass().getName() : "null" + ), + sourceValue, + newValue); } } } diff --git a/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java new file mode 100644 index 00000000..2044a655 --- /dev/null +++ b/src/test/java/org/apache/pulsar/io/jcloud/partitioner/TopicsToPathMappingPartitionerTest.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.jcloud.partitioner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.google.common.base.Supplier; +import java.io.File; +import java.text.MessageFormat; +import java.util.Optional; +import junit.framework.TestCase; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.jcloud.BlobStoreAbstractConfig; +import org.apache.pulsar.io.jcloud.partitioner.legacy.Partitioner; +import org.apache.pulsar.io.jcloud.partitioner.legacy.SimplePartitioner; +import org.apache.pulsar.io.jcloud.partitioner.legacy.TimePartitioner; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + + +/** + * partitioner unit test. + */ +@RunWith(Parameterized.class) +public class TopicsToPathMappingPartitionerTest extends TestCase { + + private static final String pathSeparator = File.separator; + + @Parameterized.Parameter(0) + public Partitioner partitioner; + + @Parameterized.Parameter(1) + public String expected; + + @Parameterized.Parameter(2) + public String expectedPartitionedPath; + + @Parameterized.Parameter(3) + public Record pulsarRecord; + + @Parameterized.Parameters + public static Object[][] data() { + BlobStoreAbstractConfig blobStoreAbstractConfig1 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig1.setTimePartitionDuration("1d"); + blobStoreAbstractConfig1.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig1.setSliceTopicPartitionPath(false); + blobStoreAbstractConfig1.setWithTopicPartitionNumber(false); + blobStoreAbstractConfig1.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner1 = new SimplePartitioner<>(); + simplePartitioner1.configure(blobStoreAbstractConfig1); + TimePartitioner dayPartitioner1 = new TimePartitioner<>(); + dayPartitioner1.configure(blobStoreAbstractConfig1); + + BlobStoreAbstractConfig blobStoreAbstractConfig2 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig2.setTimePartitionDuration("1d"); + blobStoreAbstractConfig2.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig2.setSliceTopicPartitionPath(false); + blobStoreAbstractConfig2.setWithTopicPartitionNumber(true); + blobStoreAbstractConfig2.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner2 = new SimplePartitioner<>(); + simplePartitioner2.configure(blobStoreAbstractConfig2); + TimePartitioner dayPartitioner2 = new TimePartitioner<>(); + dayPartitioner2.configure(blobStoreAbstractConfig2); + + BlobStoreAbstractConfig blobStoreAbstractConfig3 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig3.setTimePartitionDuration("1d"); + blobStoreAbstractConfig3.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig3.setSliceTopicPartitionPath(true); + blobStoreAbstractConfig3.setWithTopicPartitionNumber(false); + blobStoreAbstractConfig3.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner3 = new SimplePartitioner<>(); + simplePartitioner3.configure(blobStoreAbstractConfig3); + TimePartitioner dayPartitioner3 = new TimePartitioner<>(); + dayPartitioner3.configure(blobStoreAbstractConfig3); + + BlobStoreAbstractConfig blobStoreAbstractConfig4 = new BlobStoreAbstractConfig(); + blobStoreAbstractConfig4.setTimePartitionDuration("1d"); + blobStoreAbstractConfig4.setTimePartitionPattern("yyyy-MM-dd"); + blobStoreAbstractConfig4.setSliceTopicPartitionPath(true); + blobStoreAbstractConfig4.setWithTopicPartitionNumber(true); + blobStoreAbstractConfig4.setTopicsToPathMapping("public/default/test=path1," + + "public/default/test-partition-1=path2,public/default/test/1=path3"); + SimplePartitioner simplePartitioner4 = new SimplePartitioner<>(); + simplePartitioner4.configure(blobStoreAbstractConfig4); + TimePartitioner dayPartitioner4 = new TimePartitioner<>(); + dayPartitioner4.configure(blobStoreAbstractConfig4); + + return new Object[][]{ + new Object[]{ + simplePartitioner1, + "3221225506", + "path1" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner1, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner1, + "3221225506", + "path1" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner1, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner2, + "3221225506", + "path1" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner2, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner2, + "3221225506", + "path2" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner2, + "2020-09-08" + pathSeparator + "3221225506", + "path2/2020-09-08" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner3, + "3221225506", + "path1" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner3, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner3, + "3221225506", + "path1" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner3, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + simplePartitioner4, + "3221225506", + "path1" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + dayPartitioner4, + "2020-09-08" + pathSeparator + "3221225506", + "path1/2020-09-08" + pathSeparator + "3221225506", + getTopic() + }, + new Object[]{ + simplePartitioner4, + "3221225506", + "path3" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + new Object[]{ + dayPartitioner4, + "2020-09-08" + pathSeparator + "3221225506", + "path3/2020-09-08" + pathSeparator + "3221225506", + getPartitionedTopic() + }, + }; + } + + public static Record getPartitionedTopic() { + @SuppressWarnings("unchecked") + Message mock = mock(Message.class); + when(mock.getPublishTime()).thenReturn(1599578218610L); + when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); + String topic = TopicName.get("test-partition-1").toString(); + Record mockRecord = mock(Record.class); + when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); + when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); + when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); + when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); + when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); + return mockRecord; + } + + public static Record getTopic() { + @SuppressWarnings("unchecked") + Message mock = mock(Message.class); + when(mock.getPublishTime()).thenReturn(1599578218610L); + when(mock.getMessageId()).thenReturn(new MessageIdImpl(12, 34, 1)); + String topic = TopicName.get("test").toString(); + Record mockRecord = mock(Record.class); + when(mockRecord.getTopicName()).thenReturn(Optional.of(topic)); + when(mockRecord.getPartitionIndex()).thenReturn(Optional.of(1)); + when(mockRecord.getMessage()).thenReturn(Optional.of(mock)); + when(mockRecord.getPartitionId()).thenReturn(Optional.of(String.format("%s-%s", topic, 1))); + when(mockRecord.getRecordSequence()).thenReturn(Optional.of(3221225506L)); + return mockRecord; + } + + @Test + public void testEncodePartition() { + String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); + Supplier supplier = + () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); + Assert.assertEquals(supplier.get(), expected, encodePartition); + } + + @Test + public void testGeneratePartitionedPath() { + String encodePartition = partitioner.encodePartition(pulsarRecord, System.currentTimeMillis()); + String partitionedPath = + partitioner.generatePartitionedPath(pulsarRecord.getTopicName().get(), encodePartition); + + Supplier supplier = + () -> MessageFormat.format("expected: {0}\nactual: {1}", expected, encodePartition); + Assert.assertEquals(supplier.get(), expectedPartitionedPath, partitionedPath); + } +} From 76908c2cf8cf05c16d5a7c7de7876c7494ae35a9 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:32:13 +0530 Subject: [PATCH 16/58] [maven-release-plugin] prepare release pulsar-io-cloud-storage-3.2.0 --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index f4e9204d..1bc1c5d1 100644 --- a/pom.xml +++ b/pom.xml @@ -18,8 +18,7 @@ under the License. --> - + org.apache apache @@ -30,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0-SNAPSHOT + 3.2.0 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + pulsar-io-cloud-storage-3.2.0 From 06392e4751d0717751574191ad230e0eb7dbf3b0 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:32:31 +0530 Subject: [PATCH 17/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1bc1c5d1..d1990299 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0 + 3.2.1-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - pulsar-io-cloud-storage-3.2.0 + HEAD From 5dd0810bece305c5215016451291f22833cb76f6 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:35:23 +0530 Subject: [PATCH 18/58] Revert "[maven-release-plugin] prepare for next development iteration" This reverts commit 06392e4751d0717751574191ad230e0eb7dbf3b0. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d1990299..1bc1c5d1 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.1-SNAPSHOT + 3.2.0 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + pulsar-io-cloud-storage-3.2.0 From e16e382cd29874de20faf5af7082eceb1fbd117c Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:35:43 +0530 Subject: [PATCH 19/58] Revert "[maven-release-plugin] prepare release pulsar-io-cloud-storage-3.2.0" This reverts commit 76908c2cf8cf05c16d5a7c7de7876c7494ae35a9. --- pom.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 1bc1c5d1..f4e9204d 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,8 @@ under the License. --> - + org.apache apache @@ -29,7 +30,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0 + 3.2.0-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - pulsar-io-cloud-storage-3.2.0 + HEAD From 5119aa24df11fabc6df84956dd2a058e56636fd2 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:37:51 +0530 Subject: [PATCH 20/58] [maven-release-plugin] prepare release v3.2.0 --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index f4e9204d..e0547fd0 100644 --- a/pom.xml +++ b/pom.xml @@ -18,8 +18,7 @@ under the License. --> - + org.apache apache @@ -30,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0-SNAPSHOT + 3.2.0 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.0 From 9a46a6086a066142fa73892b9943b7e2096a3a68 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Tue, 18 Jun 2024 16:38:09 +0530 Subject: [PATCH 21/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e0547fd0..d1990299 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.0 + 3.2.1-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.0 + HEAD From 56fd4677719af976ff0cd47195d436ff6a1297cf Mon Sep 17 00:00:00 2001 From: nikhil-ctds <151718832+nikhil-ctds@users.noreply.github.com> Date: Tue, 16 Jul 2024 22:07:38 +0530 Subject: [PATCH 22/58] Removed duplicate avro version for runtime (#7) --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1990299..58eaaf41 100644 --- a/pom.xml +++ b/pom.xml @@ -421,7 +421,6 @@ org.apache.avro avro - 1.10.2 org.apache.hadoop From 3bd84183bf276f39982ee8a5d2399aaf0af08442 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 17 Jul 2024 10:35:30 +0530 Subject: [PATCH 23/58] [maven-release-plugin] prepare release v3.2.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 58eaaf41..70f03ff7 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.1-SNAPSHOT + 3.2.1 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. From 0a6c5782593074f374a8b6dba984e4a845de83bc Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 17 Jul 2024 10:46:28 +0530 Subject: [PATCH 24/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 70f03ff7..0d63e55f 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.1 + 3.2.2-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. From e660c90dead8332d180747edf27ffb2148b0c5a7 Mon Sep 17 00:00:00 2001 From: nikhil-ctds <151718832+nikhil-ctds@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:25:44 +0530 Subject: [PATCH 25/58] Upgrade Avro to 1.11.4 to address CVE-2024-47561 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0d63e55f..820944e2 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ 1.18.32 com.datastax.oss 3.1.4.2 - 1.11.3 + 1.11.4 3.3.6 1.13.1 1.12.698 From db61b59c724e27ee867a2d927c5eee7c88109220 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Sat, 5 Oct 2024 11:02:45 +0530 Subject: [PATCH 26/58] [maven-release-plugin] prepare release v3.2.2 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 820944e2..de63b4f1 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.2-SNAPSHOT + 3.2.2 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.2 From cdfed3a3cdc5088128dd5efc46f1f505c8100033 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Sat, 5 Oct 2024 11:04:27 +0530 Subject: [PATCH 27/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index de63b4f1..fd91d2ab 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.2 + 3.2.3-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -101,7 +101,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.2 + HEAD From bae0fb1c0527815d2f7ffb33c8c66704d146cfbe Mon Sep 17 00:00:00 2001 From: nikhil-ctds <151718832+nikhil-ctds@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:03:35 +0530 Subject: [PATCH 28/58] Upgrade protobuf3 & async-http-client to address CVE-2024-7254 & CVE-2024-53990 (#11) * Upgrade protobuf3 to 3.25.5 to address CVE-2024-7254 * Include async-http-client with updated version to fix vulnerable transitive dependency. * Add explanation comment and scope. --------- Co-authored-by: ganesh-ctds --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd91d2ab..2c1a2e9e 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,8 @@ 1.26.0 2.17.1 2.4.9 - 3.19.6 + 3.25.5 + 2.12.4 ${protobuf3.version} 1.42.1 ${grpc.version} @@ -168,6 +169,13 @@ + + + org.asynchttpclient + async-http-client + ${asynchttpclient.version} + runtime + com.google.protobuf protobuf-bom From f60387a196fbd5a2bbf4fe678b1d8da3582f9d77 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 13:30:42 +0530 Subject: [PATCH 29/58] [maven-release-plugin] prepare release pulsar-io-cloud-storage-3.2.3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2c1a2e9e..2d624a0d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.3-SNAPSHOT + 3.2.3 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + pulsar-io-cloud-storage-3.2.3 From d9e4c3606513f00dc731ba1c994ef4802cfa78b4 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 13:30:58 +0530 Subject: [PATCH 30/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2d624a0d..5340941e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.3 + 3.2.4-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - pulsar-io-cloud-storage-3.2.3 + HEAD From f86fd31ee6fa0e821373b3fc6a97e4c679305ada Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 14:19:43 +0530 Subject: [PATCH 31/58] Revert "[maven-release-plugin] prepare for next development iteration" This reverts commit d9e4c3606513f00dc731ba1c994ef4802cfa78b4. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5340941e..2d624a0d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.4-SNAPSHOT + 3.2.3 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + pulsar-io-cloud-storage-3.2.3 From b840645cb8c49e628f2971788c9e1c1fd7412b45 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 14:19:47 +0530 Subject: [PATCH 32/58] Revert "[maven-release-plugin] prepare release pulsar-io-cloud-storage-3.2.3" This reverts commit f60387a196fbd5a2bbf4fe678b1d8da3582f9d77. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2d624a0d..2c1a2e9e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.3 + 3.2.3-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - pulsar-io-cloud-storage-3.2.3 + HEAD From 4a135992be77fef7dabf53b08e135255c70599ea Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 14:45:32 +0530 Subject: [PATCH 33/58] [maven-release-plugin] prepare release v3.2.3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2c1a2e9e..2971768c 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.3-SNAPSHOT + 3.2.3 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.3 From b3718450e961f8154e3e286c179975a93457e143 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 29 Apr 2025 14:45:50 +0530 Subject: [PATCH 34/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2971768c..5340941e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.3 + 3.2.4-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.3 + HEAD From 7c0d7b4c204b998e0bdc6409e5b6fcf286247abe Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 6 May 2025 14:41:09 +0530 Subject: [PATCH 35/58] bump parquet version (#13) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5340941e..1342cf64 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 3.1.4.2 1.11.4 3.3.6 - 1.13.1 + 1.15.1 1.12.698 2.16.104 2.3.1 From 2e0e01ac1512f3a3e110ceceeb92c171dc45228e Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 6 May 2025 15:51:49 +0530 Subject: [PATCH 36/58] [maven-release-plugin] prepare release v3.2.4 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1342cf64..b0bfd5b7 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.4-SNAPSHOT + 3.2.4 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.4 From e238adb35f8b700a263197d4a5b672e1fba6ec9c Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Tue, 6 May 2025 15:52:06 +0530 Subject: [PATCH 37/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b0bfd5b7..0e02e60a 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.4 + 3.2.5-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -102,7 +102,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.4 + HEAD From cdea49b3c2320590e3e4ff24d31e9cbf0eb13a8c Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Mon, 8 Dec 2025 11:36:45 +0530 Subject: [PATCH 38/58] Update pom.xml to resolve vulnerabilities --- pom.xml | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 0e02e60a..d139e656 100644 --- a/pom.xml +++ b/pom.xml @@ -44,14 +44,14 @@ 2 - 2.13.2 - 2.13.4.2 + 2.15.4 + 2.15.4 1.18.32 com.datastax.oss 3.1.4.2 1.11.4 - 3.3.6 - 1.15.1 + 3.4.0 + 1.15.2 1.12.698 2.16.104 2.3.1 @@ -61,6 +61,14 @@ 2.4.9 3.25.5 2.12.4 + 2.46 + 3.6.0 + 1.5.4 + 1.1.10.4 + 9.37.4 + 1.11.0 + 2.10.1 + 1.79 ${protobuf3.version} 1.42.1 ${grpc.version} @@ -72,6 +80,7 @@ 3.7.7 2.0.2 1.15.2 + 3.18.0 3.0 @@ -86,7 +95,7 @@ 2.0 1.2.24 5.4.0 - 4.1.86.Final + 4.1.125.Final 0.17.0 @@ -176,6 +185,54 @@ ${asynchttpclient.version} runtime + + org.glassfish.jersey.core + jersey-client + ${jersey-client.version} + runtime + + + dnsjava + dnsjava + ${dnsjava.version} + runtime + + + org.codehaus.jettison + jettison + ${jettison.version} + runtime + + + org.xerial.snappy + snappy-java + ${snappy.version} + runtime + + + com.nimbusds + nimbus-jose-jwt + ${nimbusds.version} + runtime + + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + runtime + + + org.apache.commons + commons-configuration2 + ${commons-configuration2.version} + runtime + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + runtime + com.google.protobuf protobuf-bom @@ -355,10 +412,12 @@ com.fasterxml.jackson.core jackson-databind + ${jackson-databind.version} com.fasterxml.jackson.dataformat jackson-dataformat-yaml + ${jackson.version} ${pulsar.groupId} @@ -397,6 +456,7 @@ org.apache.parquet parquet-avro + ${parquet.version} org.apache.avro @@ -407,18 +467,22 @@ com.google.protobuf protobuf-java + ${protobuf3.version} com.google.protobuf protobuf-java-util + ${protobuf3.version} org.apache.parquet parquet-protobuf + ${parquet.version} org.apache.parquet parquet-hadoop + ${parquet.version} org.apache.avro @@ -429,6 +493,7 @@ org.apache.avro avro + ${avro.version} org.apache.hadoop @@ -644,7 +709,7 @@ org.apache.commons commons-lang3 - 3.11 + ${commons-lang3.version} ${pulsar.groupId} From d1fba747cf4a2de9ef8aea4f64fcfb1803ce261b Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Tue, 9 Dec 2025 12:32:04 +0530 Subject: [PATCH 39/58] [maven-release-plugin] prepare release v3.2.5 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d139e656..685e4c73 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.5-SNAPSHOT + 3.2.5 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -111,7 +111,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.5 From a0bc4e3ed4de5dbdc7ac0a20fa403747ecf684c0 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Tue, 9 Dec 2025 12:32:20 +0530 Subject: [PATCH 40/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 685e4c73..5d3e697c 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.5 + 3.2.6-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -111,7 +111,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.5 + HEAD From a32eac3328d23d3f7bf10076ef4df9ba44257a59 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Thu, 18 Dec 2025 16:18:03 +0530 Subject: [PATCH 41/58] Upgrade dependencies to address multiple CVEs (#15) * Upgrade dependencies to resolve vulnerabilities & Remove duplicate dependencies * Revert netty.version back to resolve failing integration tests --- pom.xml | 70 ++++++++++++++++++--------------------------------------- 1 file changed, 22 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 5d3e697c..3a7a88d3 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ com.datastax.oss 3.1.4.2 1.11.4 - 3.4.0 + 3.4.1 1.15.2 1.12.698 2.16.104 @@ -68,6 +68,7 @@ 9.37.4 1.11.0 2.10.1 + 2.14.0 1.79 ${protobuf3.version} 1.42.1 @@ -185,54 +186,74 @@ ${asynchttpclient.version} runtime + org.glassfish.jersey.core jersey-client ${jersey-client.version} runtime + dnsjava dnsjava ${dnsjava.version} runtime + org.codehaus.jettison jettison ${jettison.version} runtime + org.xerial.snappy snappy-java ${snappy.version} runtime + com.nimbusds nimbus-jose-jwt ${nimbusds.version} runtime + commons-beanutils commons-beanutils ${commons-beanutils.version} runtime + org.apache.commons commons-configuration2 ${commons-configuration2.version} runtime + org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} runtime + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + runtime + + + commons-io + commons-io + ${commons-io.version} + com.google.protobuf protobuf-bom @@ -456,7 +477,6 @@ org.apache.parquet parquet-avro - ${parquet.version} org.apache.avro @@ -467,22 +487,18 @@ com.google.protobuf protobuf-java - ${protobuf3.version} com.google.protobuf protobuf-java-util - ${protobuf3.version} org.apache.parquet parquet-protobuf - ${parquet.version} org.apache.parquet parquet-hadoop - ${parquet.version} org.apache.avro @@ -493,34 +509,6 @@ org.apache.avro avro - ${avro.version} - - - org.apache.hadoop - hadoop-client - 3.3.3 - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-yarn-client - - - org.apache.hadoop - hadoop-yarn-common - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - org.apache.hadoop - hadoop-hdfs-client - - org.apache.hadoop @@ -711,20 +699,11 @@ commons-lang3 ${commons-lang3.version} - - ${pulsar.groupId} - pulsar-io-common - javax.xml.bind jaxb-api ${jaxb-api} - - ${pulsar.groupId} - pulsar-client-original - ${pulsar.version} - ${pulsar.groupId} pulsar-client-admin-original @@ -736,11 +715,6 @@ 5.7.0 test - - ${pulsar.groupId} - pulsar-common - ${pulsar.version} - From dea6ad483cfdf56b914874ca1490689d89b5e2bb Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 19 Dec 2025 17:28:13 +0530 Subject: [PATCH 42/58] [maven-release-plugin] prepare release v3.2.6 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3a7a88d3..4d451c83 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.6-SNAPSHOT + 3.2.6 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -112,7 +112,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.6 From ccb3fb6924d3b9e26c945b9197300f846fe17cdb Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 19 Dec 2025 17:28:30 +0530 Subject: [PATCH 43/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4d451c83..bfda4a2c 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.6 + 3.2.7-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -112,7 +112,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.6 + HEAD From 3afe37ec973563d125aabbf7504fbb741700aa73 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Mon, 9 Mar 2026 18:02:32 +0530 Subject: [PATCH 44/58] Bump netty & aircompressor versions to fix CVE (#16) * bump netty version & override aircompressor version * Update pulsar version to 3.1.4.22 to fix CI failures. * bump netty version. * bump jackson version. --- pom.xml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index bfda4a2c..d316de5b 100644 --- a/pom.xml +++ b/pom.xml @@ -44,11 +44,11 @@ 2 - 2.15.4 - 2.15.4 + 2.18.6 + 2.18.6 1.18.32 com.datastax.oss - 3.1.4.2 + 3.1.4.22 1.11.4 3.4.1 1.15.2 @@ -69,6 +69,7 @@ 1.11.0 2.10.1 2.14.0 + 2.0.3 1.79 ${protobuf3.version} 1.42.1 @@ -96,7 +97,7 @@ 2.0 1.2.24 5.4.0 - 4.1.125.Final + 4.1.130.Final 0.17.0 @@ -254,6 +255,11 @@ commons-io ${commons-io.version} + + io.airlift + aircompressor + ${aircompressor.version} + com.google.protobuf protobuf-bom From 21ada94cbabd3a1e8a73350560fd44e2295844cb Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Thu, 12 Mar 2026 20:12:27 +0530 Subject: [PATCH 45/58] [maven-release-plugin] prepare release v3.2.7 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d316de5b..1b4ab0f5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.7-SNAPSHOT + 3.2.7 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -113,7 +113,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.7 From 31b82ddc3a52cee19a88462d05034e51ede72a74 Mon Sep 17 00:00:00 2001 From: ganesh-ctds Date: Thu, 12 Mar 2026 20:12:45 +0530 Subject: [PATCH 46/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1b4ab0f5..876b3d95 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.7 + 3.2.8-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -113,7 +113,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.7 + HEAD From 9bba0d1af129e5be35440fa25e9c47ec95754ba1 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Tue, 28 Apr 2026 15:36:46 +0530 Subject: [PATCH 47/58] Upgrade dependencies to remediate CVEs (#17) --- pom.xml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 876b3d95..2d05c9d9 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ 2.18.6 - 2.18.6 + 2.21.2 1.18.32 com.datastax.oss 3.1.4.22 @@ -60,7 +60,7 @@ 2.17.1 2.4.9 3.25.5 - 2.12.4 + 2.14.5 2.46 3.6.0 1.5.4 @@ -70,7 +70,7 @@ 2.10.1 2.14.0 2.0.3 - 1.79 + 1.84 ${protobuf3.version} 1.42.1 ${grpc.version} @@ -97,7 +97,8 @@ 2.0 1.2.24 5.4.0 - 4.1.130.Final + 4.1.132.Final + 2.1.0 0.17.0 @@ -250,6 +251,13 @@ ${bouncycastle.version} runtime + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin-stdlib.version} + runtime + commons-io commons-io From 77636c690c42ce9dbb3229216090e33b391b5d2d Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Wed, 29 Apr 2026 12:35:56 +0530 Subject: [PATCH 48/58] [maven-release-plugin] prepare release v3.2.8 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2d05c9d9..a243749d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.8-SNAPSHOT + 3.2.8 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -114,7 +114,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.8 From aeafcc4f749c251cdc0235d1fd18abfb36358002 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Wed, 29 Apr 2026 12:36:15 +0530 Subject: [PATCH 49/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a243749d..3f328184 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.8 + 3.2.9-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -114,7 +114,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.8 + HEAD From 99a5136b4fd9048bebf1bd98f0b9ef0bddedfe92 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Tue, 9 Jun 2026 13:34:47 +0530 Subject: [PATCH 50/58] Bump dependencies to remediate multiple CVEs (#18) --- pom.xml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 3f328184..5ba52732 100644 --- a/pom.xml +++ b/pom.xml @@ -60,14 +60,14 @@ 2.17.1 2.4.9 3.25.5 - 2.14.5 + 2.15.0 2.46 3.6.0 1.5.4 1.1.10.4 9.37.4 1.11.0 - 2.10.1 + 2.15.0 2.14.0 2.0.3 1.84 @@ -97,9 +97,10 @@ 2.0 1.2.24 5.4.0 - 4.1.132.Final + 4.1.133.Final 2.1.0 - 0.17.0 + 0.23.0 + 5.3.5 @@ -258,6 +259,12 @@ ${kotlin-stdlib.version} runtime + + + org.apache.httpcomponents.core5 + httpcore5-h2 + ${httpcore5-h2.version} + commons-io commons-io From 3fd806dfe89f58d7fc5f8206a593c8f35ea740df Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 12 Jun 2026 12:50:00 +0530 Subject: [PATCH 51/58] Upgrade netty to 4.1.135.Final to remediate CVE-2026-44249, CVE-2026-45416 (#19) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ba52732..31380501 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ 2.0 1.2.24 5.4.0 - 4.1.133.Final + 4.1.135.Final 2.1.0 0.23.0 5.3.5 From 11856acef56316ce9c9cd81dc4f13c3475c4279f Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 12 Jun 2026 12:56:31 +0530 Subject: [PATCH 52/58] [maven-release-plugin] prepare release v3.2.9 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 31380501..9d395361 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.9-SNAPSHOT + 3.2.9 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -115,7 +115,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.9 From e628a7d9db733c7c460ade8dd46b43f5dc2b6579 Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 12 Jun 2026 12:56:49 +0530 Subject: [PATCH 53/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9d395361..9d65055a 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.9 + 3.2.10-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -115,7 +115,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.9 + HEAD From 82cd4fc8754b415ddb9e7db7c2c6c43dd4675430 Mon Sep 17 00:00:00 2001 From: sandeep-ctds Date: Mon, 27 Jul 2026 19:09:37 +0530 Subject: [PATCH 54/58] Updated pulsar version to latest in 3.1.4.x for bouncycastle vulnerability --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d65055a..ef71b9b1 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 2.21.2 1.18.32 com.datastax.oss - 3.1.4.22 + 3.1.4.28 1.11.4 3.4.1 1.15.2 From 9ff4f5e33bf71308c8cc4ba567681f21e18f5d9d Mon Sep 17 00:00:00 2001 From: sandeep-ctds Date: Mon, 3 Aug 2026 12:48:39 +0530 Subject: [PATCH 55/58] [maven-release-plugin] prepare release v3.2.10 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ef71b9b1..176b4cff 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.10-SNAPSHOT + 3.2.10 Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -115,7 +115,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - HEAD + v3.2.10 From 9e3e61319ad0cc14d3bb219e65c6b94f31d266c1 Mon Sep 17 00:00:00 2001 From: sandeep-ctds Date: Mon, 3 Aug 2026 12:48:57 +0530 Subject: [PATCH 56/58] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 176b4cff..91b1176d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ io.streamnative.ecosystem pulsar-io-cloud-storage - 3.2.10 + 3.2.11-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Cloud Storage Project Cloud Storage Connector integrates Apache Pulsar with Cloud Storage. @@ -115,7 +115,7 @@ scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git scm:git:git@github.com:datastax/pulsar-io-cloud-storage.git https://github.com/datastax/pulsar-io-cloud-storage - v3.2.10 + HEAD From a3f4c8069baa0fe970601e9c4ef0997b0160136b Mon Sep 17 00:00:00 2001 From: sandeep-ctds Date: Thu, 20 Aug 2026 17:36:17 +0530 Subject: [PATCH 57/58] Added sonarqube scanning to PR, commit and workflows --- .github/workflows/sonarqube.yaml | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/sonarqube.yaml diff --git a/.github/workflows/sonarqube.yaml b/.github/workflows/sonarqube.yaml new file mode 100644 index 00000000..694cfee4 --- /dev/null +++ b/.github/workflows/sonarqube.yaml @@ -0,0 +1,107 @@ +name: SonarQube Scan + +on: + push: + branches: + - 4.0_ds + - 3.2_ds + pull_request: + types: [opened, synchronize, reopened] + branches: + - 4.0_ds + - 3.2_ds + workflow_dispatch: + inputs: + scan_type: + description: 'Type of scan to run' + required: true + type: choice + options: + - branch + - pr + default: branch + ref: + description: 'Commit SHA, branch, or tag to check out and scan (leave blank to use the default branch)' + required: false + type: string + default: '' + +concurrency: + group: sonarqube-${{ github.ref }} + cancel-in-progress: true + +env: + SONAR_HOST_URL: https://sonarqube-prod.whitewater.ibm.com + TRUSTSTORE_PATH: ./ibm_castorevpcprod + # Maven tuning — mirrors settings used across the rest of the CI workflows. + MAVEN_OPTS: >- + -Xss1500k + -Xmx2048m + -XX:+UnlockDiagnosticVMOptions + -XX:+IgnoreUnrecognizedVMOptions + -XX:GCLockerRetryAllocationCount=100 + -Daether.connector.http.reuseConnections=false + -Daether.connector.requestTimeout=60000 + -Dhttp.keepAlive=false + -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.class=standard + -Dmaven.wagon.http.retryHandler.count=3 + -Dmaven.wagon.http.retryHandler.requestSentEnabled=true + -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard + -Dmaven.wagon.rto=60000 + +jobs: + sonarqube-scan: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # Full history required for accurate blame and new-code period detection. + fetch-depth: 0 + # When triggered manually with a specific ref (commit SHA, branch, tag), + # check out that ref; otherwise fall back to the default event ref. + ref: ${{ (github.event_name == 'workflow_dispatch' && inputs.ref != '') && inputs.ref || github.ref }} + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: '17' + cache: maven + + # Decode the IBM truststore from the base64 secret and write it to disk. + # Secret is bound to an env var to avoid shell-injection (S7636). + - name: Decode IBM SonarQube truststore + env: + TRUSTSTORE_B64: ${{ secrets.IBM_SONARQUBE_CERT_TRUSTSTORE_BASE64 }} + run: | + echo "$TRUSTSTORE_B64" | base64 --decode > "${{ env.TRUSTSTORE_PATH }}" + + # Produce .class files required by SonarQube's Java bytecode analyser. + - name: Build + run: mvn -B package -DskipTests --no-transfer-progress + + # Pinned to v8.2.1 (SHA: 22918119ff8e1ca75a623e15c8296b6ea4fbe28f). + - name: Run SonarQube scan + uses: SonarSource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f + env: + SONAR_TOKEN: ${{ secrets.IBM_SONARQUBE_API_TOKEN }} + SONAR_HOST_URL: ${{ env.SONAR_HOST_URL }} + with: + args: > + -D sonar.projectKey=${{ vars.SONAR_PROJECT_KEY }} + -D sonar.projectName=${{ vars.SONAR_PROJECT_NAME }} + -D sonar.java.source=17 + -D sonar.sources=. + -D sonar.exclusions=**/proto/**,**/shade/**,**/shaded/**,**/target/** + -D sonar.java.binaries=**/target/classes + -D sonar.scanner.truststorePath=${{ env.TRUSTSTORE_PATH }} + -D sonar.scanner.truststorePassword=${{ secrets.IBM_SONARQUBE_CERT_PASSWORD }} + ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.scan_type == 'pr')) + && format('-Dsonar.pullrequest.key={0} -Dsonar.pullrequest.branch={1} -Dsonar.pullrequest.base={2}', + github.event.pull_request.number, github.head_ref, github.base_ref) + || format('-Dsonar.branch.name={0}', + (github.event_name == 'workflow_dispatch' && inputs.ref != '') && inputs.ref || github.ref_name) }} \ No newline at end of file From 75d1c639d692c5b9c532676dc966b4cabb685f0f Mon Sep 17 00:00:00 2001 From: sandeep-ctds Date: Thu, 20 Aug 2026 17:38:07 +0530 Subject: [PATCH 58/58] Updated sonarqube.yaml to use sonar_host_url from vars --- .github/workflows/sonarqube.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sonarqube.yaml b/.github/workflows/sonarqube.yaml index 694cfee4..35506dfa 100644 --- a/.github/workflows/sonarqube.yaml +++ b/.github/workflows/sonarqube.yaml @@ -31,7 +31,6 @@ concurrency: cancel-in-progress: true env: - SONAR_HOST_URL: https://sonarqube-prod.whitewater.ibm.com TRUSTSTORE_PATH: ./ibm_castorevpcprod # Maven tuning — mirrors settings used across the rest of the CI workflows. MAVEN_OPTS: >- @@ -89,7 +88,7 @@ jobs: uses: SonarSource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f env: SONAR_TOKEN: ${{ secrets.IBM_SONARQUBE_API_TOKEN }} - SONAR_HOST_URL: ${{ env.SONAR_HOST_URL }} + SONAR_HOST_URL: ${{ vars.SONAR_HOST_URL }} with: args: > -D sonar.projectKey=${{ vars.SONAR_PROJECT_KEY }}