From 6a14d645fc418d2690fc81a20f01f4d1a1974b3f Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 05:54:42 +0000 Subject: [PATCH 01/20] initial 4 byte supoty with ITs --- .../sql/mysql_collation_order_query.sql | 61 +++++---- .../sql/postgresql_collation_order_query.sql | 2 +- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 119 ++++++++++++++++++ ...reSQLSourceDbToSpanner4ByteStringPKIT.java | 119 ++++++++++++++++++ 4 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java create mode 100644 v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index bdd2168447..93f90df8cb 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -33,52 +33,51 @@ SET @byte_literals = CONCAT( -- Four byte code points. SET @four_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h, t4.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t4 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' + 'SELECT CONCAT(t1.h, t2.h, t3.h, t4.h) AS hex_val ', + 'FROM (', @byte_literals, ') AS t1 ', + 'CROSS JOIN (', @byte_literals, ') AS t2 ', + 'CROSS JOIN (', @byte_literals, ') AS t3 ', + 'CROSS JOIN (', @byte_literals, ') AS t4 ', + 'WHERE t1.h BETWEEN ''f0'' AND ''f4'' AND t2.h BETWEEN ''80'' AND ''bf'' AND t3.h BETWEEN ''80'' AND ''bf'' AND t4.h BETWEEN ''80'' AND ''bf''' ); -- Three byte code points. SET @three_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' + 'SELECT CONCAT(t1.h, t2.h, t3.h) AS hex_val ', + 'FROM (', @byte_literals, ') AS t1 ', + 'CROSS JOIN (', @byte_literals, ') AS t2 ', + 'CROSS JOIN (', @byte_literals, ') AS t3 ', + 'WHERE t1.h BETWEEN ''e0'' AND ''ef'' AND t2.h BETWEEN ''80'' AND ''bf'' AND t3.h BETWEEN ''80'' AND ''bf''' ); -- Two byte code points. SET @two_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' + 'SELECT CONCAT(t1.h, t2.h) AS hex_val ', + 'FROM (', @byte_literals, ') AS t1 ', + 'CROSS JOIN (', @byte_literals, ') AS t2 ', + 'WHERE t1.h BETWEEN ''c2'' AND ''df'' AND t2.h BETWEEN ''80'' AND ''bf''' ); -- Single byte code points. SET @one_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(t1.h) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1', - ') AS dt ', -- derived table - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' + 'SELECT t1.h AS hex_val ', + 'FROM (', @byte_literals, ') AS t1 ', + 'WHERE t1.h BETWEEN ''00'' AND ''7f''' ); +SET @all_utf8_hex = CONCAT(@four_byte_codepoints, ' UNION ALL ', @three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); + -- all variable length code points representing a single character within the @db_charset from length 0 till 4. -SET @charset_chars = CONCAT(@three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); +SET @charset_chars = CONCAT( + '(SELECT charset_char FROM ( ', + 'SELECT CONVERT(CONVERT(UNHEX(hex_val) USING utf8mb4) USING ', @db_charset, ') AS charset_char, ', + 'CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char ', + 'FROM (', @all_utf8_hex, ') AS all_chars ', + 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''eda080'' AND ''edbfbf'' ', + ') AS valid_utf8_chars ', + 'WHERE charset_char IS NOT NULL AND (charset_char != ''?'' OR utf8_char = ''?'') ', + 'AND CHAR_LENGTH(charset_char) <= 1)' +); SET @SPACE=CONCAT('CONVERT('' '' USING ', @db_charset,')'); SET @ALPHABET=CONCAT('CONVERT(''a'' USING ', @db_charset,')'); diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql index d2bd311db1..032fa406ff 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql @@ -5,7 +5,7 @@ CREATE OR REPLACE FUNCTION pg_temp.safe_convert_from(codepoint int8, charset text) RETURNS return_type_replacement_tag AS ' BEGIN - RETURN convert_from(decode(to_hex(codepoint), ''hex''), charset); + RETURN convert_from(convert_to(chr(codepoint::integer), charset), charset); EXCEPTION WHEN OTHERS THEN RETURN NULL; END; diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java new file mode 100644 index 0000000000..ba8cfc7eed --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.beam.it.jdbc.MySQLResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SourceDbToSpanner.class) +@RunWith(JUnit4.class) +public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { + private static PipelineLauncher.LaunchInfo jobInfo; + + public static MySQLResourceManager mySQLResourceManager; + public static SpannerResourceManager spannerResourceManager; + + private static final String TABLE = "table4bytepk"; + private static final String ID = "id"; + private static final String DESCRIPTION = "description"; + private static final String SPANNER_DDL_RESOURCE = + "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; + + private JDBCResourceManager.JDBCSchema getMySQLSchema() { + HashMap columns = new HashMap<>(); + columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL"); + columns.put(DESCRIPTION, "VARCHAR(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ID); + } + + private List> getMySQLData() { + List> data = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put(ID, "😀"); + row1.put(DESCRIPTION, "Grinning Face"); + data.add(row1); + + Map row2 = new HashMap<>(); + row2.put(ID, "😁"); + row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); + data.add(row2); + + Map row3 = new HashMap<>(); + row3.put(ID, "😂"); + row3.put(DESCRIPTION, "Face with Tears of Joy"); + data.add(row3); + + return data; + } + + @Before + public void setUp() { + mySQLResourceManager = setUpMySQLResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources(spannerResourceManager, mySQLResourceManager); + } + + @Test + public void testMySqlToSpanner() throws IOException { + List> mySQLData = getMySQLData(); + mySQLResourceManager.createTable(TABLE, getMySQLSchema()); + mySQLResourceManager.write(TABLE, mySQLData); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + mySQLResourceManager, + spannerResourceManager, + null, + null); + PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo)); + assertThatResult(result).isLaunchFinished(); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(TABLE, ID, DESCRIPTION)) + .hasRecordsUnorderedCaseInsensitiveColumns(mySQLData); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java new file mode 100644 index 0000000000..bcb44dd579 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.beam.it.jdbc.PostgresResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SourceDbToSpanner.class) +@RunWith(JUnit4.class) +public class PostgreSQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PostgresResourceManager postgreSQLResourceManager; + public static SpannerResourceManager spannerResourceManager; + + private static final String TABLE = "table4bytepk"; + private static final String ID = "id"; + private static final String DESCRIPTION = "description"; + private static final String SPANNER_DDL_RESOURCE = + "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; + + private JDBCResourceManager.JDBCSchema getPostgreSQLSchema() { + HashMap columns = new HashMap<>(); + columns.put(ID, "VARCHAR(200) NOT NULL"); + columns.put(DESCRIPTION, "VARCHAR(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ID); + } + + private List> getPostgreSQLData() { + List> data = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put(ID, "😀"); + row1.put(DESCRIPTION, "Grinning Face"); + data.add(row1); + + Map row2 = new HashMap<>(); + row2.put(ID, "😁"); + row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); + data.add(row2); + + Map row3 = new HashMap<>(); + row3.put(ID, "😂"); + row3.put(DESCRIPTION, "Face with Tears of Joy"); + data.add(row3); + + return data; + } + + @Before + public void setUp() { + postgreSQLResourceManager = setUpPostgreSQLResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources(spannerResourceManager, postgreSQLResourceManager); + } + + @Test + public void testPostgreSQLToSpanner() throws IOException { + List> postgreSQLData = getPostgreSQLData(); + postgreSQLResourceManager.createTable(TABLE, getPostgreSQLSchema()); + postgreSQLResourceManager.write(TABLE, postgreSQLData); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + postgreSQLResourceManager, + spannerResourceManager, + null, + null); + PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo)); + assertThatResult(result).isLaunchFinished(); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(TABLE, ID, DESCRIPTION)) + .hasRecordsUnorderedCaseInsensitiveColumns(postgreSQLData); + } +} From 3960cf2ec5299115bcb64f987cccd605c3835754 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:05:16 +0000 Subject: [PATCH 02/20] upadte --- .../MySQLSourceDbToSpanner4ByteStringPKIT.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index ba8cfc7eed..5e6c932c8e 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -96,7 +96,16 @@ public void cleanUp() { public void testMySqlToSpanner() throws IOException { List> mySQLData = getMySQLData(); mySQLResourceManager.createTable(TABLE, getMySQLSchema()); - mySQLResourceManager.write(TABLE, mySQLData); + + // Insert 4-byte characters directly into the database. + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES ('😀', 'Grinning Face')"); + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + + TABLE + + "(id, description) VALUES ('😁', 'Beaming Face with Smiling Eyes')"); + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES ('😂', 'Face with Tears of Joy')"); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); From 01559dbb467c28237eb616485ad7078780c7cc4b Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:08:05 +0000 Subject: [PATCH 03/20] upadte --- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 5e6c932c8e..92b04aa02e 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -97,15 +97,23 @@ public void testMySqlToSpanner() throws IOException { List> mySQLData = getMySQLData(); mySQLResourceManager.createTable(TABLE, getMySQLSchema()); - // Insert 4-byte characters directly into the database. - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('😀', 'Grinning Face')"); - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " - + TABLE - + "(id, description) VALUES ('😁', 'Beaming Face with Smiling Eyes')"); - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('😂', 'Face with Tears of Joy')"); + // The default MySQL Testcontainers JDBC connection uses utf8mb3 or latin1, causing 4-byte + // emojis to be downcasted to '?'. We bypass the manager to inject 'characterEncoding=UTF-8'. + String jdbcUrl = mySQLResourceManager.getUri() + "?characterEncoding=UTF-8"; + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + jdbcUrl, mySQLResourceManager.getUsername(), mySQLResourceManager.getPassword()); + java.sql.Statement stmt = conn.createStatement()) { + stmt.executeUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES ('😀', 'Grinning Face')"); + stmt.executeUpdate( + "INSERT INTO " + + TABLE + + "(id, description) VALUES ('😁', 'Beaming Face with Smiling Eyes')"); + stmt.executeUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES ('😂', 'Face with Tears of Joy')"); + } catch (java.sql.SQLException e) { + throw new RuntimeException("Failed to insert 4-byte characters", e); + } createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); From ed558adc3b42499e82d4cc4309ab10c73c91a3df Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:11:41 +0000 Subject: [PATCH 04/20] upadte --- .../v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 92b04aa02e..db0d1211a8 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -103,6 +103,11 @@ public void testMySqlToSpanner() throws IOException { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( jdbcUrl, mySQLResourceManager.getUsername(), mySQLResourceManager.getPassword()); java.sql.Statement stmt = conn.createStatement()) { + + // Force the MySQL connection to use utf8mb4 for this session, ensuring the driver + // doesn't downcast the emojis to '?' before execution. + stmt.executeUpdate("SET NAMES utf8mb4"); + stmt.executeUpdate( "INSERT INTO " + TABLE + "(id, description) VALUES ('😀', 'Grinning Face')"); stmt.executeUpdate( From 4f094ad964b42f52e79e006a578bca4225b7184a Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:14:02 +0000 Subject: [PATCH 05/20] upadte --- .../MySQLSourceDbToSpanner4ByteStringPKIT.java | 12 ++++++------ .../PostgreSQLSourceDbToSpanner4ByteStringPKIT.java | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index db0d1211a8..773714360f 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -64,17 +64,17 @@ private List> getMySQLData() { List> data = new ArrayList<>(); Map row1 = new HashMap<>(); - row1.put(ID, "😀"); + row1.put(ID, "\uD83D\uDE00"); row1.put(DESCRIPTION, "Grinning Face"); data.add(row1); Map row2 = new HashMap<>(); - row2.put(ID, "😁"); + row2.put(ID, "\uD83D\uDE01"); row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); data.add(row2); Map row3 = new HashMap<>(); - row3.put(ID, "😂"); + row3.put(ID, "\uD83D\uDE02"); row3.put(DESCRIPTION, "Face with Tears of Joy"); data.add(row3); @@ -109,13 +109,13 @@ public void testMySqlToSpanner() throws IOException { stmt.executeUpdate("SET NAMES utf8mb4"); stmt.executeUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('😀', 'Grinning Face')"); + "INSERT INTO " + TABLE + "(id, description) VALUES ('\uD83D\uDE00', 'Grinning Face')"); stmt.executeUpdate( "INSERT INTO " + TABLE - + "(id, description) VALUES ('😁', 'Beaming Face with Smiling Eyes')"); + + "(id, description) VALUES ('\uD83D\uDE01', 'Beaming Face with Smiling Eyes')"); stmt.executeUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('😂', 'Face with Tears of Joy')"); + "INSERT INTO " + TABLE + "(id, description) VALUES ('\uD83D\uDE02', 'Face with Tears of Joy')"); } catch (java.sql.SQLException e) { throw new RuntimeException("Failed to insert 4-byte characters", e); } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java index bcb44dd579..657d10cdbd 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java @@ -64,17 +64,17 @@ private List> getPostgreSQLData() { List> data = new ArrayList<>(); Map row1 = new HashMap<>(); - row1.put(ID, "😀"); + row1.put(ID, "\uD83D\uDE00"); row1.put(DESCRIPTION, "Grinning Face"); data.add(row1); Map row2 = new HashMap<>(); - row2.put(ID, "😁"); + row2.put(ID, "\uD83D\uDE01"); row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); data.add(row2); Map row3 = new HashMap<>(); - row3.put(ID, "😂"); + row3.put(ID, "\uD83D\uDE02"); row3.put(DESCRIPTION, "Face with Tears of Joy"); data.add(row3); From 3520000d919d8e40689396f382f5cf0bbb1d242c Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:16:03 +0000 Subject: [PATCH 06/20] upadte --- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 773714360f..48262d4671 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -97,28 +97,17 @@ public void testMySqlToSpanner() throws IOException { List> mySQLData = getMySQLData(); mySQLResourceManager.createTable(TABLE, getMySQLSchema()); - // The default MySQL Testcontainers JDBC connection uses utf8mb3 or latin1, causing 4-byte - // emojis to be downcasted to '?'. We bypass the manager to inject 'characterEncoding=UTF-8'. - String jdbcUrl = mySQLResourceManager.getUri() + "?characterEncoding=UTF-8"; - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - jdbcUrl, mySQLResourceManager.getUsername(), mySQLResourceManager.getPassword()); - java.sql.Statement stmt = conn.createStatement()) { - - // Force the MySQL connection to use utf8mb4 for this session, ensuring the driver - // doesn't downcast the emojis to '?' before execution. - stmt.executeUpdate("SET NAMES utf8mb4"); - - stmt.executeUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('\uD83D\uDE00', 'Grinning Face')"); - stmt.executeUpdate( - "INSERT INTO " - + TABLE - + "(id, description) VALUES ('\uD83D\uDE01', 'Beaming Face with Smiling Eyes')"); - stmt.executeUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES ('\uD83D\uDE02', 'Face with Tears of Joy')"); - } catch (java.sql.SQLException e) { - throw new RuntimeException("Failed to insert 4-byte characters", e); - } + // The MySQL JDBC driver in this environment stubbornly downcasts 4-byte characters to '?' + // even with characterEncoding=UTF-8 and SET NAMES utf8mb4. To successfully insert them, + // we must bypass JDBC string encoding by passing the raw UTF-8 hex bytes directly to MySQL. + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES (UNHEX('F09F9880'), 'Grinning Face')"); + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + + TABLE + + "(id, description) VALUES (UNHEX('F09F9881'), 'Beaming Face with Smiling Eyes')"); + mySQLResourceManager.runSQLUpdate( + "INSERT INTO " + TABLE + "(id, description) VALUES (UNHEX('F09F9882'), 'Face with Tears of Joy')"); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); From cae29d0893547284f76eef237c961f56bb185029 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 10 Jul 2026 07:18:14 +0000 Subject: [PATCH 07/20] upadte --- .../sql/mysql_collation_order_query.sql | 3 ++- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 23 ++++++------------- .../spanner-schema.sql | 4 ++++ 3 files changed, 13 insertions(+), 17 deletions(-) create mode 100644 v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index 93f90df8cb..7fc3e89b2b 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -71,7 +71,8 @@ SET @all_utf8_hex = CONCAT(@four_byte_codepoints, ' UNION ALL ', @three_byte_cod SET @charset_chars = CONCAT( '(SELECT charset_char FROM ( ', 'SELECT CONVERT(CONVERT(UNHEX(hex_val) USING utf8mb4) USING ', @db_charset, ') AS charset_char, ', - 'CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char ', + 'CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char, ', + 'hex_val ', 'FROM (', @all_utf8_hex, ') AS all_chars ', 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''eda080'' AND ''edbfbf'' ', ') AS valid_utf8_chars ', diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 48262d4671..3ed99e994d 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -30,7 +30,7 @@ import org.apache.beam.it.gcp.spanner.SpannerResourceManager; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.jdbc.JDBCResourceManager; -import org.apache.beam.it.jdbc.MySQLResourceManager; +import org.apache.beam.it.gcp.cloudsql.CloudMySQLResourceManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -44,7 +44,7 @@ public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { private static PipelineLauncher.LaunchInfo jobInfo; - public static MySQLResourceManager mySQLResourceManager; + public static CloudMySQLResourceManager mySQLResourceManager; public static SpannerResourceManager spannerResourceManager; private static final String TABLE = "table4bytepk"; @@ -53,9 +53,11 @@ public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBa private static final String SPANNER_DDL_RESOURCE = "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; + + private JDBCResourceManager.JDBCSchema getMySQLSchema() { HashMap columns = new HashMap<>(); - columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL"); + columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL"); columns.put(DESCRIPTION, "VARCHAR(200)"); return new JDBCResourceManager.JDBCSchema(columns, ID); } @@ -83,7 +85,7 @@ private List> getMySQLData() { @Before public void setUp() { - mySQLResourceManager = setUpMySQLResourceManager(); + mySQLResourceManager = setUpCloudMySQLResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); } @@ -96,18 +98,7 @@ public void cleanUp() { public void testMySqlToSpanner() throws IOException { List> mySQLData = getMySQLData(); mySQLResourceManager.createTable(TABLE, getMySQLSchema()); - - // The MySQL JDBC driver in this environment stubbornly downcasts 4-byte characters to '?' - // even with characterEncoding=UTF-8 and SET NAMES utf8mb4. To successfully insert them, - // we must bypass JDBC string encoding by passing the raw UTF-8 hex bytes directly to MySQL. - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES (UNHEX('F09F9880'), 'Grinning Face')"); - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " - + TABLE - + "(id, description) VALUES (UNHEX('F09F9881'), 'Beaming Face with Smiling Eyes')"); - mySQLResourceManager.runSQLUpdate( - "INSERT INTO " + TABLE + "(id, description) VALUES (UNHEX('F09F9882'), 'Face with Tears of Joy')"); + mySQLResourceManager.write(TABLE, mySQLData); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); diff --git a/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql new file mode 100644 index 0000000000..146aed9c34 --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE table4bytepk ( + id STRING(200), + description STRING(200) +) PRIMARY KEY (id); From d035c667cd5c64d0fc005888b35fc2306a90aa28 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Mon, 13 Jul 2026 06:02:16 +0000 Subject: [PATCH 08/20] char to string --- .../stringmapper/CollationIndex.java | 20 +++--- .../stringmapper/CollationMapper.java | 45 ++++++++------ .../stringmapper/CollationOrderRow.java | 29 +++------ .../sql/mysql_collation_order_query.sql | 62 +++++++++---------- .../BoundaryTypeMapperImplTest.java | 12 ++-- .../stringmapper/CollationIndexTest.java | 46 +++++++------- .../stringmapper/CollationMapperTest.java | 42 ++++++------- .../stringmapper/CollationOrderRowTest.java | 41 +++--------- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 4 +- 9 files changed, 136 insertions(+), 165 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java index 789bbe7446..64a874f407 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java @@ -43,7 +43,7 @@ public abstract class CollationIndex implements Serializable { * Map of character to it's index position based on collation order. Helps us map a string to big * integer. */ - public abstract ImmutableMap characterToIndex(); + public abstract ImmutableMap characterToIndex(); /** * Map if Index back to character based on collation order. Helps us unmap a big integer to @@ -51,7 +51,7 @@ public abstract class CollationIndex implements Serializable { * case-insensitive collations, 'a' and 'A' will have the same index in {@link * #characterToIndex()} and {@link #indexToCharacter()} will map the index to 'A'. */ - public abstract ImmutableMap indexToCharacter(); + public abstract ImmutableMap indexToCharacter(); public static CollationIndex.Builder builder() { return new AutoValue_CollationIndex.Builder(); @@ -61,11 +61,11 @@ public long getCharsetSize() { return indexToCharacter().size(); } - public long getOrdinalPosition(Character c) { + public long getOrdinalPosition(String c) { return characterToIndex().get(c); } - public Character getCharacterFromPosition(Long position) { + public String getCharacterFromPosition(Long position) { return indexToCharacter().get(position); } @@ -80,15 +80,15 @@ public abstract static class Builder { abstract CollationIndexType indexType(); - private Map charToIndexCache = new HashMap<>(); - private Map indexToCharacterCache = new HashMap<>(); - private Map indexToCharacterReverseCache = new HashMap<>(); + private Map charToIndexCache = new HashMap<>(); + private Map indexToCharacterCache = new HashMap<>(); + private Map indexToCharacterReverseCache = new HashMap<>(); - abstract Builder setIndexToCharacter(ImmutableMap value); + abstract Builder setIndexToCharacter(ImmutableMap value); - abstract Builder setCharacterToIndex(ImmutableMap value); + abstract Builder setCharacterToIndex(ImmutableMap value); - public Builder addCharacter(Character charsetChar, Character equivalentChar, Long index) { + public Builder addCharacter(String charsetChar, String equivalentChar, Long index) { logger.debug( "Registering character order for {}, index-type = {}, character = {}, equivalentCharacter = {}, index = {}, isBlank = {}", collationReference(), diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java index 910216af97..c60bdcf6f3 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java @@ -81,7 +81,7 @@ public abstract class CollationMapper implements Serializable { * using utf8mb4), 'b') = 'ab' COLLATE ;} returns 1. TODO(vardhanvthigle): Check this * behavior for PG and other databases. */ - public abstract ImmutableSet emptyCharacters(); + public abstract ImmutableSet emptyCharacters(); /** * Space Characters. MySQL ignores trailing space characters in comparisons for PAD space @@ -90,11 +90,11 @@ public abstract class CollationMapper implements Serializable { * (UNHEX(C2H0)) when the collation is Pad Space. These have same behavior to ascii space as far * as trailing or non-trailing comparison is concerned. */ - public abstract ImmutableSet spaceCharacters(); + public abstract ImmutableSet spaceCharacters(); @Memoized String allSpaceCharacters() { - return this.spaceCharacters().stream().map(String::valueOf).collect(Collectors.joining("")); + return this.spaceCharacters().stream().collect(Collectors.joining("")); } @Memoized @@ -103,8 +103,7 @@ String emptyReplacePattern() { return ""; } return "[" - + Pattern.quote( - this.emptyCharacters().stream().map(String::valueOf).collect(Collectors.joining(""))) + + Pattern.quote(this.emptyCharacters().stream().collect(Collectors.joining(""))) + "]"; } @@ -150,14 +149,20 @@ public BigInteger mapString(@Nullable String element, int lengthToPad) { } // Convert the string to BigInteger. - for (int index = 0; index < element.length(); index++) { - Character c = element.charAt(index); + java.util.List codePoints = new java.util.ArrayList<>(); + for (int i = 0; i < element.length(); ) { + int cp = element.codePointAt(i); + codePoints.add(new String(Character.toChars(cp))); + i += Character.charCount(cp); + } + for (int index = 0; index < codePoints.size(); index++) { + String c = codePoints.get(index); ret = - ret.multiply(BigInteger.valueOf(getCharsetSize(index == (element.length() - 1)))) - .add(BigInteger.valueOf(getOrdinalPosition(c, index == (element.length() - 1)))); + ret.multiply(BigInteger.valueOf(getCharsetSize(index == (codePoints.size() - 1)))) + .add(BigInteger.valueOf(getOrdinalPosition(c, index == (codePoints.size() - 1)))); } - for (int index = element.length(); index < lengthToPad; index++) { - ret = ret.multiply(BigInteger.valueOf(getCharsetSize(index == (element.length() - 1)))); + for (int index = codePoints.size(); index < lengthToPad; index++) { + ret = ret.multiply(BigInteger.valueOf(getCharsetSize(index == (codePoints.size() - 1)))); } return ret; } @@ -189,21 +194,21 @@ public String unMapString(BigInteger element) { // Base Case that the string just represents single character if (element == BigInteger.ZERO) { - char c = getCharacterFromPosition(element.longValue(), true); - return String.valueOf(c); + String c = getCharacterFromPosition(element.longValue(), true); + return c; } while (element != BigInteger.ZERO) { long charsetSize = getCharsetSize(index == 0); BigInteger reminder = element.mod(BigInteger.valueOf(charsetSize)); - char c = getCharacterFromPosition(reminder.longValue(), (index == 0)); - word.append(c); + String c = getCharacterFromPosition(reminder.longValue(), (index == 0)); + word.insert(0, c); element = element.divide(BigInteger.valueOf(charsetSize)); index++; } - String ret = word.reverse().toString(); + String ret = word.toString(); return ret; } @@ -275,13 +280,13 @@ private long getCharsetSize(boolean lastCharacter) { : this.allPositionsIndex().getCharsetSize(); } - private long getOrdinalPosition(Character c, boolean lastCharacter) { + private long getOrdinalPosition(String c, boolean lastCharacter) { return (lastCharacter && collationReference().padSpace()) ? this.trailingPositionsPadSpace().getOrdinalPosition(c) : this.allPositionsIndex().getOrdinalPosition(c); } - private Character getCharacterFromPosition(long ordinalPosition, boolean firstIteration) { + private String getCharacterFromPosition(long ordinalPosition, boolean firstIteration) { return (firstIteration && collationReference().padSpace()) ? this.trailingPositionsPadSpace().getCharacterFromPosition(ordinalPosition) : this.allPositionsIndex().getCharacterFromPosition(ordinalPosition); @@ -307,9 +312,9 @@ public abstract static class Builder { abstract CollationIndex.Builder trailingPositionsPadSpaceBuilder(); - abstract ImmutableSet.Builder emptyCharactersBuilder(); + abstract ImmutableSet.Builder emptyCharactersBuilder(); - abstract ImmutableSet.Builder spaceCharactersBuilder(); + abstract ImmutableSet.Builder spaceCharactersBuilder(); public Builder addCharacter(CollationOrderRow collationOrderRow) { diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java index 633a60fcab..4c85d5bd25 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java @@ -24,7 +24,6 @@ import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_SPACE_COL; import com.google.auto.value.AutoValue; -import com.google.common.base.Preconditions; import java.sql.ResultSet; import java.sql.SQLException; import org.slf4j.Logger; @@ -41,10 +40,10 @@ public abstract class CollationOrderRow { private static final Logger logger = LoggerFactory.getLogger(CollationOrderRow.class); /** Character in the character set. */ - public abstract Character charsetChar(); + public abstract String charsetChar(); /** A character with lowest rank charset_char character is equal to as per the collation. */ - public abstract Character equivalentChar(); + public abstract String equivalentChar(); /** 0 offset rank of this character as per the collation sort ordering at all positions. */ public abstract Long codepointRank(); @@ -54,7 +53,7 @@ public abstract class CollationOrderRow { * trailing position, in case a PAD SPACE comparison is needed. Unless you are looking at space * like characters, this will be exactly same as equivalent_character. */ - public abstract Character equivalentCharPadSpace(); + public abstract String equivalentCharPadSpace(); /** * A character with lowest rank charset_char character is equal to as per the collation at @@ -110,21 +109,11 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { isEmpty, isSpace); - Preconditions.checkArgument( - charSetChar.length() <= 1, "Found a long character in collation output " + charSetChar); - Preconditions.checkArgument( - equivalentCharsetChar.length() <= 1, - "Found a long equivalent character in collation output " + equivalentCharsetChar); - Preconditions.checkArgument( - equivalentCharsetCharPadSpace.length() <= 1, - "Found a long equivalent character for pad space in collation output " - + equivalentCharsetChar); - return CollationOrderRow.builder() - .setCharsetChar(charSetChar.charAt(0)) - .setEquivalentChar(equivalentCharsetChar.charAt(0)) + .setCharsetChar(charSetChar) + .setEquivalentChar(equivalentCharsetChar) .setCodepointRank(codePointRank) - .setEquivalentCharPadSpace(equivalentCharsetCharPadSpace.charAt(0)) + .setEquivalentCharPadSpace(equivalentCharsetCharPadSpace) .setCodepointRankPadSpace(codePointRankPadSpace) .setIsEmpty(isEmpty) .setIsSpace(isSpace) @@ -134,13 +123,13 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { @AutoValue.Builder public abstract static class Builder { - public abstract Builder setCharsetChar(Character value); + public abstract Builder setCharsetChar(String value); - public abstract Builder setEquivalentChar(Character value); + public abstract Builder setEquivalentChar(String value); public abstract Builder setCodepointRank(Long value); - public abstract Builder setEquivalentCharPadSpace(Character value); + public abstract Builder setEquivalentCharPadSpace(String value); public abstract Builder setCodepointRankPadSpace(Long value); diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index 7fc3e89b2b..bdd2168447 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -33,52 +33,52 @@ SET @byte_literals = CONCAT( -- Four byte code points. SET @four_byte_codepoints = CONCAT( - 'SELECT CONCAT(t1.h, t2.h, t3.h, t4.h) AS hex_val ', - 'FROM (', @byte_literals, ') AS t1 ', - 'CROSS JOIN (', @byte_literals, ') AS t2 ', - 'CROSS JOIN (', @byte_literals, ') AS t3 ', - 'CROSS JOIN (', @byte_literals, ') AS t4 ', - 'WHERE t1.h BETWEEN ''f0'' AND ''f4'' AND t2.h BETWEEN ''80'' AND ''bf'' AND t3.h BETWEEN ''80'' AND ''bf'' AND t4.h BETWEEN ''80'' AND ''bf''' + '(SELECT * FROM (SELECT ', + 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h, t4.h)) USING ', @db_charset, ') AS charset_char ', + 'FROM (', @byte_literals, ') AS t1 ', + 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', + 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', + 'LEFT JOIN (', @byte_literals, ') AS t4 ON 1=1 ', + ') AS dt ', + 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' + ')' ); -- Three byte code points. SET @three_byte_codepoints = CONCAT( - 'SELECT CONCAT(t1.h, t2.h, t3.h) AS hex_val ', - 'FROM (', @byte_literals, ') AS t1 ', - 'CROSS JOIN (', @byte_literals, ') AS t2 ', - 'CROSS JOIN (', @byte_literals, ') AS t3 ', - 'WHERE t1.h BETWEEN ''e0'' AND ''ef'' AND t2.h BETWEEN ''80'' AND ''bf'' AND t3.h BETWEEN ''80'' AND ''bf''' + '(SELECT * FROM (SELECT ', + 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h)) USING ', @db_charset, ') AS charset_char ', + 'FROM (', @byte_literals, ') AS t1 ', + 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', + 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', + ') AS dt ', + 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' + ')' ); -- Two byte code points. SET @two_byte_codepoints = CONCAT( - 'SELECT CONCAT(t1.h, t2.h) AS hex_val ', - 'FROM (', @byte_literals, ') AS t1 ', - 'CROSS JOIN (', @byte_literals, ') AS t2 ', - 'WHERE t1.h BETWEEN ''c2'' AND ''df'' AND t2.h BETWEEN ''80'' AND ''bf''' + '(SELECT * FROM (SELECT ', + 'CONVERT(UNHEX(CONCAT(t1.h, t2.h)) USING ', @db_charset, ') AS charset_char ', + 'FROM (', @byte_literals, ') AS t1 ', + 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', + ') AS dt ', + 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' + ')' ); -- Single byte code points. SET @one_byte_codepoints = CONCAT( - 'SELECT t1.h AS hex_val ', - 'FROM (', @byte_literals, ') AS t1 ', - 'WHERE t1.h BETWEEN ''00'' AND ''7f''' + '(SELECT * FROM (SELECT ', + 'CONVERT(UNHEX(t1.h) USING ', @db_charset, ') AS charset_char ', + 'FROM (', @byte_literals, ') AS t1', + ') AS dt ', -- derived table + 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' + ')' ); -SET @all_utf8_hex = CONCAT(@four_byte_codepoints, ' UNION ALL ', @three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); - -- all variable length code points representing a single character within the @db_charset from length 0 till 4. -SET @charset_chars = CONCAT( - '(SELECT charset_char FROM ( ', - 'SELECT CONVERT(CONVERT(UNHEX(hex_val) USING utf8mb4) USING ', @db_charset, ') AS charset_char, ', - 'CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char, ', - 'hex_val ', - 'FROM (', @all_utf8_hex, ') AS all_chars ', - 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''eda080'' AND ''edbfbf'' ', - ') AS valid_utf8_chars ', - 'WHERE charset_char IS NOT NULL AND (charset_char != ''?'' OR utf8_char = ''?'') ', - 'AND CHAR_LENGTH(charset_char) <= 1)' -); +SET @charset_chars = CONCAT(@three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); SET @SPACE=CONCAT('CONVERT('' '' USING ', @db_charset,')'); SET @ALPHABET=CONCAT('CONVERT(''a'' USING ', @db_charset,')'); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java index dd23372717..a0864a176a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java @@ -58,9 +58,9 @@ public void testBoundaryTypeMappingImpl() { collationMapperBuilder .addCharacter( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("a") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -68,9 +68,9 @@ public void testBoundaryTypeMappingImpl() { .build()) .addCharacter( CollationOrderRow.builder() - .setCharsetChar('A') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("A") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java index 3a62e7e526..d6df1e8e45 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java @@ -38,8 +38,8 @@ public void testCollationIndexBasic() { CollationIndex.builder() .setCollationReference(testCollationReference) .setIndexType(CollationIndexType.TRAILING_POSITION_PAD_SPACE) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 0L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 0L) .build(); assertThat(collationIndex.indexType()) @@ -47,8 +47,8 @@ public void testCollationIndexBasic() { assertThat(collationIndex.collationReference()).isEqualTo(testCollationReference); assertThat(collationIndex.getCharsetSize()).isEqualTo(1); assertThat(collationIndex.characterToIndex().size()).isEqualTo(2); - assertThat(collationIndex.getCharacterFromPosition(0L)).isEqualTo('A'); - assertThat(collationIndex.getOrdinalPosition('a')).isEqualTo(0L); + assertThat(collationIndex.getCharacterFromPosition(0L)).isEqualTo("A"); + assertThat(collationIndex.getOrdinalPosition("a")).isEqualTo(0L); } @Test @@ -67,8 +67,8 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('a', 'A', 0L) + .addCharacter("a", "A", 0L) + .addCharacter("a", "A", 0L) .build()); // Duplicate Index assertThrows( @@ -77,16 +77,16 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 2L)); + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 2L)); assertThrows( IllegalStateException.class, () -> CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('z', 'Z', 0L)); + .addCharacter("a", "A", 0L) + .addCharacter("z", "Z", 0L)); // Index with Holes. assertThrows( IllegalStateException.class, @@ -94,10 +94,10 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 0L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 0L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); // Index Character not part of basic character set. assertThrows( @@ -106,10 +106,10 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'M', 0L) - .addCharacter('A', 'A', 5L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "M", 0L) + .addCharacter("A", "A", 5L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); // Index Character does not map to itself assertThrows( @@ -118,11 +118,11 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'M', 5L) - .addCharacter('M', 'M', 5L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "M", 5L) + .addCharacter("M", "M", 5L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java index 4e62f36205..9e2c469d58 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java @@ -71,9 +71,9 @@ public void testCollationMapperBasic() { for (Character c : enAlphabetsBuilder.build()) { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar(c) - .setEquivalentChar(Character.toUpperCase(c)) - .setEquivalentCharPadSpace(Character.toUpperCase(c)) + .setCharsetChar(String.valueOf(c)) + .setEquivalentChar(String.valueOf(Character.toUpperCase(c))) + .setEquivalentCharPadSpace(String.valueOf(Character.toUpperCase(c))) .setCodepointRank((long) (Character.toUpperCase(c) - 'A')) .setCodepointRankPadSpace((long) (Character.toUpperCase(c) - 'A')) .setIsEmpty(false) @@ -84,9 +84,9 @@ public void testCollationMapperBasic() { /** Add blank character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('\0') - .setEquivalentChar('\0') - .setEquivalentCharPadSpace('\0') + .setCharsetChar("\0") + .setEquivalentChar("\0") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(true) @@ -136,9 +136,9 @@ public void testCollationMapperPadSpace() { for (Character c : enAlphabetsBuilder.build()) { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar(c) - .setEquivalentChar(Character.toUpperCase(c)) - .setEquivalentCharPadSpace(Character.toUpperCase(c)) + .setCharsetChar(String.valueOf(c)) + .setEquivalentChar(String.valueOf(Character.toUpperCase(c))) + .setEquivalentCharPadSpace(String.valueOf(Character.toUpperCase(c))) .setCodepointRank((long) (Character.toUpperCase(c) - 'A' + 1)) .setCodepointRankPadSpace((long) (Character.toUpperCase(c) - 'A')) .setIsEmpty(false) @@ -149,9 +149,9 @@ public void testCollationMapperPadSpace() { /** Add Space Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar(' ') - .setEquivalentChar(' ') - .setEquivalentCharPadSpace('\0') + .setCharsetChar(" ") + .setEquivalentChar(" ") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -183,9 +183,9 @@ public void testCollationMapperEmptyStrings() { /* Add space character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar(' ') - .setEquivalentChar(' ') - .setEquivalentCharPadSpace('\0') + .setCharsetChar(" ") + .setEquivalentChar(" ") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -194,9 +194,9 @@ public void testCollationMapperEmptyStrings() { /* Add empty Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('\0') - .setEquivalentChar('\0') - .setEquivalentCharPadSpace('\0') + .setCharsetChar("\0") + .setEquivalentChar("\0") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(true) @@ -224,9 +224,9 @@ public void testCollationMapperSingleCharacterString() { /* Add Single Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('a') - .setEquivalentCharPadSpace('a') + .setCharsetChar("a") + .setEquivalentChar("a") + .setEquivalentCharPadSpace("a") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java index 1d600b9a2d..4e5118c74b 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java @@ -23,7 +23,7 @@ import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_EMPTY_COL; import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_SPACE_COL; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; + import static org.mockito.Mockito.when; import java.sql.ResultSet; @@ -42,9 +42,9 @@ public class CollationOrderRowTest { public void testCollationOrderRowBasic() { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("a") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(1L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -52,9 +52,9 @@ public void testCollationOrderRowBasic() { .build(); assertThat(collationOrderRow.codepointRank()).isEqualTo(1L); assertThat(collationOrderRow.codepointRankPadSpace()).isEqualTo(0L); - assertThat(collationOrderRow.charsetChar()).isEqualTo('a'); - assertThat(collationOrderRow.equivalentChar()).isEqualTo('A'); - assertThat(collationOrderRow.equivalentCharPadSpace()).isEqualTo('A'); + assertThat(collationOrderRow.charsetChar()).isEqualTo("a"); + assertThat(collationOrderRow.equivalentChar()).isEqualTo("A"); + assertThat(collationOrderRow.equivalentCharPadSpace()).isEqualTo("A"); assertThat(collationOrderRow.isEmpty()).isFalse(); assertThat(collationOrderRow.isSpace()).isFalse(); } @@ -74,9 +74,9 @@ public void testCollationOrderRowFromRsBasic() throws SQLException { assertThat(collationOrderRow) .isEqualTo( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('a') - .setEquivalentCharPadSpace('a') + .setCharsetChar("a") + .setEquivalentChar("a") + .setEquivalentCharPadSpace("a") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -84,25 +84,4 @@ public void testCollationOrderRowFromRsBasic() throws SQLException { .build()); } - @Test - public void testCollationOrderRowFromRsException() throws SQLException { - int expcetedIllegalArgumentExceptionCount = 0; - when(mockResultSet.getString(CHARSET_CHAR_COL)).thenReturn("aa").thenReturn("a"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getString(EQUIVALENT_CHARSET_CHAR_COL)).thenReturn("a").thenReturn("aa"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getString(EQUIVALENT_CHARSET_CHAR_PAD_SPACE_COL)) - .thenReturn("a") - .thenReturn("a") - .thenReturn("aa"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getLong(CODEPOINT_RANK_COL)).thenReturn(0L); - when(mockResultSet.getLong(CODEPOINT_RANK_COL)).thenReturn(0L); - when(mockResultSet.getBoolean(IS_EMPTY_COL)).thenReturn(false); - when(mockResultSet.getBoolean(IS_SPACE_COL)).thenReturn(false); - - for (int i = 0; i < expcetedIllegalArgumentExceptionCount; i++) { - assertThrows(IllegalArgumentException.class, () -> CollationOrderRow.fromRS(mockResultSet)); - } - } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 3ed99e994d..eecd8ab66d 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -27,10 +27,10 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudMySQLResourceManager; import org.apache.beam.it.gcp.spanner.SpannerResourceManager; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.jdbc.JDBCResourceManager; -import org.apache.beam.it.gcp.cloudsql.CloudMySQLResourceManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -53,8 +53,6 @@ public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBa private static final String SPANNER_DDL_RESOURCE = "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; - - private JDBCResourceManager.JDBCSchema getMySQLSchema() { HashMap columns = new HashMap<>(); columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL"); From 072d6117fc6aafdc088e03d64de46e34ff89fc40 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Mon, 13 Jul 2026 06:38:12 +0000 Subject: [PATCH 09/20] formatting --- .../uniformsplitter/stringmapper/CollationOrderRowTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java index 4e5118c74b..dd3078fde0 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java @@ -23,7 +23,6 @@ import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_EMPTY_COL; import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_SPACE_COL; import static com.google.common.truth.Truth.assertThat; - import static org.mockito.Mockito.when; import java.sql.ResultSet; @@ -83,5 +82,4 @@ public void testCollationOrderRowFromRsBasic() throws SQLException { .setIsSpace(false) .build()); } - } From 1ba919855a25ceef8b155b352a83c64b0fd6b272 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Mon, 13 Jul 2026 06:49:32 +0000 Subject: [PATCH 10/20] query changes --- .../sql/mysql_collation_order_query.sql | 107 +++++++----------- .../v2/templates/SourceDbToSpannerITBase.java | 11 ++ 2 files changed, 51 insertions(+), 67 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index bdd2168447..5589bc738d 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -11,74 +11,47 @@ SET @db_collation = 'collation_replacement_tag'; --- A union of single byte literals from 0x00 to 0xff. -SET @byte_literals = CONCAT( - 'SELECT ''00'' AS h UNION ALL SELECT ''01'' UNION ALL SELECT ''02'' UNION ALL SELECT ''03'' UNION ALL SELECT ''04'' UNION ALL SELECT ''05'' UNION ALL SELECT ''06'' UNION ALL SELECT ''07'' UNION ALL SELECT ''08'' UNION ALL SELECT ''09'' UNION ALL SELECT ''0a'' UNION ALL SELECT ''0b'' UNION ALL SELECT ''0c'' UNION ALL SELECT ''0d'' UNION ALL SELECT ''0e'' UNION ALL SELECT ''0f''', -'UNION ALL SELECT ''10'' AS h UNION ALL SELECT ''11'' UNION ALL SELECT ''12'' UNION ALL SELECT ''13'' UNION ALL SELECT ''14'' UNION ALL SELECT ''15'' UNION ALL SELECT ''16'' UNION ALL SELECT ''17'' UNION ALL SELECT ''18'' UNION ALL SELECT ''19'' UNION ALL SELECT ''1a'' UNION ALL SELECT ''1b'' UNION ALL SELECT ''1c'' UNION ALL SELECT ''1d'' UNION ALL SELECT ''1e'' UNION ALL SELECT ''1f''', -'UNION ALL SELECT ''20'' AS h UNION ALL SELECT ''21'' UNION ALL SELECT ''22'' UNION ALL SELECT ''23'' UNION ALL SELECT ''24'' UNION ALL SELECT ''25'' UNION ALL SELECT ''26'' UNION ALL SELECT ''27'' UNION ALL SELECT ''28'' UNION ALL SELECT ''29'' UNION ALL SELECT ''2a'' UNION ALL SELECT ''2b'' UNION ALL SELECT ''2c'' UNION ALL SELECT ''2d'' UNION ALL SELECT ''2e'' UNION ALL SELECT ''2f''', -'UNION ALL SELECT ''30'' AS h UNION ALL SELECT ''31'' UNION ALL SELECT ''32'' UNION ALL SELECT ''33'' UNION ALL SELECT ''34'' UNION ALL SELECT ''35'' UNION ALL SELECT ''36'' UNION ALL SELECT ''37'' UNION ALL SELECT ''38'' UNION ALL SELECT ''39'' UNION ALL SELECT ''3a'' UNION ALL SELECT ''3b'' UNION ALL SELECT ''3c'' UNION ALL SELECT ''3d'' UNION ALL SELECT ''3e'' UNION ALL SELECT ''3f''', -'UNION ALL SELECT ''40'' AS h UNION ALL SELECT ''41'' UNION ALL SELECT ''42'' UNION ALL SELECT ''43'' UNION ALL SELECT ''44'' UNION ALL SELECT ''45'' UNION ALL SELECT ''46'' UNION ALL SELECT ''47'' UNION ALL SELECT ''48'' UNION ALL SELECT ''49'' UNION ALL SELECT ''4a'' UNION ALL SELECT ''4b'' UNION ALL SELECT ''4c'' UNION ALL SELECT ''4d'' UNION ALL SELECT ''4e'' UNION ALL SELECT ''4f''', -'UNION ALL SELECT ''50'' AS h UNION ALL SELECT ''51'' UNION ALL SELECT ''52'' UNION ALL SELECT ''53'' UNION ALL SELECT ''54'' UNION ALL SELECT ''55'' UNION ALL SELECT ''56'' UNION ALL SELECT ''57'' UNION ALL SELECT ''58'' UNION ALL SELECT ''59'' UNION ALL SELECT ''5a'' UNION ALL SELECT ''5b'' UNION ALL SELECT ''5c'' UNION ALL SELECT ''5d'' UNION ALL SELECT ''5e'' UNION ALL SELECT ''5f''', -'UNION ALL SELECT ''60'' AS h UNION ALL SELECT ''61'' UNION ALL SELECT ''62'' UNION ALL SELECT ''63'' UNION ALL SELECT ''64'' UNION ALL SELECT ''65'' UNION ALL SELECT ''66'' UNION ALL SELECT ''67'' UNION ALL SELECT ''68'' UNION ALL SELECT ''69'' UNION ALL SELECT ''6a'' UNION ALL SELECT ''6b'' UNION ALL SELECT ''6c'' UNION ALL SELECT ''6d'' UNION ALL SELECT ''6e'' UNION ALL SELECT ''6f''', -'UNION ALL SELECT ''70'' AS h UNION ALL SELECT ''71'' UNION ALL SELECT ''72'' UNION ALL SELECT ''73'' UNION ALL SELECT ''74'' UNION ALL SELECT ''75'' UNION ALL SELECT ''76'' UNION ALL SELECT ''77'' UNION ALL SELECT ''78'' UNION ALL SELECT ''79'' UNION ALL SELECT ''7a'' UNION ALL SELECT ''7b'' UNION ALL SELECT ''7c'' UNION ALL SELECT ''7d'' UNION ALL SELECT ''7e'' UNION ALL SELECT ''7f''', -'UNION ALL SELECT ''80'' AS h UNION ALL SELECT ''81'' UNION ALL SELECT ''82'' UNION ALL SELECT ''83'' UNION ALL SELECT ''84'' UNION ALL SELECT ''85'' UNION ALL SELECT ''86'' UNION ALL SELECT ''87'' UNION ALL SELECT ''88'' UNION ALL SELECT ''89'' UNION ALL SELECT ''8a'' UNION ALL SELECT ''8b'' UNION ALL SELECT ''8c'' UNION ALL SELECT ''8d'' UNION ALL SELECT ''8e'' UNION ALL SELECT ''8f''', -'UNION ALL SELECT ''90'' AS h UNION ALL SELECT ''91'' UNION ALL SELECT ''92'' UNION ALL SELECT ''93'' UNION ALL SELECT ''94'' UNION ALL SELECT ''95'' UNION ALL SELECT ''96'' UNION ALL SELECT ''97'' UNION ALL SELECT ''98'' UNION ALL SELECT ''99'' UNION ALL SELECT ''9a'' UNION ALL SELECT ''9b'' UNION ALL SELECT ''9c'' UNION ALL SELECT ''9d'' UNION ALL SELECT ''9e'' UNION ALL SELECT ''9f''', -'UNION ALL SELECT ''a0'' AS h UNION ALL SELECT ''a1'' UNION ALL SELECT ''a2'' UNION ALL SELECT ''a3'' UNION ALL SELECT ''a4'' UNION ALL SELECT ''a5'' UNION ALL SELECT ''a6'' UNION ALL SELECT ''a7'' UNION ALL SELECT ''a8'' UNION ALL SELECT ''a9'' UNION ALL SELECT ''aa'' UNION ALL SELECT ''ab'' UNION ALL SELECT ''ac'' UNION ALL SELECT ''ad'' UNION ALL SELECT ''ae'' UNION ALL SELECT ''af''', -'UNION ALL SELECT ''b0'' AS h UNION ALL SELECT ''b1'' UNION ALL SELECT ''b2'' UNION ALL SELECT ''b3'' UNION ALL SELECT ''b4'' UNION ALL SELECT ''b5'' UNION ALL SELECT ''b6'' UNION ALL SELECT ''b7'' UNION ALL SELECT ''b8'' UNION ALL SELECT ''b9'' UNION ALL SELECT ''ba'' UNION ALL SELECT ''bb'' UNION ALL SELECT ''bc'' UNION ALL SELECT ''bd'' UNION ALL SELECT ''be'' UNION ALL SELECT ''bf''', -'UNION ALL SELECT ''c0'' AS h UNION ALL SELECT ''c1'' UNION ALL SELECT ''c2'' UNION ALL SELECT ''c3'' UNION ALL SELECT ''c4'' UNION ALL SELECT ''c5'' UNION ALL SELECT ''c6'' UNION ALL SELECT ''c7'' UNION ALL SELECT ''c8'' UNION ALL SELECT ''c9'' UNION ALL SELECT ''ca'' UNION ALL SELECT ''cb'' UNION ALL SELECT ''cc'' UNION ALL SELECT ''cd'' UNION ALL SELECT ''ce'' UNION ALL SELECT ''cf''', -'UNION ALL SELECT ''d0'' AS h UNION ALL SELECT ''d1'' UNION ALL SELECT ''d2'' UNION ALL SELECT ''d3'' UNION ALL SELECT ''d4'' UNION ALL SELECT ''d5'' UNION ALL SELECT ''d6'' UNION ALL SELECT ''d7'' UNION ALL SELECT ''d8'' UNION ALL SELECT ''d9'' UNION ALL SELECT ''da'' UNION ALL SELECT ''db'' UNION ALL SELECT ''dc'' UNION ALL SELECT ''dd'' UNION ALL SELECT ''de'' UNION ALL SELECT ''df''', -'UNION ALL SELECT ''e0'' AS h UNION ALL SELECT ''e1'' UNION ALL SELECT ''e2'' UNION ALL SELECT ''e3'' UNION ALL SELECT ''e4'' UNION ALL SELECT ''e5'' UNION ALL SELECT ''e6'' UNION ALL SELECT ''e7'' UNION ALL SELECT ''e8'' UNION ALL SELECT ''e9'' UNION ALL SELECT ''ea'' UNION ALL SELECT ''eb'' UNION ALL SELECT ''ec'' UNION ALL SELECT ''ed'' UNION ALL SELECT ''ee'' UNION ALL SELECT ''ef''', -'UNION ALL SELECT ''f0'' AS h UNION ALL SELECT ''f1'' UNION ALL SELECT ''f2'' UNION ALL SELECT ''f3'' UNION ALL SELECT ''f4'' UNION ALL SELECT ''f5'' UNION ALL SELECT ''f6'' UNION ALL SELECT ''f7'' UNION ALL SELECT ''f8'' UNION ALL SELECT ''f9'' UNION ALL SELECT ''fa'' UNION ALL SELECT ''fb'' UNION ALL SELECT ''fc'' UNION ALL SELECT ''fd'' UNION ALL SELECT ''fe'' UNION ALL SELECT ''ff''' -); - --- Four byte code points. -SET @four_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h, t4.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t4 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Three byte code points. -SET @three_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Two byte code points. -SET @two_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Single byte code points. -SET @one_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(t1.h) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1', - ') AS dt ', -- derived table - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); +-- Enumerating valid utf8mb4 byte sequences +SET @all_chars = ' +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n, n5.n, n6.n, n7.n, n8.n) AS hex_val +FROM (SELECT ''F'' AS n) n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n5 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n6 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n7 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n8 +UNION ALL +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n, n5.n, n6.n) AS hex_val +FROM (SELECT ''E'' AS n) n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n5 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n6 +UNION ALL +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n) AS hex_val +FROM (SELECT ''C'' AS n UNION ALL SELECT ''D'') n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +UNION ALL +SELECT CONCAT(n1.n, n2.n) AS hex_val +FROM (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'') n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +'; + +SET @charset_chars = CONCAT( + '(SELECT charset_char FROM ( ', + 'SELECT hex_val, CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char, ', + 'CONVERT(CONVERT(UNHEX(hex_val) USING utf8mb4) USING ', @db_charset, ') AS charset_char ', + 'FROM ( ', @all_chars, ' ) AS all_chars ', + 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''EDA080'' AND ''EDBFBF'' ', + ') AS valid_utf8_chars ', + 'WHERE charset_char IS NOT NULL AND (charset_char != ''?'' OR utf8_char = ''?'') ', +')'); --- all variable length code points representing a single character within the @db_charset from length 0 till 4. -SET @charset_chars = CONCAT(@three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); SET @SPACE=CONCAT('CONVERT('' '' USING ', @db_charset,')'); SET @ALPHABET=CONCAT('CONVERT(''a'' USING ', @db_charset,')'); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java index dbabfc4423..f52195d23a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java @@ -34,8 +34,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.time.Duration; import java.util.stream.Collectors; import org.apache.beam.it.cassandra.CassandraResourceManager; +import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.ResourceManager; import org.apache.beam.it.common.utils.IORedirectUtil; @@ -396,4 +398,13 @@ private String driverClassNameFrom(JDBCResourceManager jdbcResourceManager) { throw new IllegalArgumentException(e); } } + + @Override + protected PipelineOperator.Config.Builder wrapConfiguration( + PipelineOperator.Config.Builder builder) { + if (System.getProperty("directRunnerTest") != null) { + return builder.setTimeoutAfter(Duration.ofMinutes(15)); + } + return builder; + } } From 0a799d42d0486d7a35c22d0eab4884e1109ce045 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Tue, 14 Jul 2026 04:22:02 +0000 Subject: [PATCH 11/20] format --- .../cloud/teleport/v2/templates/SourceDbToSpannerITBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java index f52195d23a..b0af7bc064 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java @@ -30,15 +30,15 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.time.Duration; import java.util.stream.Collectors; import org.apache.beam.it.cassandra.CassandraResourceManager; -import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.ResourceManager; import org.apache.beam.it.common.utils.IORedirectUtil; import org.apache.beam.it.common.utils.PipelineUtils; From 280dd57431e839f5a2061ff52c699f27972d2cd9 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 22 Jul 2026 06:56:02 +0000 Subject: [PATCH 12/20] minor change" --- .../sql/mysql_collation_order_query.sql | 2 +- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 34 +++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index 5589bc738d..b1429a0cc6 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -49,7 +49,7 @@ SET @charset_chars = CONCAT( 'FROM ( ', @all_chars, ' ) AS all_chars ', 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''EDA080'' AND ''EDBFBF'' ', ') AS valid_utf8_chars ', - 'WHERE charset_char IS NOT NULL AND (charset_char != ''?'' OR utf8_char = ''?'') ', + 'WHERE charset_char IS NOT NULL AND (HEX(CONVERT(charset_char USING utf8mb4)) != ''3F'' OR hex_val = ''3F'') ', ')'); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index eecd8ab66d..38df9477a4 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -27,10 +27,10 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.cloudsql.CloudMySQLResourceManager; import org.apache.beam.it.gcp.spanner.SpannerResourceManager; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.beam.it.jdbc.MySQLResourceManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -44,37 +44,37 @@ public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { private static PipelineLauncher.LaunchInfo jobInfo; - public static CloudMySQLResourceManager mySQLResourceManager; + public static MySQLResourceManager mySQLResourceManager; public static SpannerResourceManager spannerResourceManager; - private static final String TABLE = "table4bytepk"; + private static final String TABLE_4BYTE = "table4bytepk"; private static final String ID = "id"; private static final String DESCRIPTION = "description"; private static final String SPANNER_DDL_RESOURCE = "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; - private JDBCResourceManager.JDBCSchema getMySQLSchema() { + private JDBCResourceManager.JDBCSchema getMySQL4ByteSchema() { HashMap columns = new HashMap<>(); columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL"); columns.put(DESCRIPTION, "VARCHAR(200)"); return new JDBCResourceManager.JDBCSchema(columns, ID); } - private List> getMySQLData() { + private List> getMySQL4ByteData() { List> data = new ArrayList<>(); Map row1 = new HashMap<>(); - row1.put(ID, "\uD83D\uDE00"); + row1.put(ID, "😀"); row1.put(DESCRIPTION, "Grinning Face"); data.add(row1); Map row2 = new HashMap<>(); - row2.put(ID, "\uD83D\uDE01"); + row2.put(ID, "😁"); row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); data.add(row2); Map row3 = new HashMap<>(); - row3.put(ID, "\uD83D\uDE02"); + row3.put(ID, "😂"); row3.put(DESCRIPTION, "Face with Tears of Joy"); data.add(row3); @@ -83,7 +83,7 @@ private List> getMySQLData() { @Before public void setUp() { - mySQLResourceManager = setUpCloudMySQLResourceManager(); + mySQLResourceManager = setUpMySQLResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); } @@ -94,12 +94,16 @@ public void cleanUp() { @Test public void testMySqlToSpanner() throws IOException { - List> mySQLData = getMySQLData(); - mySQLResourceManager.createTable(TABLE, getMySQLSchema()); - mySQLResourceManager.write(TABLE, mySQLData); + List> mySQL4ByteData = getMySQL4ByteData(); + mySQLResourceManager.createTable(TABLE_4BYTE, getMySQL4ByteSchema()); + mySQLResourceManager.write(TABLE_4BYTE, mySQL4ByteData); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + Map jobParameters = new HashMap<>(); + jobParameters.put("uniformizationStageCountHint", "-1"); + jobParameters.put("numPartitions", "5"); + jobInfo = launchDataflowJob( getClass().getSimpleName(), @@ -107,13 +111,13 @@ public void testMySqlToSpanner() throws IOException { null, mySQLResourceManager, spannerResourceManager, - null, + jobParameters, null); PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo)); assertThatResult(result).isLaunchFinished(); SpannerAsserts.assertThatStructs( - spannerResourceManager.readTableRecords(TABLE, ID, DESCRIPTION)) - .hasRecordsUnorderedCaseInsensitiveColumns(mySQLData); + spannerResourceManager.readTableRecords(TABLE_4BYTE, ID, DESCRIPTION)) + .hasRecordsUnorderedCaseInsensitiveColumns(mySQL4ByteData); } } From c08df07d76a081fb9eff99e01e99236e09d434b2 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 22 Jul 2026 07:07:40 +0000 Subject: [PATCH 13/20] minor change in tests --- .../PostgreSQLSourceDbToSpanner4ByteStringPKIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java index 657d10cdbd..bcb44dd579 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java @@ -64,17 +64,17 @@ private List> getPostgreSQLData() { List> data = new ArrayList<>(); Map row1 = new HashMap<>(); - row1.put(ID, "\uD83D\uDE00"); + row1.put(ID, "😀"); row1.put(DESCRIPTION, "Grinning Face"); data.add(row1); Map row2 = new HashMap<>(); - row2.put(ID, "\uD83D\uDE01"); + row2.put(ID, "😁"); row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); data.add(row2); Map row3 = new HashMap<>(); - row3.put(ID, "\uD83D\uDE02"); + row3.put(ID, "😂"); row3.put(DESCRIPTION, "Face with Tears of Joy"); data.add(row3); From 391226c463fca0b347486e16181ff7f4318cebac Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Fri, 24 Jul 2026 07:28:16 +0000 Subject: [PATCH 14/20] address comments --- .../stringmapper/CollationMapper.java | 33 ++++++++++++------- ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 8 ++--- ...reSQLSourceDbToSpanner4ByteStringPKIT.java | 8 ++--- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java index c60bdcf6f3..6e8826dcc3 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java @@ -130,6 +130,14 @@ public BigInteger mapString(@Nullable String element, int lengthToPad) { if (element == null) { return BigInteger.valueOf(-1); } + // 'ret' stores the mapped value using a variable-base encoding. + // The base (charset size) can change depending on whether it's the trailing position + // in a pad-space collation. + // Example: For string "abcd" with lengthToPad = 6, let non-trailing base = 100 and trailing + // base = 90. + // If ordinals are a=1, b=2, c=3, d=4, the mapping evaluates to: + // ret = ((((1 * 100 + 2) * 100) + 3) * 90 + 4) * (100 ^ 2) + // unMapString reverses this by extracting modulo the trailing base first. BigInteger ret = BigInteger.ZERO; // MySQL ignores empty character in string comparisons. @@ -149,20 +157,21 @@ public BigInteger mapString(@Nullable String element, int lengthToPad) { } // Convert the string to BigInteger. - java.util.List codePoints = new java.util.ArrayList<>(); - for (int i = 0; i < element.length(); ) { - int cp = element.codePointAt(i); - codePoints.add(new String(Character.toChars(cp))); - i += Character.charCount(cp); - } + java.util.List codePoints = + element + .codePoints() + .mapToObj(cp -> new String(Character.toChars(cp))) + .collect(Collectors.toList()); for (int index = 0; index < codePoints.size(); index++) { String c = codePoints.get(index); ret = ret.multiply(BigInteger.valueOf(getCharsetSize(index == (codePoints.size() - 1)))) .add(BigInteger.valueOf(getOrdinalPosition(c, index == (codePoints.size() - 1)))); } - for (int index = codePoints.size(); index < lengthToPad; index++) { - ret = ret.multiply(BigInteger.valueOf(getCharsetSize(index == (codePoints.size() - 1)))); + if (lengthToPad > codePoints.size()) { + ret = + ret.multiply( + BigInteger.valueOf(getCharsetSize(false)).pow(lengthToPad - codePoints.size())); } return ret; } @@ -193,22 +202,22 @@ public String unMapString(BigInteger element) { } // Base Case that the string just represents single character - if (element == BigInteger.ZERO) { + if (element.equals(BigInteger.ZERO)) { String c = getCharacterFromPosition(element.longValue(), true); return c; } - while (element != BigInteger.ZERO) { + while (!element.equals(BigInteger.ZERO)) { long charsetSize = getCharsetSize(index == 0); BigInteger reminder = element.mod(BigInteger.valueOf(charsetSize)); String c = getCharacterFromPosition(reminder.longValue(), (index == 0)); - word.insert(0, c); + word.append(c); element = element.divide(BigInteger.valueOf(charsetSize)); index++; } - String ret = word.toString(); + String ret = word.reverse().toString(); return ret; } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java index 38df9477a4..b760a8ab95 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java @@ -42,10 +42,8 @@ @TemplateIntegrationTest(SourceDbToSpanner.class) @RunWith(JUnit4.class) public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { - private static PipelineLauncher.LaunchInfo jobInfo; - - public static MySQLResourceManager mySQLResourceManager; - public static SpannerResourceManager spannerResourceManager; + private MySQLResourceManager mySQLResourceManager; + private SpannerResourceManager spannerResourceManager; private static final String TABLE_4BYTE = "table4bytepk"; private static final String ID = "id"; @@ -104,7 +102,7 @@ public void testMySqlToSpanner() throws IOException { jobParameters.put("uniformizationStageCountHint", "-1"); jobParameters.put("numPartitions", "5"); - jobInfo = + PipelineLauncher.LaunchInfo jobInfo = launchDataflowJob( getClass().getSimpleName(), null, diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java index bcb44dd579..844ba0f4ae 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java @@ -42,10 +42,8 @@ @TemplateIntegrationTest(SourceDbToSpanner.class) @RunWith(JUnit4.class) public class PostgreSQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { - private static PipelineLauncher.LaunchInfo jobInfo; - - public static PostgresResourceManager postgreSQLResourceManager; - public static SpannerResourceManager spannerResourceManager; + private PostgresResourceManager postgreSQLResourceManager; + private SpannerResourceManager spannerResourceManager; private static final String TABLE = "table4bytepk"; private static final String ID = "id"; @@ -100,7 +98,7 @@ public void testPostgreSQLToSpanner() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - jobInfo = + PipelineLauncher.LaunchInfo jobInfo = launchDataflowJob( getClass().getSimpleName(), null, From a154038d5f199c442610dcde212ca84d82beea14 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 29 Jul 2026 09:24:59 +0000 Subject: [PATCH 15/20] address comments --- .../stringmapper/CollationIndex.java | 5 ++++ .../stringmapper/CollationOrderRow.java | 14 ++++++++++ .../sql/mysql_collation_order_query.sql | 3 +++ .../sql/postgresql_collation_order_query.sql | 3 +++ .../stringmapper/CollationIndexTest.java | 26 +++++++++++++++++++ 5 files changed, 51 insertions(+) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java index 64a874f407..e0cd3a6dfa 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java @@ -16,6 +16,7 @@ package com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper; import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.Serializable; @@ -89,6 +90,10 @@ public abstract static class Builder { abstract Builder setCharacterToIndex(ImmutableMap value); public Builder addCharacter(String charsetChar, String equivalentChar, Long index) { + Preconditions.checkNotNull(charsetChar, "charsetChar cannot be null"); + Preconditions.checkNotNull(equivalentChar, "equivalentChar cannot be null"); + Preconditions.checkNotNull(index, "index cannot be null"); + logger.debug( "Registering character order for {}, index-type = {}, character = {}, equivalentCharacter = {}, index = {}, isBlank = {}", collationReference(), diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java index 4c85d5bd25..1b3f8eaca0 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java @@ -24,6 +24,7 @@ import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_SPACE_COL; import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; import java.sql.ResultSet; import java.sql.SQLException; import org.slf4j.Logger; @@ -109,6 +110,19 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { isEmpty, isSpace); + Preconditions.checkArgument( + charSetChar.codePointCount(0, charSetChar.length()) == 1, + "Found a multi-codepoint character in collation output: " + charSetChar); + Preconditions.checkArgument( + equivalentCharsetChar.codePointCount(0, equivalentCharsetChar.length()) == 1, + "Found a multi-codepoint equivalent character in collation output: " + + equivalentCharsetChar); + Preconditions.checkArgument( + equivalentCharsetCharPadSpace.codePointCount(0, equivalentCharsetCharPadSpace.length()) + == 1, + "Found a multi-codepoint equivalent character for pad space in collation output: " + + equivalentCharsetCharPadSpace); + return CollationOrderRow.builder() .setCharsetChar(charSetChar) .setEquivalentChar(equivalentCharsetChar) diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index b1429a0cc6..3b32052351 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -12,6 +12,9 @@ SET @db_collation = 'collation_replacement_tag'; -- Enumerating valid utf8mb4 byte sequences +-- There are a total of 1,112,064 valid code points within the Unicode codespace. +-- All of them are generated by this enumeration. +-- https://en.wikipedia.org/wiki/Unicode#:~:text=There%20are%20a%20total%20of%201112064%20valid%20code%20points%20within%20the%20codespace SET @all_chars = ' SELECT CONCAT(n1.n, n2.n, n3.n, n4.n, n5.n, n6.n, n7.n, n8.n) AS hex_val FROM (SELECT ''F'' AS n) n1 diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql index 032fa406ff..df6fe69baf 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql @@ -17,6 +17,9 @@ WITH -- Generate 1 byte (U+0000 to U+007F), 2 bytes (U+0080 to U+07FF), -- 3 bytes (U+0800 to U+FFFF), and 4 bytes (U+10000 to U+10FFFF) unicode -- codepoints +-- There are a total of 1,112,064 valid code points within the Unicode codespace. +-- All of them are generated by this enumeration. +-- https://en.wikipedia.org/wiki/Unicode#:~:text=There%20are%20a%20total%20of%201112064%20valid%20code%20points%20within%20the%20codespace charset_chars AS ( SELECT * FROM ( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java index d6df1e8e45..0c440e6b8f 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java @@ -60,6 +60,32 @@ public void testCollationIndexPreConditions() { .setPadSpace(true) .build(); + // Null arguments + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter(null, "A", 0L) + .build()); + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter("a", null, 0L) + .build()); + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter("a", "A", null) + .build()); + // Duplicate Characters assertThrows( IllegalStateException.class, From c63b7952c765c14600f512df01089ee728fdc346 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 29 Jul 2026 11:27:23 +0000 Subject: [PATCH 16/20] fix: use <= 1 for codePointCount to allow empty strings and handle nulls --- .../stringmapper/CollationOrderRow.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java index 1b3f8eaca0..a4d4a7fcaf 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java @@ -111,15 +111,17 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { isSpace); Preconditions.checkArgument( - charSetChar.codePointCount(0, charSetChar.length()) == 1, + charSetChar.codePointCount(0, charSetChar.length()) <= 1, "Found a multi-codepoint character in collation output: " + charSetChar); Preconditions.checkArgument( - equivalentCharsetChar.codePointCount(0, equivalentCharsetChar.length()) == 1, + equivalentCharsetChar.codePointCount(0, equivalentCharsetChar.length()) <= 1, "Found a multi-codepoint equivalent character in collation output: " + equivalentCharsetChar); Preconditions.checkArgument( - equivalentCharsetCharPadSpace.codePointCount(0, equivalentCharsetCharPadSpace.length()) - == 1, + equivalentCharsetCharPadSpace == null + || equivalentCharsetCharPadSpace.codePointCount( + 0, equivalentCharsetCharPadSpace.length()) + <= 1, "Found a multi-codepoint equivalent character for pad space in collation output: " + equivalentCharsetCharPadSpace); From bc9d1c9b9892b4461f479617ad89bccd713e071e Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Mon, 3 Aug 2026 09:45:49 +0000 Subject: [PATCH 17/20] change splitting logic --- .../range/BoundarySplitterFactory.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java index 82acaeed06..78902e4324 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java @@ -39,6 +39,8 @@ public class BoundarySplitterFactory { private static final BigInteger SECONDS_TO_NANOS = BigInteger.valueOf(Duration.ofSeconds(1).toNanos()); + @VisibleForTesting protected static final int MAX_STRING_PARTITION_PAD_LENGTH = 300; + private static final ImmutableMap> splittermap = ImmutableMap.>builder() .put( @@ -413,15 +415,30 @@ private static String splitStrings( // during a run. // To avoid undefined behaviour in the padding logic, we take the max of the input strings and // the partition column width. + int commonPrefixLength = 0; + while (commonPrefixLength < start.length() + && commonPrefixLength < end.length() + && start.charAt(commonPrefixLength) == end.charAt(commonPrefixLength)) { + commonPrefixLength++; + } + String commonPrefix = start.substring(0, commonPrefixLength); + String suffixStart = start.substring(commonPrefixLength); + String suffixEnd = end.substring(commonPrefixLength); + int lengthToPad = Math.max( - Math.max(start.length(), end.length()), partitionColumn.stringMaxLength().intValue()); + Math.max(suffixStart.length(), suffixEnd.length()), + Math.min( + Math.max(0, partitionColumn.stringMaxLength().intValue() - commonPrefixLength), + MAX_STRING_PARTITION_PAD_LENGTH)); BigInteger bigIntegerStart = - (BigInteger) typeMapper.mapStringToBigInteger(start, lengthToPad, partitionColumn, c); + (BigInteger) typeMapper.mapStringToBigInteger(suffixStart, lengthToPad, partitionColumn, c); BigInteger bigIntegerEnd = - (BigInteger) typeMapper.mapStringToBigInteger(end, lengthToPad, partitionColumn, c); + (BigInteger) typeMapper.mapStringToBigInteger(suffixEnd, lengthToPad, partitionColumn, c); BigInteger bigIntegerSplit = splitBigIntegers(bigIntegerStart, bigIntegerEnd); - return (String) typeMapper.unMapStringFromBigInteger(bigIntegerSplit, partitionColumn, c); + String suffixMid = + (String) typeMapper.unMapStringFromBigInteger(bigIntegerSplit, partitionColumn, c); + return commonPrefix + suffixMid; } @VisibleForTesting From d3fb44b57e697cd39fc3898cdcf3b86eeb461fcf Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Tue, 4 Aug 2026 06:22:51 +0000 Subject: [PATCH 18/20] refine split logic --- .../range/BoundarySplitterFactory.java | 14 ++++++---- .../range/BoundarySplitterFactoryTest.java | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java index 78902e4324..462fec825f 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java @@ -390,7 +390,8 @@ private static byte[] padLeadingZeroBytes(byte[] array, int expectedLength) { return result; } - private static String splitStrings( + @VisibleForTesting + protected static String splitStrings( String start, String end, PartitionColumn partitionColumn, @@ -416,10 +417,13 @@ private static String splitStrings( // To avoid undefined behaviour in the padding logic, we take the max of the input strings and // the partition column width. int commonPrefixLength = 0; - while (commonPrefixLength < start.length() - && commonPrefixLength < end.length() - && start.charAt(commonPrefixLength) == end.charAt(commonPrefixLength)) { - commonPrefixLength++; + while (commonPrefixLength < start.length() && commonPrefixLength < end.length()) { + int cpStart = start.codePointAt(commonPrefixLength); + int cpEnd = end.codePointAt(commonPrefixLength); + if (cpStart != cpEnd) { + break; + } + commonPrefixLength += Character.charCount(cpStart); } String commonPrefix = start.substring(0, commonPrefixLength); String suffixStart = start.substring(commonPrefixLength); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java index d1dd34f885..d4d43de8b7 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java @@ -733,6 +733,33 @@ public void testDurationBoundarySplitter() { .build())); } + @Test + public void testStringBoundarySplitterSurrogatePairs() { + CollationReference collationReference = + CollationReference.builder() + .setDbCharacterSet("utf8mb4") + .setDbCollation("utf8mb4_bin") + .setPadSpace(true) + .build(); + PartitionColumn partitionColumn = + PartitionColumn.builder() + .setColumnTypeName("VARCHAR") + .setColumnName("col1") + .setColumnClass(String.class) + .setStringMaxLength(200) + .setStringCollation(collationReference) + .build(); + + TestBoundaryTypeMapper typeMapper = new TestBoundaryTypeMapper(); + // Test with strings sharing a surrogate pair prefix (emoji 😀) + String start = "😀a"; + String end = "😀c"; + + String split = + BoundarySplitterFactory.splitStrings(start, end, partitionColumn, typeMapper, null); + assertThat(split).startsWith("😀b"); + } + /* Not for production as it does not look at collation ordering */ private class TestBoundaryTypeMapper implements BoundaryTypeMapper { From 50606756c1a196786ce59c17dfdb627b9a5d2425 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Tue, 4 Aug 2026 06:32:39 +0000 Subject: [PATCH 19/20] simplify IT --- .../v2/templates/MySQLDataTypesIT.java | 1 + ...MySQLSourceDbToSpanner4ByteStringPKIT.java | 121 ------------------ .../DataTypesIT/mysql-data-types.sql | 8 ++ .../DataTypesIT/mysql-spanner-schema.sql | 6 + .../spanner-schema.sql | 4 - 5 files changed, 15 insertions(+), 125 deletions(-) delete mode 100644 v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java delete mode 100644 v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java index ab9c7a3c3f..51bb25faa8 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java @@ -481,6 +481,7 @@ private Map>> getExpectedData() { "uuid_pk", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12")); + expectedData.put("utf8mb4_pk", createRows("utf8mb4_pk", "😀", "😁", "😂")); return expectedData; } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java deleted file mode 100644 index b760a8ab95..0000000000 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSourceDbToSpanner4ByteStringPKIT.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates; - -import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; - -import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; -import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.beam.it.common.PipelineLauncher; -import org.apache.beam.it.common.PipelineOperator; -import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.spanner.SpannerResourceManager; -import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; -import org.apache.beam.it.jdbc.JDBCResourceManager; -import org.apache.beam.it.jdbc.MySQLResourceManager; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) -@TemplateIntegrationTest(SourceDbToSpanner.class) -@RunWith(JUnit4.class) -public class MySQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { - private MySQLResourceManager mySQLResourceManager; - private SpannerResourceManager spannerResourceManager; - - private static final String TABLE_4BYTE = "table4bytepk"; - private static final String ID = "id"; - private static final String DESCRIPTION = "description"; - private static final String SPANNER_DDL_RESOURCE = - "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; - - private JDBCResourceManager.JDBCSchema getMySQL4ByteSchema() { - HashMap columns = new HashMap<>(); - columns.put(ID, "VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL"); - columns.put(DESCRIPTION, "VARCHAR(200)"); - return new JDBCResourceManager.JDBCSchema(columns, ID); - } - - private List> getMySQL4ByteData() { - List> data = new ArrayList<>(); - - Map row1 = new HashMap<>(); - row1.put(ID, "😀"); - row1.put(DESCRIPTION, "Grinning Face"); - data.add(row1); - - Map row2 = new HashMap<>(); - row2.put(ID, "😁"); - row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); - data.add(row2); - - Map row3 = new HashMap<>(); - row3.put(ID, "😂"); - row3.put(DESCRIPTION, "Face with Tears of Joy"); - data.add(row3); - - return data; - } - - @Before - public void setUp() { - mySQLResourceManager = setUpMySQLResourceManager(); - spannerResourceManager = setUpSpannerResourceManager(); - } - - @After - public void cleanUp() { - ResourceManagerUtils.cleanResources(spannerResourceManager, mySQLResourceManager); - } - - @Test - public void testMySqlToSpanner() throws IOException { - List> mySQL4ByteData = getMySQL4ByteData(); - mySQLResourceManager.createTable(TABLE_4BYTE, getMySQL4ByteSchema()); - mySQLResourceManager.write(TABLE_4BYTE, mySQL4ByteData); - - createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - - Map jobParameters = new HashMap<>(); - jobParameters.put("uniformizationStageCountHint", "-1"); - jobParameters.put("numPartitions", "5"); - - PipelineLauncher.LaunchInfo jobInfo = - launchDataflowJob( - getClass().getSimpleName(), - null, - null, - mySQLResourceManager, - spannerResourceManager, - jobParameters, - null); - PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo)); - assertThatResult(result).isLaunchFinished(); - - SpannerAsserts.assertThatStructs( - spannerResourceManager.readTableRecords(TABLE_4BYTE, ID, DESCRIPTION)) - .hasRecordsUnorderedCaseInsensitiveColumns(mySQL4ByteData); - } -} diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql index d89c51c844..6b02e6b1d4 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql @@ -912,3 +912,11 @@ CREATE TABLE IF NOT EXISTS `uuid_pk_table` ( ); INSERT INTO `uuid_pk_table` (`id`, `uuid_pk_col`) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); + +CREATE TABLE IF NOT EXISTS `utf8mb4_pk_table` ( + `id` VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PRIMARY KEY, + `utf8mb4_pk_col` VARCHAR(200) NOT NULL +); + +INSERT INTO `utf8mb4_pk_table` (`id`, `utf8mb4_pk_col`) VALUES ('😀', '😀'), ('😁', '😁'), ('😂', '😂'); + diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql index 8448be66dd..7aa8ae709f 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql @@ -556,3 +556,9 @@ CREATE TABLE IF NOT EXISTS uuid_pk_table ( id UUID NOT NULL, uuid_pk_col UUID NOT NULL, ) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS utf8mb4_pk_table ( + id STRING(200) NOT NULL, + utf8mb4_pk_col STRING(200) NOT NULL, +) PRIMARY KEY(id); + diff --git a/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql deleted file mode 100644 index 146aed9c34..0000000000 --- a/v2/sourcedb-to-spanner/src/test/resources/SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE table4bytepk ( - id STRING(200), - description STRING(200) -) PRIMARY KEY (id); From 23e58eebfdf3917849154b0e4360e7f9f5dab72e Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Tue, 4 Aug 2026 18:22:41 +0000 Subject: [PATCH 20/20] add num partitions --- .../google/cloud/teleport/v2/templates/MySQLDataTypesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java index 51bb25faa8..ef67377e76 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java @@ -86,7 +86,7 @@ public void allTypesTest() throws Exception { null, mySQLResourceManager, spannerResourceManager, - Map.of("maxConnections", "4"), + Map.of("maxConnections", "4", "numPartitions", "10"), null); PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo, Duration.ofMinutes(15L)));