diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java index 82acaeed06..462fec825f 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactory.java @@ -39,6 +39,8 @@ public class BoundarySplitterFactory { private static final BigInteger SECONDS_TO_NANOS = BigInteger.valueOf(Duration.ofSeconds(1).toNanos()); + @VisibleForTesting protected static final int MAX_STRING_PARTITION_PAD_LENGTH = 300; + private static final ImmutableMap> splittermap = ImmutableMap.>builder() .put( @@ -388,7 +390,8 @@ private static byte[] padLeadingZeroBytes(byte[] array, int expectedLength) { return result; } - private static String splitStrings( + @VisibleForTesting + protected static String splitStrings( String start, String end, PartitionColumn partitionColumn, @@ -413,15 +416,33 @@ private static String splitStrings( // during a run. // To avoid undefined behaviour in the padding logic, we take the max of the input strings and // the partition column width. + int commonPrefixLength = 0; + while (commonPrefixLength < start.length() && commonPrefixLength < end.length()) { + int cpStart = start.codePointAt(commonPrefixLength); + int cpEnd = end.codePointAt(commonPrefixLength); + if (cpStart != cpEnd) { + break; + } + commonPrefixLength += Character.charCount(cpStart); + } + String commonPrefix = start.substring(0, commonPrefixLength); + String suffixStart = start.substring(commonPrefixLength); + String suffixEnd = end.substring(commonPrefixLength); + int lengthToPad = Math.max( - Math.max(start.length(), end.length()), partitionColumn.stringMaxLength().intValue()); + Math.max(suffixStart.length(), suffixEnd.length()), + Math.min( + Math.max(0, partitionColumn.stringMaxLength().intValue() - commonPrefixLength), + MAX_STRING_PARTITION_PAD_LENGTH)); BigInteger bigIntegerStart = - (BigInteger) typeMapper.mapStringToBigInteger(start, lengthToPad, partitionColumn, c); + (BigInteger) typeMapper.mapStringToBigInteger(suffixStart, lengthToPad, partitionColumn, c); BigInteger bigIntegerEnd = - (BigInteger) typeMapper.mapStringToBigInteger(end, lengthToPad, partitionColumn, c); + (BigInteger) typeMapper.mapStringToBigInteger(suffixEnd, lengthToPad, partitionColumn, c); BigInteger bigIntegerSplit = splitBigIntegers(bigIntegerStart, bigIntegerEnd); - return (String) typeMapper.unMapStringFromBigInteger(bigIntegerSplit, partitionColumn, c); + String suffixMid = + (String) typeMapper.unMapStringFromBigInteger(bigIntegerSplit, partitionColumn, c); + return commonPrefix + suffixMid; } @VisibleForTesting diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java index 789bbe7446..e0cd3a6dfa 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndex.java @@ -16,6 +16,7 @@ package com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper; import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.Serializable; @@ -43,7 +44,7 @@ public abstract class CollationIndex implements Serializable { * Map of character to it's index position based on collation order. Helps us map a string to big * integer. */ - public abstract ImmutableMap characterToIndex(); + public abstract ImmutableMap characterToIndex(); /** * Map if Index back to character based on collation order. Helps us unmap a big integer to @@ -51,7 +52,7 @@ public abstract class CollationIndex implements Serializable { * case-insensitive collations, 'a' and 'A' will have the same index in {@link * #characterToIndex()} and {@link #indexToCharacter()} will map the index to 'A'. */ - public abstract ImmutableMap indexToCharacter(); + public abstract ImmutableMap indexToCharacter(); public static CollationIndex.Builder builder() { return new AutoValue_CollationIndex.Builder(); @@ -61,11 +62,11 @@ public long getCharsetSize() { return indexToCharacter().size(); } - public long getOrdinalPosition(Character c) { + public long getOrdinalPosition(String c) { return characterToIndex().get(c); } - public Character getCharacterFromPosition(Long position) { + public String getCharacterFromPosition(Long position) { return indexToCharacter().get(position); } @@ -80,15 +81,19 @@ public abstract static class Builder { abstract CollationIndexType indexType(); - private Map charToIndexCache = new HashMap<>(); - private Map indexToCharacterCache = new HashMap<>(); - private Map indexToCharacterReverseCache = new HashMap<>(); + private Map charToIndexCache = new HashMap<>(); + private Map indexToCharacterCache = new HashMap<>(); + private Map indexToCharacterReverseCache = new HashMap<>(); - abstract Builder setIndexToCharacter(ImmutableMap value); + abstract Builder setIndexToCharacter(ImmutableMap value); - abstract Builder setCharacterToIndex(ImmutableMap value); + abstract Builder setCharacterToIndex(ImmutableMap value); + + public Builder addCharacter(String charsetChar, String equivalentChar, Long index) { + Preconditions.checkNotNull(charsetChar, "charsetChar cannot be null"); + Preconditions.checkNotNull(equivalentChar, "equivalentChar cannot be null"); + Preconditions.checkNotNull(index, "index cannot be null"); - public Builder addCharacter(Character charsetChar, Character equivalentChar, Long index) { logger.debug( "Registering character order for {}, index-type = {}, character = {}, equivalentCharacter = {}, index = {}, isBlank = {}", collationReference(), diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java index 910216af97..6e8826dcc3 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapper.java @@ -81,7 +81,7 @@ public abstract class CollationMapper implements Serializable { * using utf8mb4), 'b') = 'ab' COLLATE ;} returns 1. TODO(vardhanvthigle): Check this * behavior for PG and other databases. */ - public abstract ImmutableSet emptyCharacters(); + public abstract ImmutableSet emptyCharacters(); /** * Space Characters. MySQL ignores trailing space characters in comparisons for PAD space @@ -90,11 +90,11 @@ public abstract class CollationMapper implements Serializable { * (UNHEX(C2H0)) when the collation is Pad Space. These have same behavior to ascii space as far * as trailing or non-trailing comparison is concerned. */ - public abstract ImmutableSet spaceCharacters(); + public abstract ImmutableSet spaceCharacters(); @Memoized String allSpaceCharacters() { - return this.spaceCharacters().stream().map(String::valueOf).collect(Collectors.joining("")); + return this.spaceCharacters().stream().collect(Collectors.joining("")); } @Memoized @@ -103,8 +103,7 @@ String emptyReplacePattern() { return ""; } return "[" - + Pattern.quote( - this.emptyCharacters().stream().map(String::valueOf).collect(Collectors.joining(""))) + + Pattern.quote(this.emptyCharacters().stream().collect(Collectors.joining(""))) + "]"; } @@ -131,6 +130,14 @@ public BigInteger mapString(@Nullable String element, int lengthToPad) { if (element == null) { return BigInteger.valueOf(-1); } + // 'ret' stores the mapped value using a variable-base encoding. + // The base (charset size) can change depending on whether it's the trailing position + // in a pad-space collation. + // Example: For string "abcd" with lengthToPad = 6, let non-trailing base = 100 and trailing + // base = 90. + // If ordinals are a=1, b=2, c=3, d=4, the mapping evaluates to: + // ret = ((((1 * 100 + 2) * 100) + 3) * 90 + 4) * (100 ^ 2) + // unMapString reverses this by extracting modulo the trailing base first. BigInteger ret = BigInteger.ZERO; // MySQL ignores empty character in string comparisons. @@ -150,14 +157,21 @@ public BigInteger mapString(@Nullable String element, int lengthToPad) { } // Convert the string to BigInteger. - for (int index = 0; index < element.length(); index++) { - Character c = element.charAt(index); + java.util.List codePoints = + element + .codePoints() + .mapToObj(cp -> new String(Character.toChars(cp))) + .collect(Collectors.toList()); + for (int index = 0; index < codePoints.size(); index++) { + String c = codePoints.get(index); ret = - ret.multiply(BigInteger.valueOf(getCharsetSize(index == (element.length() - 1)))) - .add(BigInteger.valueOf(getOrdinalPosition(c, index == (element.length() - 1)))); + ret.multiply(BigInteger.valueOf(getCharsetSize(index == (codePoints.size() - 1)))) + .add(BigInteger.valueOf(getOrdinalPosition(c, index == (codePoints.size() - 1)))); } - for (int index = element.length(); index < lengthToPad; index++) { - ret = ret.multiply(BigInteger.valueOf(getCharsetSize(index == (element.length() - 1)))); + if (lengthToPad > codePoints.size()) { + ret = + ret.multiply( + BigInteger.valueOf(getCharsetSize(false)).pow(lengthToPad - codePoints.size())); } return ret; } @@ -188,16 +202,16 @@ public String unMapString(BigInteger element) { } // Base Case that the string just represents single character - if (element == BigInteger.ZERO) { - char c = getCharacterFromPosition(element.longValue(), true); - return String.valueOf(c); + if (element.equals(BigInteger.ZERO)) { + String c = getCharacterFromPosition(element.longValue(), true); + return c; } - while (element != BigInteger.ZERO) { + while (!element.equals(BigInteger.ZERO)) { long charsetSize = getCharsetSize(index == 0); BigInteger reminder = element.mod(BigInteger.valueOf(charsetSize)); - char c = getCharacterFromPosition(reminder.longValue(), (index == 0)); + String c = getCharacterFromPosition(reminder.longValue(), (index == 0)); word.append(c); element = element.divide(BigInteger.valueOf(charsetSize)); @@ -275,13 +289,13 @@ private long getCharsetSize(boolean lastCharacter) { : this.allPositionsIndex().getCharsetSize(); } - private long getOrdinalPosition(Character c, boolean lastCharacter) { + private long getOrdinalPosition(String c, boolean lastCharacter) { return (lastCharacter && collationReference().padSpace()) ? this.trailingPositionsPadSpace().getOrdinalPosition(c) : this.allPositionsIndex().getOrdinalPosition(c); } - private Character getCharacterFromPosition(long ordinalPosition, boolean firstIteration) { + private String getCharacterFromPosition(long ordinalPosition, boolean firstIteration) { return (firstIteration && collationReference().padSpace()) ? this.trailingPositionsPadSpace().getCharacterFromPosition(ordinalPosition) : this.allPositionsIndex().getCharacterFromPosition(ordinalPosition); @@ -307,9 +321,9 @@ public abstract static class Builder { abstract CollationIndex.Builder trailingPositionsPadSpaceBuilder(); - abstract ImmutableSet.Builder emptyCharactersBuilder(); + abstract ImmutableSet.Builder emptyCharactersBuilder(); - abstract ImmutableSet.Builder spaceCharactersBuilder(); + abstract ImmutableSet.Builder spaceCharactersBuilder(); public Builder addCharacter(CollationOrderRow collationOrderRow) { diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java index 633a60fcab..a4d4a7fcaf 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRow.java @@ -41,10 +41,10 @@ public abstract class CollationOrderRow { private static final Logger logger = LoggerFactory.getLogger(CollationOrderRow.class); /** Character in the character set. */ - public abstract Character charsetChar(); + public abstract String charsetChar(); /** A character with lowest rank charset_char character is equal to as per the collation. */ - public abstract Character equivalentChar(); + public abstract String equivalentChar(); /** 0 offset rank of this character as per the collation sort ordering at all positions. */ public abstract Long codepointRank(); @@ -54,7 +54,7 @@ public abstract class CollationOrderRow { * trailing position, in case a PAD SPACE comparison is needed. Unless you are looking at space * like characters, this will be exactly same as equivalent_character. */ - public abstract Character equivalentCharPadSpace(); + public abstract String equivalentCharPadSpace(); /** * A character with lowest rank charset_char character is equal to as per the collation at @@ -111,20 +111,25 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { isSpace); Preconditions.checkArgument( - charSetChar.length() <= 1, "Found a long character in collation output " + charSetChar); + charSetChar.codePointCount(0, charSetChar.length()) <= 1, + "Found a multi-codepoint character in collation output: " + charSetChar); Preconditions.checkArgument( - equivalentCharsetChar.length() <= 1, - "Found a long equivalent character in collation output " + equivalentCharsetChar); - Preconditions.checkArgument( - equivalentCharsetCharPadSpace.length() <= 1, - "Found a long equivalent character for pad space in collation output " + equivalentCharsetChar.codePointCount(0, equivalentCharsetChar.length()) <= 1, + "Found a multi-codepoint equivalent character in collation output: " + equivalentCharsetChar); + Preconditions.checkArgument( + equivalentCharsetCharPadSpace == null + || equivalentCharsetCharPadSpace.codePointCount( + 0, equivalentCharsetCharPadSpace.length()) + <= 1, + "Found a multi-codepoint equivalent character for pad space in collation output: " + + equivalentCharsetCharPadSpace); return CollationOrderRow.builder() - .setCharsetChar(charSetChar.charAt(0)) - .setEquivalentChar(equivalentCharsetChar.charAt(0)) + .setCharsetChar(charSetChar) + .setEquivalentChar(equivalentCharsetChar) .setCodepointRank(codePointRank) - .setEquivalentCharPadSpace(equivalentCharsetCharPadSpace.charAt(0)) + .setEquivalentCharPadSpace(equivalentCharsetCharPadSpace) .setCodepointRankPadSpace(codePointRankPadSpace) .setIsEmpty(isEmpty) .setIsSpace(isSpace) @@ -134,13 +139,13 @@ public static CollationOrderRow fromRS(ResultSet rs) throws SQLException { @AutoValue.Builder public abstract static class Builder { - public abstract Builder setCharsetChar(Character value); + public abstract Builder setCharsetChar(String value); - public abstract Builder setEquivalentChar(Character value); + public abstract Builder setEquivalentChar(String value); public abstract Builder setCodepointRank(Long value); - public abstract Builder setEquivalentCharPadSpace(Character value); + public abstract Builder setEquivalentCharPadSpace(String value); public abstract Builder setCodepointRankPadSpace(Long value); diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql index bdd2168447..3b32052351 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/mysql_collation_order_query.sql @@ -11,74 +11,50 @@ SET @db_collation = 'collation_replacement_tag'; --- A union of single byte literals from 0x00 to 0xff. -SET @byte_literals = CONCAT( - 'SELECT ''00'' AS h UNION ALL SELECT ''01'' UNION ALL SELECT ''02'' UNION ALL SELECT ''03'' UNION ALL SELECT ''04'' UNION ALL SELECT ''05'' UNION ALL SELECT ''06'' UNION ALL SELECT ''07'' UNION ALL SELECT ''08'' UNION ALL SELECT ''09'' UNION ALL SELECT ''0a'' UNION ALL SELECT ''0b'' UNION ALL SELECT ''0c'' UNION ALL SELECT ''0d'' UNION ALL SELECT ''0e'' UNION ALL SELECT ''0f''', -'UNION ALL SELECT ''10'' AS h UNION ALL SELECT ''11'' UNION ALL SELECT ''12'' UNION ALL SELECT ''13'' UNION ALL SELECT ''14'' UNION ALL SELECT ''15'' UNION ALL SELECT ''16'' UNION ALL SELECT ''17'' UNION ALL SELECT ''18'' UNION ALL SELECT ''19'' UNION ALL SELECT ''1a'' UNION ALL SELECT ''1b'' UNION ALL SELECT ''1c'' UNION ALL SELECT ''1d'' UNION ALL SELECT ''1e'' UNION ALL SELECT ''1f''', -'UNION ALL SELECT ''20'' AS h UNION ALL SELECT ''21'' UNION ALL SELECT ''22'' UNION ALL SELECT ''23'' UNION ALL SELECT ''24'' UNION ALL SELECT ''25'' UNION ALL SELECT ''26'' UNION ALL SELECT ''27'' UNION ALL SELECT ''28'' UNION ALL SELECT ''29'' UNION ALL SELECT ''2a'' UNION ALL SELECT ''2b'' UNION ALL SELECT ''2c'' UNION ALL SELECT ''2d'' UNION ALL SELECT ''2e'' UNION ALL SELECT ''2f''', -'UNION ALL SELECT ''30'' AS h UNION ALL SELECT ''31'' UNION ALL SELECT ''32'' UNION ALL SELECT ''33'' UNION ALL SELECT ''34'' UNION ALL SELECT ''35'' UNION ALL SELECT ''36'' UNION ALL SELECT ''37'' UNION ALL SELECT ''38'' UNION ALL SELECT ''39'' UNION ALL SELECT ''3a'' UNION ALL SELECT ''3b'' UNION ALL SELECT ''3c'' UNION ALL SELECT ''3d'' UNION ALL SELECT ''3e'' UNION ALL SELECT ''3f''', -'UNION ALL SELECT ''40'' AS h UNION ALL SELECT ''41'' UNION ALL SELECT ''42'' UNION ALL SELECT ''43'' UNION ALL SELECT ''44'' UNION ALL SELECT ''45'' UNION ALL SELECT ''46'' UNION ALL SELECT ''47'' UNION ALL SELECT ''48'' UNION ALL SELECT ''49'' UNION ALL SELECT ''4a'' UNION ALL SELECT ''4b'' UNION ALL SELECT ''4c'' UNION ALL SELECT ''4d'' UNION ALL SELECT ''4e'' UNION ALL SELECT ''4f''', -'UNION ALL SELECT ''50'' AS h UNION ALL SELECT ''51'' UNION ALL SELECT ''52'' UNION ALL SELECT ''53'' UNION ALL SELECT ''54'' UNION ALL SELECT ''55'' UNION ALL SELECT ''56'' UNION ALL SELECT ''57'' UNION ALL SELECT ''58'' UNION ALL SELECT ''59'' UNION ALL SELECT ''5a'' UNION ALL SELECT ''5b'' UNION ALL SELECT ''5c'' UNION ALL SELECT ''5d'' UNION ALL SELECT ''5e'' UNION ALL SELECT ''5f''', -'UNION ALL SELECT ''60'' AS h UNION ALL SELECT ''61'' UNION ALL SELECT ''62'' UNION ALL SELECT ''63'' UNION ALL SELECT ''64'' UNION ALL SELECT ''65'' UNION ALL SELECT ''66'' UNION ALL SELECT ''67'' UNION ALL SELECT ''68'' UNION ALL SELECT ''69'' UNION ALL SELECT ''6a'' UNION ALL SELECT ''6b'' UNION ALL SELECT ''6c'' UNION ALL SELECT ''6d'' UNION ALL SELECT ''6e'' UNION ALL SELECT ''6f''', -'UNION ALL SELECT ''70'' AS h UNION ALL SELECT ''71'' UNION ALL SELECT ''72'' UNION ALL SELECT ''73'' UNION ALL SELECT ''74'' UNION ALL SELECT ''75'' UNION ALL SELECT ''76'' UNION ALL SELECT ''77'' UNION ALL SELECT ''78'' UNION ALL SELECT ''79'' UNION ALL SELECT ''7a'' UNION ALL SELECT ''7b'' UNION ALL SELECT ''7c'' UNION ALL SELECT ''7d'' UNION ALL SELECT ''7e'' UNION ALL SELECT ''7f''', -'UNION ALL SELECT ''80'' AS h UNION ALL SELECT ''81'' UNION ALL SELECT ''82'' UNION ALL SELECT ''83'' UNION ALL SELECT ''84'' UNION ALL SELECT ''85'' UNION ALL SELECT ''86'' UNION ALL SELECT ''87'' UNION ALL SELECT ''88'' UNION ALL SELECT ''89'' UNION ALL SELECT ''8a'' UNION ALL SELECT ''8b'' UNION ALL SELECT ''8c'' UNION ALL SELECT ''8d'' UNION ALL SELECT ''8e'' UNION ALL SELECT ''8f''', -'UNION ALL SELECT ''90'' AS h UNION ALL SELECT ''91'' UNION ALL SELECT ''92'' UNION ALL SELECT ''93'' UNION ALL SELECT ''94'' UNION ALL SELECT ''95'' UNION ALL SELECT ''96'' UNION ALL SELECT ''97'' UNION ALL SELECT ''98'' UNION ALL SELECT ''99'' UNION ALL SELECT ''9a'' UNION ALL SELECT ''9b'' UNION ALL SELECT ''9c'' UNION ALL SELECT ''9d'' UNION ALL SELECT ''9e'' UNION ALL SELECT ''9f''', -'UNION ALL SELECT ''a0'' AS h UNION ALL SELECT ''a1'' UNION ALL SELECT ''a2'' UNION ALL SELECT ''a3'' UNION ALL SELECT ''a4'' UNION ALL SELECT ''a5'' UNION ALL SELECT ''a6'' UNION ALL SELECT ''a7'' UNION ALL SELECT ''a8'' UNION ALL SELECT ''a9'' UNION ALL SELECT ''aa'' UNION ALL SELECT ''ab'' UNION ALL SELECT ''ac'' UNION ALL SELECT ''ad'' UNION ALL SELECT ''ae'' UNION ALL SELECT ''af''', -'UNION ALL SELECT ''b0'' AS h UNION ALL SELECT ''b1'' UNION ALL SELECT ''b2'' UNION ALL SELECT ''b3'' UNION ALL SELECT ''b4'' UNION ALL SELECT ''b5'' UNION ALL SELECT ''b6'' UNION ALL SELECT ''b7'' UNION ALL SELECT ''b8'' UNION ALL SELECT ''b9'' UNION ALL SELECT ''ba'' UNION ALL SELECT ''bb'' UNION ALL SELECT ''bc'' UNION ALL SELECT ''bd'' UNION ALL SELECT ''be'' UNION ALL SELECT ''bf''', -'UNION ALL SELECT ''c0'' AS h UNION ALL SELECT ''c1'' UNION ALL SELECT ''c2'' UNION ALL SELECT ''c3'' UNION ALL SELECT ''c4'' UNION ALL SELECT ''c5'' UNION ALL SELECT ''c6'' UNION ALL SELECT ''c7'' UNION ALL SELECT ''c8'' UNION ALL SELECT ''c9'' UNION ALL SELECT ''ca'' UNION ALL SELECT ''cb'' UNION ALL SELECT ''cc'' UNION ALL SELECT ''cd'' UNION ALL SELECT ''ce'' UNION ALL SELECT ''cf''', -'UNION ALL SELECT ''d0'' AS h UNION ALL SELECT ''d1'' UNION ALL SELECT ''d2'' UNION ALL SELECT ''d3'' UNION ALL SELECT ''d4'' UNION ALL SELECT ''d5'' UNION ALL SELECT ''d6'' UNION ALL SELECT ''d7'' UNION ALL SELECT ''d8'' UNION ALL SELECT ''d9'' UNION ALL SELECT ''da'' UNION ALL SELECT ''db'' UNION ALL SELECT ''dc'' UNION ALL SELECT ''dd'' UNION ALL SELECT ''de'' UNION ALL SELECT ''df''', -'UNION ALL SELECT ''e0'' AS h UNION ALL SELECT ''e1'' UNION ALL SELECT ''e2'' UNION ALL SELECT ''e3'' UNION ALL SELECT ''e4'' UNION ALL SELECT ''e5'' UNION ALL SELECT ''e6'' UNION ALL SELECT ''e7'' UNION ALL SELECT ''e8'' UNION ALL SELECT ''e9'' UNION ALL SELECT ''ea'' UNION ALL SELECT ''eb'' UNION ALL SELECT ''ec'' UNION ALL SELECT ''ed'' UNION ALL SELECT ''ee'' UNION ALL SELECT ''ef''', -'UNION ALL SELECT ''f0'' AS h UNION ALL SELECT ''f1'' UNION ALL SELECT ''f2'' UNION ALL SELECT ''f3'' UNION ALL SELECT ''f4'' UNION ALL SELECT ''f5'' UNION ALL SELECT ''f6'' UNION ALL SELECT ''f7'' UNION ALL SELECT ''f8'' UNION ALL SELECT ''f9'' UNION ALL SELECT ''fa'' UNION ALL SELECT ''fb'' UNION ALL SELECT ''fc'' UNION ALL SELECT ''fd'' UNION ALL SELECT ''fe'' UNION ALL SELECT ''ff''' -); - --- Four byte code points. -SET @four_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h, t4.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t4 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Three byte code points. -SET @three_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h, t3.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - 'LEFT JOIN (', @byte_literals, ') AS t3 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Two byte code points. -SET @two_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(CONCAT(t1.h, t2.h)) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1 ', - 'LEFT JOIN (', @byte_literals, ') AS t2 ON 1=1 ', - ') AS dt ', - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); - --- Single byte code points. -SET @one_byte_codepoints = CONCAT( - '(SELECT * FROM (SELECT ', - 'CONVERT(UNHEX(t1.h) USING ', @db_charset, ') AS charset_char ', - 'FROM (', @byte_literals, ') AS t1', - ') AS dt ', -- derived table - 'WHERE CHAR_LENGTH(charset_char) <= 1 AND charset_char IS NOT NULL' - ')' -); +-- Enumerating valid utf8mb4 byte sequences +-- There are a total of 1,112,064 valid code points within the Unicode codespace. +-- All of them are generated by this enumeration. +-- https://en.wikipedia.org/wiki/Unicode#:~:text=There%20are%20a%20total%20of%201112064%20valid%20code%20points%20within%20the%20codespace +SET @all_chars = ' +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n, n5.n, n6.n, n7.n, n8.n) AS hex_val +FROM (SELECT ''F'' AS n) n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n5 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n6 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n7 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n8 +UNION ALL +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n, n5.n, n6.n) AS hex_val +FROM (SELECT ''E'' AS n) n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n5 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n6 +UNION ALL +SELECT CONCAT(n1.n, n2.n, n3.n, n4.n) AS hex_val +FROM (SELECT ''C'' AS n UNION ALL SELECT ''D'') n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +CROSS JOIN (SELECT ''8'' AS n UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'') n3 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n4 +UNION ALL +SELECT CONCAT(n1.n, n2.n) AS hex_val +FROM (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'') n1 +CROSS JOIN (SELECT ''0'' AS n UNION ALL SELECT ''1'' UNION ALL SELECT ''2'' UNION ALL SELECT ''3'' UNION ALL SELECT ''4'' UNION ALL SELECT ''5'' UNION ALL SELECT ''6'' UNION ALL SELECT ''7'' UNION ALL SELECT ''8'' UNION ALL SELECT ''9'' UNION ALL SELECT ''A'' UNION ALL SELECT ''B'' UNION ALL SELECT ''C'' UNION ALL SELECT ''D'' UNION ALL SELECT ''E'' UNION ALL SELECT ''F'') n2 +'; + +SET @charset_chars = CONCAT( + '(SELECT charset_char FROM ( ', + 'SELECT hex_val, CONVERT(UNHEX(hex_val) USING utf8mb4) AS utf8_char, ', + 'CONVERT(CONVERT(UNHEX(hex_val) USING utf8mb4) USING ', @db_charset, ') AS charset_char ', + 'FROM ( ', @all_chars, ' ) AS all_chars ', + 'HAVING utf8_char IS NOT NULL AND hex_val NOT BETWEEN ''EDA080'' AND ''EDBFBF'' ', + ') AS valid_utf8_chars ', + 'WHERE charset_char IS NOT NULL AND (HEX(CONVERT(charset_char USING utf8mb4)) != ''3F'' OR hex_val = ''3F'') ', +')'); --- all variable length code points representing a single character within the @db_charset from length 0 till 4. -SET @charset_chars = CONCAT(@three_byte_codepoints, ' UNION ALL ', @two_byte_codepoints, ' UNION ALL ', @one_byte_codepoints); SET @SPACE=CONCAT('CONVERT('' '' USING ', @db_charset,')'); SET @ALPHABET=CONCAT('CONVERT(''a'' USING ', @db_charset,')'); diff --git a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql index d2bd311db1..df6fe69baf 100644 --- a/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql +++ b/v2/sourcedb-to-spanner/src/main/resources/sql/postgresql_collation_order_query.sql @@ -5,7 +5,7 @@ CREATE OR REPLACE FUNCTION pg_temp.safe_convert_from(codepoint int8, charset text) RETURNS return_type_replacement_tag AS ' BEGIN - RETURN convert_from(decode(to_hex(codepoint), ''hex''), charset); + RETURN convert_from(convert_to(chr(codepoint::integer), charset), charset); EXCEPTION WHEN OTHERS THEN RETURN NULL; END; @@ -17,6 +17,9 @@ WITH -- Generate 1 byte (U+0000 to U+007F), 2 bytes (U+0080 to U+07FF), -- 3 bytes (U+0800 to U+FFFF), and 4 bytes (U+10000 to U+10FFFF) unicode -- codepoints +-- There are a total of 1,112,064 valid code points within the Unicode codespace. +-- All of them are generated by this enumeration. +-- https://en.wikipedia.org/wiki/Unicode#:~:text=There%20are%20a%20total%20of%201112064%20valid%20code%20points%20within%20the%20codespace charset_chars AS ( SELECT * FROM ( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java index d1dd34f885..d4d43de8b7 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/range/BoundarySplitterFactoryTest.java @@ -733,6 +733,33 @@ public void testDurationBoundarySplitter() { .build())); } + @Test + public void testStringBoundarySplitterSurrogatePairs() { + CollationReference collationReference = + CollationReference.builder() + .setDbCharacterSet("utf8mb4") + .setDbCollation("utf8mb4_bin") + .setPadSpace(true) + .build(); + PartitionColumn partitionColumn = + PartitionColumn.builder() + .setColumnTypeName("VARCHAR") + .setColumnName("col1") + .setColumnClass(String.class) + .setStringMaxLength(200) + .setStringCollation(collationReference) + .build(); + + TestBoundaryTypeMapper typeMapper = new TestBoundaryTypeMapper(); + // Test with strings sharing a surrogate pair prefix (emoji 😀) + String start = "😀a"; + String end = "😀c"; + + String split = + BoundarySplitterFactory.splitStrings(start, end, partitionColumn, typeMapper, null); + assertThat(split).startsWith("😀b"); + } + /* Not for production as it does not look at collation ordering */ private class TestBoundaryTypeMapper implements BoundaryTypeMapper { diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java index dd23372717..a0864a176a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/BoundaryTypeMapperImplTest.java @@ -58,9 +58,9 @@ public void testBoundaryTypeMappingImpl() { collationMapperBuilder .addCharacter( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("a") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -68,9 +68,9 @@ public void testBoundaryTypeMappingImpl() { .build()) .addCharacter( CollationOrderRow.builder() - .setCharsetChar('A') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("A") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java index 3a62e7e526..0c440e6b8f 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationIndexTest.java @@ -38,8 +38,8 @@ public void testCollationIndexBasic() { CollationIndex.builder() .setCollationReference(testCollationReference) .setIndexType(CollationIndexType.TRAILING_POSITION_PAD_SPACE) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 0L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 0L) .build(); assertThat(collationIndex.indexType()) @@ -47,8 +47,8 @@ public void testCollationIndexBasic() { assertThat(collationIndex.collationReference()).isEqualTo(testCollationReference); assertThat(collationIndex.getCharsetSize()).isEqualTo(1); assertThat(collationIndex.characterToIndex().size()).isEqualTo(2); - assertThat(collationIndex.getCharacterFromPosition(0L)).isEqualTo('A'); - assertThat(collationIndex.getOrdinalPosition('a')).isEqualTo(0L); + assertThat(collationIndex.getCharacterFromPosition(0L)).isEqualTo("A"); + assertThat(collationIndex.getOrdinalPosition("a")).isEqualTo(0L); } @Test @@ -60,6 +60,32 @@ public void testCollationIndexPreConditions() { .setPadSpace(true) .build(); + // Null arguments + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter(null, "A", 0L) + .build()); + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter("a", null, 0L) + .build()); + assertThrows( + NullPointerException.class, + () -> + CollationIndex.builder() + .setIndexType(CollationIndexType.ALL_POSITIONS) + .setCollationReference(testCollationReference) + .addCharacter("a", "A", null) + .build()); + // Duplicate Characters assertThrows( IllegalStateException.class, @@ -67,8 +93,8 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('a', 'A', 0L) + .addCharacter("a", "A", 0L) + .addCharacter("a", "A", 0L) .build()); // Duplicate Index assertThrows( @@ -77,16 +103,16 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 2L)); + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 2L)); assertThrows( IllegalStateException.class, () -> CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('z', 'Z', 0L)); + .addCharacter("a", "A", 0L) + .addCharacter("z", "Z", 0L)); // Index with Holes. assertThrows( IllegalStateException.class, @@ -94,10 +120,10 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'A', 0L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "A", 0L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); // Index Character not part of basic character set. assertThrows( @@ -106,10 +132,10 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'M', 0L) - .addCharacter('A', 'A', 5L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "M", 0L) + .addCharacter("A", "A", 5L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); // Index Character does not map to itself assertThrows( @@ -118,11 +144,11 @@ public void testCollationIndexPreConditions() { CollationIndex.builder() .setIndexType(CollationIndexType.ALL_POSITIONS) .setCollationReference(testCollationReference) - .addCharacter('a', 'A', 0L) - .addCharacter('A', 'M', 5L) - .addCharacter('M', 'M', 5L) - .addCharacter('z', 'Z', 10L) - .addCharacter('Z', 'Z', 10L) + .addCharacter("a", "A", 0L) + .addCharacter("A", "M", 5L) + .addCharacter("M", "M", 5L) + .addCharacter("z", "Z", 10L) + .addCharacter("Z", "Z", 10L) .build()); } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java index 4e62f36205..9e2c469d58 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationMapperTest.java @@ -71,9 +71,9 @@ public void testCollationMapperBasic() { for (Character c : enAlphabetsBuilder.build()) { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar(c) - .setEquivalentChar(Character.toUpperCase(c)) - .setEquivalentCharPadSpace(Character.toUpperCase(c)) + .setCharsetChar(String.valueOf(c)) + .setEquivalentChar(String.valueOf(Character.toUpperCase(c))) + .setEquivalentCharPadSpace(String.valueOf(Character.toUpperCase(c))) .setCodepointRank((long) (Character.toUpperCase(c) - 'A')) .setCodepointRankPadSpace((long) (Character.toUpperCase(c) - 'A')) .setIsEmpty(false) @@ -84,9 +84,9 @@ public void testCollationMapperBasic() { /** Add blank character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('\0') - .setEquivalentChar('\0') - .setEquivalentCharPadSpace('\0') + .setCharsetChar("\0") + .setEquivalentChar("\0") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(true) @@ -136,9 +136,9 @@ public void testCollationMapperPadSpace() { for (Character c : enAlphabetsBuilder.build()) { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar(c) - .setEquivalentChar(Character.toUpperCase(c)) - .setEquivalentCharPadSpace(Character.toUpperCase(c)) + .setCharsetChar(String.valueOf(c)) + .setEquivalentChar(String.valueOf(Character.toUpperCase(c))) + .setEquivalentCharPadSpace(String.valueOf(Character.toUpperCase(c))) .setCodepointRank((long) (Character.toUpperCase(c) - 'A' + 1)) .setCodepointRankPadSpace((long) (Character.toUpperCase(c) - 'A')) .setIsEmpty(false) @@ -149,9 +149,9 @@ public void testCollationMapperPadSpace() { /** Add Space Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar(' ') - .setEquivalentChar(' ') - .setEquivalentCharPadSpace('\0') + .setCharsetChar(" ") + .setEquivalentChar(" ") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -183,9 +183,9 @@ public void testCollationMapperEmptyStrings() { /* Add space character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar(' ') - .setEquivalentChar(' ') - .setEquivalentCharPadSpace('\0') + .setCharsetChar(" ") + .setEquivalentChar(" ") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -194,9 +194,9 @@ public void testCollationMapperEmptyStrings() { /* Add empty Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('\0') - .setEquivalentChar('\0') - .setEquivalentCharPadSpace('\0') + .setCharsetChar("\0") + .setEquivalentChar("\0") + .setEquivalentCharPadSpace("\0") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(true) @@ -224,9 +224,9 @@ public void testCollationMapperSingleCharacterString() { /* Add Single Character */ collationMapperBuilder.addCharacter( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('a') - .setEquivalentCharPadSpace('a') + .setCharsetChar("a") + .setEquivalentChar("a") + .setEquivalentCharPadSpace("a") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java index 1d600b9a2d..dd3078fde0 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/reader/io/jdbc/uniformsplitter/stringmapper/CollationOrderRowTest.java @@ -23,7 +23,6 @@ import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_EMPTY_COL; import static com.google.cloud.teleport.v2.reader.io.jdbc.uniformsplitter.stringmapper.CollationOrderRow.CollationsOrderQueryColumns.IS_SPACE_COL; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.when; import java.sql.ResultSet; @@ -42,9 +41,9 @@ public class CollationOrderRowTest { public void testCollationOrderRowBasic() { CollationOrderRow collationOrderRow = CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('A') - .setEquivalentCharPadSpace('A') + .setCharsetChar("a") + .setEquivalentChar("A") + .setEquivalentCharPadSpace("A") .setCodepointRank(1L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) @@ -52,9 +51,9 @@ public void testCollationOrderRowBasic() { .build(); assertThat(collationOrderRow.codepointRank()).isEqualTo(1L); assertThat(collationOrderRow.codepointRankPadSpace()).isEqualTo(0L); - assertThat(collationOrderRow.charsetChar()).isEqualTo('a'); - assertThat(collationOrderRow.equivalentChar()).isEqualTo('A'); - assertThat(collationOrderRow.equivalentCharPadSpace()).isEqualTo('A'); + assertThat(collationOrderRow.charsetChar()).isEqualTo("a"); + assertThat(collationOrderRow.equivalentChar()).isEqualTo("A"); + assertThat(collationOrderRow.equivalentCharPadSpace()).isEqualTo("A"); assertThat(collationOrderRow.isEmpty()).isFalse(); assertThat(collationOrderRow.isSpace()).isFalse(); } @@ -74,35 +73,13 @@ public void testCollationOrderRowFromRsBasic() throws SQLException { assertThat(collationOrderRow) .isEqualTo( CollationOrderRow.builder() - .setCharsetChar('a') - .setEquivalentChar('a') - .setEquivalentCharPadSpace('a') + .setCharsetChar("a") + .setEquivalentChar("a") + .setEquivalentCharPadSpace("a") .setCodepointRank(0L) .setCodepointRankPadSpace(0L) .setIsEmpty(false) .setIsSpace(false) .build()); } - - @Test - public void testCollationOrderRowFromRsException() throws SQLException { - int expcetedIllegalArgumentExceptionCount = 0; - when(mockResultSet.getString(CHARSET_CHAR_COL)).thenReturn("aa").thenReturn("a"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getString(EQUIVALENT_CHARSET_CHAR_COL)).thenReturn("a").thenReturn("aa"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getString(EQUIVALENT_CHARSET_CHAR_PAD_SPACE_COL)) - .thenReturn("a") - .thenReturn("a") - .thenReturn("aa"); - expcetedIllegalArgumentExceptionCount++; - when(mockResultSet.getLong(CODEPOINT_RANK_COL)).thenReturn(0L); - when(mockResultSet.getLong(CODEPOINT_RANK_COL)).thenReturn(0L); - when(mockResultSet.getBoolean(IS_EMPTY_COL)).thenReturn(false); - when(mockResultSet.getBoolean(IS_SPACE_COL)).thenReturn(false); - - for (int i = 0; i < expcetedIllegalArgumentExceptionCount; i++) { - assertThrows(IllegalArgumentException.class, () -> CollationOrderRow.fromRS(mockResultSet)); - } - } } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java index ab9c7a3c3f..ef67377e76 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java @@ -86,7 +86,7 @@ public void allTypesTest() throws Exception { null, mySQLResourceManager, spannerResourceManager, - Map.of("maxConnections", "4"), + Map.of("maxConnections", "4", "numPartitions", "10"), null); PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo, Duration.ofMinutes(15L))); @@ -481,6 +481,7 @@ private Map>> getExpectedData() { "uuid_pk", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12")); + expectedData.put("utf8mb4_pk", createRows("utf8mb4_pk", "😀", "😁", "😂")); return expectedData; } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java new file mode 100644 index 0000000000..844ba0f4ae --- /dev/null +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLSourceDbToSpanner4ByteStringPKIT.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.beam.it.jdbc.PostgresResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SourceDbToSpanner.class) +@RunWith(JUnit4.class) +public class PostgreSQLSourceDbToSpanner4ByteStringPKIT extends SourceDbToSpannerITBase { + private PostgresResourceManager postgreSQLResourceManager; + private SpannerResourceManager spannerResourceManager; + + private static final String TABLE = "table4bytepk"; + private static final String ID = "id"; + private static final String DESCRIPTION = "description"; + private static final String SPANNER_DDL_RESOURCE = + "SourceDbToSpanner4ByteStringPKIT/spanner-schema.sql"; + + private JDBCResourceManager.JDBCSchema getPostgreSQLSchema() { + HashMap columns = new HashMap<>(); + columns.put(ID, "VARCHAR(200) NOT NULL"); + columns.put(DESCRIPTION, "VARCHAR(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ID); + } + + private List> getPostgreSQLData() { + List> data = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put(ID, "😀"); + row1.put(DESCRIPTION, "Grinning Face"); + data.add(row1); + + Map row2 = new HashMap<>(); + row2.put(ID, "😁"); + row2.put(DESCRIPTION, "Beaming Face with Smiling Eyes"); + data.add(row2); + + Map row3 = new HashMap<>(); + row3.put(ID, "😂"); + row3.put(DESCRIPTION, "Face with Tears of Joy"); + data.add(row3); + + return data; + } + + @Before + public void setUp() { + postgreSQLResourceManager = setUpPostgreSQLResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources(spannerResourceManager, postgreSQLResourceManager); + } + + @Test + public void testPostgreSQLToSpanner() throws IOException { + List> postgreSQLData = getPostgreSQLData(); + postgreSQLResourceManager.createTable(TABLE, getPostgreSQLSchema()); + postgreSQLResourceManager.write(TABLE, postgreSQLData); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + postgreSQLResourceManager, + spannerResourceManager, + null, + null); + PipelineOperator.Result result = pipelineOperator().waitUntilDone(createConfig(jobInfo)); + assertThatResult(result).isLaunchFinished(); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(TABLE, ID, DESCRIPTION)) + .hasRecordsUnorderedCaseInsensitiveColumns(postgreSQLData); + } +} diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java index dbabfc4423..b0af7bc064 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java @@ -30,6 +30,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -37,6 +38,7 @@ import java.util.stream.Collectors; import org.apache.beam.it.cassandra.CassandraResourceManager; import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.ResourceManager; import org.apache.beam.it.common.utils.IORedirectUtil; import org.apache.beam.it.common.utils.PipelineUtils; @@ -396,4 +398,13 @@ private String driverClassNameFrom(JDBCResourceManager jdbcResourceManager) { throw new IllegalArgumentException(e); } } + + @Override + protected PipelineOperator.Config.Builder wrapConfiguration( + PipelineOperator.Config.Builder builder) { + if (System.getProperty("directRunnerTest") != null) { + return builder.setTimeoutAfter(Duration.ofMinutes(15)); + } + return builder; + } } diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql index d89c51c844..6b02e6b1d4 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-data-types.sql @@ -912,3 +912,11 @@ CREATE TABLE IF NOT EXISTS `uuid_pk_table` ( ); INSERT INTO `uuid_pk_table` (`id`, `uuid_pk_col`) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); + +CREATE TABLE IF NOT EXISTS `utf8mb4_pk_table` ( + `id` VARCHAR(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PRIMARY KEY, + `utf8mb4_pk_col` VARCHAR(200) NOT NULL +); + +INSERT INTO `utf8mb4_pk_table` (`id`, `utf8mb4_pk_col`) VALUES ('😀', '😀'), ('😁', '😁'), ('😂', '😂'); + diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql index 8448be66dd..7aa8ae709f 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/mysql-spanner-schema.sql @@ -556,3 +556,9 @@ CREATE TABLE IF NOT EXISTS uuid_pk_table ( id UUID NOT NULL, uuid_pk_col UUID NOT NULL, ) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS utf8mb4_pk_table ( + id STRING(200) NOT NULL, + utf8mb4_pk_col STRING(200) NOT NULL, +) PRIMARY KEY(id); +