From ac4f63936caf9c3f4373b53ea84dcdc649c5b75e Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Tue, 25 Aug 2026 08:26:57 +0000 Subject: [PATCH 1/7] added binary to string mappings --- ...reamToSpannerDataTypesAndExpressionIT.java | 4 +- .../v2/templates/MySQLDataTypesIT.java | 4 +- .../templates/MySQLDataTypesPGDialectIT.java | 4 +- .../v2/templates/PostgreSQLDataTypesIT.java | 10 ++-- .../PostgreSQLDataTypesPGDialectIT.java | 10 ++-- .../migrations/avro/AvroToValueMapper.java | 9 +++ .../source/mysql/MySQLDMLGenerator.java | 20 ++++++- .../templates/SpannerToMySqlDataTypesIT.java | 60 ++++++------------- .../SpannerToMySqlDataTypesPGDialectIT.java | 60 ++++++------------- .../source/mysql/MySQLDMLGeneratorTest.java | 50 ++++++++++++++++ 10 files changed, 127 insertions(+), 104 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java index 29736e5895..12e6fdcd48 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java @@ -268,7 +268,7 @@ private void validateResult( // These types are not mapped as expected, ignore them to avoid failing the // test. Set ignoredTypeMappings = - Set.of("bit_to_string", "date_to_string", "set_to_array", "spatial_geometrycollection"); + Set.of("date_to_string", "set_to_array", "spatial_geometrycollection"); // Validate supported data types. for (Map.Entry>> entry : expectedData.entrySet()) { String type = entry.getKey(); @@ -409,7 +409,7 @@ private Map>> getExpectedData() { "NULL")); expectedData.put("bit", createRows("bit", "f/////////8=", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java index 3367a9b831..cb5eb26c0a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java @@ -179,9 +179,7 @@ private Map>> getExpectedData() { expectedData.put("bit8", createRows("bit8", "0", "255", "NULL")); expectedData.put("bit1", createRows("bit1", "false", "true", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the long - // representation of the bits which is unexpected even if it's not necessarily incorrect - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java index 85f2c694e3..548cf1c37b 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java @@ -185,9 +185,7 @@ private Map>> getExpectedData() { expectedData.put("bit8", createRows("bit8", "0", "255", "NULL")); expectedData.put("bit1", createRows("bit1", "false", "true", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the long - // representation of the bits which is unexpected even if it's not necessarily incorrect - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java index b71be58f2a..5012ebfdf8 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java @@ -181,13 +181,11 @@ private Map>> getExpectedData() { result.put( "bit", createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the literal - // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]" - // result.put("bit_to_string", createRows("0", "1", "NULL")); + result.put( + "bit_to_string", + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - // bit_varying_to_string is commented out to avoid failing the test case; returned data is the - // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" - // result.put("bit_varying_to_string", createRows("5", "NULL")); + result.put("bit_varying_to_string", createRows("1100", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java index da149b5c94..565c479eeb 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java @@ -188,13 +188,11 @@ private Map>> getExpectedData() { result.put( "bit", createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the literal - // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]" - // result.put("bit_to_string", createRows("0", "1", "NULL")); + result.put( + "bit_to_string", + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - // bit_varying_to_string is commented out to avoid failing the test case; returned data is the - // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" - // result.put("bit_varying_to_string", createRows("5", "NULL")); + result.put("bit_varying_to_string", createRows("1100", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java index 28c1ce597c..e4ff71c846 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java @@ -331,6 +331,15 @@ static String avroFieldToString(Object recordValue, Schema fieldSchema) { if (recordValue == null) { return null; } + if (recordValue instanceof ByteBuffer) { + ByteBuffer byteBuffer = ((ByteBuffer) recordValue).duplicate(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); + } + if (recordValue instanceof byte[]) { + return new String((byte[]) recordValue, java.nio.charset.StandardCharsets.UTF_8); + } return recordValue.toString(); } catch (Exception e) { throw new AvroTypeConvertorException( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java index 9f8fec4891..75d49ba1e8 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java @@ -293,11 +293,17 @@ static String getColumnValueByType( case "multilinestring": case "polygon": case "multipolygon": + response = getQuotedEscapedString(colValue, spannerColType); + break; case "tinyblob": case "mediumblob": case "blob": case "longblob": - response = getQuotedEscapedString(colValue, spannerColType); + if (isStringType(spannerColType)) { + response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")"; + } else { + response = getQuotedEscapedString(colValue, spannerColType); + } break; case "timestamp": case "datetime": @@ -312,7 +318,11 @@ static String getColumnValueByType( break; case "binary": case "varbinary": - response = getBinaryString(colValue, spannerColType); + if (isStringType(spannerColType)) { + response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")"; + } else { + response = getBinaryString(colValue, spannerColType); + } break; default: response = colValue; @@ -320,6 +330,12 @@ static String getColumnValueByType( return response; } + private static boolean isStringType(String spannerColType) { + return "STRING".equalsIgnoreCase(spannerColType) + || "PG_VARCHAR".equalsIgnoreCase(spannerColType) + || "PG_TEXT".equalsIgnoreCase(spannerColType); + } + private static String escapeString(String input) { String cleanedNullBytes = StringUtils.replace(input, "\u0000", ""); cleanedNullBytes = StringUtils.replace(cleanedNullBytes, "'", "''"); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java index c2249017e5..4ed8b10d93 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java @@ -176,16 +176,7 @@ private void writeRowsInSpanner(Map> spannerTableData) { private ConditionCheck buildConditionCheck(Map> spannerTableData) { // These tables fail to migrate all expected rows, ignore them to avoid having to wait for the // timeout. - Set ignoredTables = - Set.of( - "binary_to_string", - "bit_to_string", - "set_to_array", - "blob_to_string", - "longblob_to_string", - "mediumblob_to_string", - "tinyblob_to_string", - "varbinary_to_string"); + Set ignoredTables = Set.of("set_to_array"); ConditionCheck combinedCondition = null; for (Map.Entry> entry : spannerTableData.entrySet()) { @@ -302,7 +293,7 @@ private Map> getSpannerTableData() { "bit", List.of(Value.bytesFromBase64("f/////////8="), Value.bytesFromBase64(null))); spannerRowData.put( "bit_to_bool", List.of(Value.bool(false), Value.bool(true), Value.bool(null))); - spannerRowData.put("bit_to_string", List.of(Value.string("7fff"), Value.string(null))); + spannerRowData.put("bit_to_string", List.of(Value.string("32767"), Value.string(null))); spannerRowData.put( "bit_to_int64", List.of(Value.int64(9223372036854775807L), Value.int64(null))); spannerRowData.put( @@ -613,23 +604,16 @@ private Map>> getExpectedData() { "bigint_unsigned", createRows("bigint_unsigned", "42", "0", "18446744073709551615", null)); expectedData.put( "binary", createRows("binary", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "binary_to_string", - // createRows( - // "binary_to_string", - // "7835383030000000000000000000000000000000", - // "ff".repeat(255), - // null)); + expectedData.put( + "binary_to_string", + createRows("binary_to_string", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); expectedData.put("bit", createRows("bit", "f/////////8=", null)); expectedData.put("bit_to_bool", createRows("bit_to_bool", false, true, null)); - // Fails to migrate, ignored to avoid failing the test. - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", null)); + expectedData.put("bit_to_string", createRows("bit_to_string", "f/8=", null)); expectedData.put("bit_to_int64", createRows("bit_to_int64", "f/////////8=", null)); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "blob_to_string", createRows("blob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "blob_to_string", createRows("blob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("bool", createRows("bool", false, true, null)); expectedData.put("bool_to_string", createRows("bool_to_string", false, true, null)); expectedData.put("boolean", createRows("boolean", false, true, null)); @@ -735,17 +719,14 @@ private Map>> getExpectedData() { expectedData.put("integer_unsigned", createRows("integer_unsigned", 0, 42, 4294967295L, null)); expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", null)); expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", null)); - expectedData.put("longblob", createRows("longblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "longblob_to_string", - // createRows("longblob_to_string", "7835383030", "ff".repeat(65535), null)); + expectedData.put( + "longblob_to_string", + createRows("longblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("longtext", createRows("longtext", "longtext", "a".repeat(65535), null)); expectedData.put("mediumblob", createRows("mediumblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "mediumblob_to_string", - // createRows("mediumblob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "mediumblob_to_string", + createRows("mediumblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("mediumint", createRows("mediumint", 20, null)); expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", null)); expectedData.put("mediumint_unsigned", createRows("mediumint_unsigned", 42, 0, 16777215, null)); @@ -800,10 +781,8 @@ private Map>> getExpectedData() { "2038-01-19 03:14:07.0", null)); expectedData.put("tinyblob", createRows("tinyblob", "eDU4MDA=", "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "tinyblob_to_string", - // createRows("tinyblob_to_string", "7835383030", "ff".repeat(255), null)); + expectedData.put( + "tinyblob_to_string", createRows("tinyblob_to_string", "eDU4MDA=", "/".repeat(340), null)); expectedData.put("tinyint", createRows("tinyint", 10, 127, -128, null)); expectedData.put( "tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", null)); @@ -811,10 +790,9 @@ private Map>> getExpectedData() { expectedData.put("tinytext", createRows("tinytext", "tinytext", "a".repeat(255), null)); expectedData.put( "varbinary", createRows("varbinary", "eDU4MDA=", "/".repeat(86666) + "8=", null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "varbinary_to_string", - // createRows("varbinary_to_string", "7835383030", "ff".repeat(65000), null)); + expectedData.put( + "varbinary_to_string", + createRows("varbinary_to_string", "eDU4MDA=", "/".repeat(86666) + "8=", null)); expectedData.put("varchar", createRows("varchar", "abc", "a".repeat(21000), null)); // Year gets read out of the DB as a java.sql.Date, so it includes the month/day (both defaulted // to 1); the actual data in the DB is just the year diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java index c28cb69af1..77ca5bb792 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java @@ -182,16 +182,7 @@ private void writeRowsInSpanner(Map> spannerTableData) { private ConditionCheck buildConditionCheck(Map> spannerTableData) { // These tables fail to migrate all expected rows, ignore them to avoid having to wait for the // timeout. - Set ignoredTables = - Set.of( - "binary_to_string", - "bit_to_string", - "set_to_array", - "blob_to_string", - "largeblob_to_string", - "mediumblob_to_string", - "tinyblob_to_string", - "varbinary_to_string"); + Set ignoredTables = Set.of("set_to_array"); ConditionCheck combinedCondition = null; for (Map.Entry> entry : spannerTableData.entrySet()) { @@ -308,7 +299,7 @@ private Map> getSpannerTableData() { "bit", List.of(Value.bytesFromBase64("f/////////8="), Value.bytesFromBase64(null))); spannerRowData.put( "bit_to_bool", List.of(Value.bool(false), Value.bool(true), Value.bool(null))); - spannerRowData.put("bit_to_string", List.of(Value.string("7fff"), Value.string(null))); + spannerRowData.put("bit_to_string", List.of(Value.string("32767"), Value.string(null))); spannerRowData.put( "bit_to_int64", List.of(Value.int64(9223372036854775807L), Value.int64(null))); spannerRowData.put( @@ -619,23 +610,16 @@ private Map>> getExpectedData() { "bigint_unsigned", createRows("bigint_unsigned", "42", "0", "18446744073709551615", null)); expectedData.put( "binary", createRows("binary", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "binary_to_string", - // createRows( - // "binary_to_string", - // "7835383030000000000000000000000000000000", - // "ff".repeat(255), - // null)); + expectedData.put( + "binary_to_string", + createRows("binary_to_string", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); expectedData.put("bit", createRows("bit", "f/////////8=", null)); expectedData.put("bit_to_bool", createRows("bit_to_bool", false, true, null)); - // Fails to migrate, ignored to avoid failing the test. - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", null)); + expectedData.put("bit_to_string", createRows("bit_to_string", "f/8=", null)); expectedData.put("bit_to_int64", createRows("bit_to_int64", "f/////////8=", null)); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "blob_to_string", createRows("blob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "blob_to_string", createRows("blob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("bool", createRows("bool", false, true, null)); expectedData.put("bool_to_string", createRows("bool_to_string", false, true, null)); expectedData.put("boolean", createRows("boolean", false, true, null)); @@ -741,17 +725,14 @@ private Map>> getExpectedData() { expectedData.put("integer_unsigned", createRows("integer_unsigned", 0, 42, 4294967295L, null)); expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", null)); expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", null)); - expectedData.put("longblob", createRows("longblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "longblob_to_string", - // createRows("longblob_to_string", "7835383030", "ff".repeat(65535), null)); + expectedData.put( + "longblob_to_string", + createRows("longblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("longtext", createRows("longtext", "longtext", "a".repeat(65535), null)); expectedData.put("mediumblob", createRows("mediumblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "mediumblob_to_string", - // createRows("mediumblob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "mediumblob_to_string", + createRows("mediumblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("mediumint", createRows("mediumint", 20, null)); expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", null)); expectedData.put("mediumint_unsigned", createRows("mediumint_unsigned", 42, 0, 16777215, null)); @@ -806,10 +787,8 @@ private Map>> getExpectedData() { "2038-01-19 03:14:07.0", null)); expectedData.put("tinyblob", createRows("tinyblob", "eDU4MDA=", "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "tinyblob_to_string", - // createRows("tinyblob_to_string", "7835383030", "ff".repeat(255), null)); + expectedData.put( + "tinyblob_to_string", createRows("tinyblob_to_string", "eDU4MDA=", "/".repeat(340), null)); expectedData.put("tinyint", createRows("tinyint", 10, 127, -128, null)); expectedData.put( "tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", null)); @@ -817,10 +796,9 @@ private Map>> getExpectedData() { expectedData.put("tinytext", createRows("tinytext", "tinytext", "a".repeat(255), null)); expectedData.put( "varbinary", createRows("varbinary", "eDU4MDA=", "/".repeat(86666) + "8=", null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "varbinary_to_string", - // createRows("varbinary_to_string", "7835383030", "ff".repeat(65000), null)); + expectedData.put( + "varbinary_to_string", + createRows("varbinary_to_string", "eDU4MDA=", "/".repeat(86666) + "8=", null)); expectedData.put("varchar", createRows("varchar", "abc", "a".repeat(21000), null)); // Year gets read out of the DB as a java.sql.Date, so it includes the month/day (both defaulted // to 1); the actual data in the DB is just the year diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java index 16ed66b1e3..850a27150e 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java @@ -1650,4 +1650,54 @@ public void testGetDMLStatement_NullPrimaryKeys() { assertThrows(InvalidDMLGenerationException.class, () -> generator.getDMLStatement(request)); } + + @Test + public void testBinaryAndBlobFromStringDML() throws Exception { + Column spannerStringCol = Mockito.mock(Column.class); + Mockito.when(spannerStringCol.name()).thenReturn("binary_col"); + Mockito.when(spannerStringCol.type()).thenReturn(Type.string()); + + SourceColumn sourceBinaryCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBinaryCol.name()).thenReturn("binary_col"); + Mockito.when(sourceBinaryCol.type()).thenReturn("binary"); + + JSONObject json = new JSONObject("{\"binary_col\":\"7835383030\"}"); + + String binaryRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerStringCol, sourceBinaryCol, json, "+00:00", null); + assertEquals("UNHEX('7835383030')", binaryRes); + + SourceColumn sourceBlobCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBlobCol.name()).thenReturn("blob_col"); + Mockito.when(sourceBlobCol.type()).thenReturn("blob"); + + Column spannerBlobStringCol = Mockito.mock(Column.class); + Mockito.when(spannerBlobStringCol.name()).thenReturn("blob_col"); + Mockito.when(spannerBlobStringCol.type()).thenReturn(Type.string()); + + JSONObject jsonBlob = new JSONObject("{\"blob_col\":\"7835383030\"}"); + String blobRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerBlobStringCol, sourceBlobCol, jsonBlob, "+00:00", null); + assertEquals("UNHEX('7835383030')", blobRes); + } + + @Test + public void testBitFromStringDML() throws Exception { + Column spannerStringCol = Mockito.mock(Column.class); + Mockito.when(spannerStringCol.name()).thenReturn("bit_col"); + Mockito.when(spannerStringCol.type()).thenReturn(Type.string()); + + SourceColumn sourceBitCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBitCol.name()).thenReturn("bit_col"); + Mockito.when(sourceBitCol.type()).thenReturn("bit"); + + JSONObject json = new JSONObject("{\"bit_col\":\"32767\"}"); + + String bitRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerStringCol, sourceBitCol, json, "+00:00", null); + assertEquals("32767", bitRes); + } } From 173b05e06d229706bdffd7621060641c3c8a5ae5 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 04:58:56 +0000 Subject: [PATCH 2/7] fix: update bit_varying_to_string expectation in postgres data types IT --- .../cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java | 2 +- .../teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java index 5012ebfdf8..45548a1211 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java @@ -185,7 +185,7 @@ private Map>> getExpectedData() { "bit_to_string", createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("1100", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java index 565c479eeb..518635543a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java @@ -192,7 +192,7 @@ private Map>> getExpectedData() { "bit_to_string", createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("1100", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); From fa52d321b3bd9ff83544e74ad09d443a766fc125 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 05:38:48 +0000 Subject: [PATCH 3/7] feat: use hex encoding for binary to string conversions in AvroToValueMapper --- .../v2/spanner/migrations/avro/AvroToValueMapper.java | 4 ++-- .../v2/spanner/migrations/avro/AvroToValueMapperTest.java | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java index e4ff71c846..59cd1a2ed6 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java @@ -335,10 +335,10 @@ static String avroFieldToString(Object recordValue, Schema fieldSchema) { ByteBuffer byteBuffer = ((ByteBuffer) recordValue).duplicate(); byte[] bytes = new byte[byteBuffer.remaining()]; byteBuffer.get(bytes); - return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); + return Hex.encodeHexString(bytes); } if (recordValue instanceof byte[]) { - return new String((byte[]) recordValue, java.nio.charset.StandardCharsets.UTF_8); + return Hex.encodeHexString((byte[]) recordValue); } return recordValue.toString(); } catch (Exception e) { diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java index 1402886095..526ac28f5a 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java @@ -290,6 +290,14 @@ public void testAvroFieldToString_valid() { result = AvroToValueMapper.avroFieldToString(325.532, SchemaBuilder.builder().doubleType()); assertEquals("325.532", result); + + byte[] byteArray = new byte[] {0x68, 0x65, 0x6c, 0x6c, 0x6f}; + result = AvroToValueMapper.avroFieldToString(byteArray, SchemaBuilder.builder().bytesType()); + assertEquals("68656c6c6f", result); + + ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray); + result = AvroToValueMapper.avroFieldToString(byteBuffer, SchemaBuilder.builder().bytesType()); + assertEquals("68656c6c6f", result); } @Test From 1b1d217f213b688d8184debd80894bd23354eec0 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 05:39:18 +0000 Subject: [PATCH 4/7] test: update postgres bit_to_string and bit_varying_to_string IT expectations for hex encoding --- .../cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java | 7 +++++-- .../v2/templates/PostgreSQLDataTypesPGDialectIT.java | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java index 45548a1211..d8715da8ea 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java @@ -183,9 +183,12 @@ private Map>> getExpectedData() { createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); result.put( "bit_to_string", - createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); + createRows( + "30".repeat(32), + "30".repeat(31) + "31", + "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("0101", "NULL")); + result.put("bit_varying_to_string", createRows("30313031", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java index 518635543a..8576b808ae 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java @@ -190,9 +190,12 @@ private Map>> getExpectedData() { createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); result.put( "bit_to_string", - createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); + createRows( + "30".repeat(32), + "30".repeat(31) + "31", + "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("0101", "NULL")); + result.put("bit_varying_to_string", createRows("30313031", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); From 67135d3cf88f4e161a084e9973ca578f15250c83 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 06:05:35 +0000 Subject: [PATCH 5/7] test: restore bit_to_string and bit_varying_to_string literal bit string expectations --- .../cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java | 7 ++----- .../v2/templates/PostgreSQLDataTypesPGDialectIT.java | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java index d8715da8ea..45548a1211 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java @@ -183,12 +183,9 @@ private Map>> getExpectedData() { createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); result.put( "bit_to_string", - createRows( - "30".repeat(32), - "30".repeat(31) + "31", - "NULL")); + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("30313031", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java index 8576b808ae..518635543a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java @@ -190,12 +190,9 @@ private Map>> getExpectedData() { createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); result.put( "bit_to_string", - createRows( - "30".repeat(32), - "30".repeat(31) + "31", - "NULL")); + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - result.put("bit_varying_to_string", createRows("30313031", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); From e51cecbefad2bb063ac8bc03ab9fd76a20d2eea0 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 07:25:41 +0000 Subject: [PATCH 6/7] fix: support hex-encoded strings mapping to bytea in PostgreSQL reverse migration --- .../source/postgres/PostgreSQLDMLGenerator.java | 12 +++++++++++- .../postgres/PostgreSQLDMLGeneratorTest.java | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java index dbc95192d5..40aebe51e3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java @@ -299,7 +299,11 @@ private static String getColumnValueByType( case "bytea": case "binary": case "varbinary": - response = colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() + if (isStringType(spannerColType)) { + response = "decode(" + getQuotedEscapedString(colValue, spannerColType) + ", 'hex')"; + } else { + response = colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() + } break; default: response = colValue; @@ -317,6 +321,12 @@ private static String escapeString(String input) { return cleanedNullBytes; } + private static boolean isStringType(String spannerColType) { + return "STRING".equalsIgnoreCase(spannerColType) + || "PG_VARCHAR".equalsIgnoreCase(spannerColType) + || "PG_TEXT".equalsIgnoreCase(spannerColType); + } + static String getQuotedEscapedString(String input, String spannerColType) { if ("BYTES".equals(spannerColType) || "PG_BYTEA".equals(spannerColType)) { return input; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java index 10f09b8e7b..48c3633303 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java @@ -465,6 +465,23 @@ public void testByteaTypeDML() throws Exception { assertEquals("'\\x48656c6c6f'", res); // "Hello" in hex is 48656c6c6f } + @Test + public void testByteaTypeDMLFromSpannerString() throws Exception { + Column spannerCol = mock(Column.class); + when(spannerCol.name()).thenReturn("bytea_column"); + when(spannerCol.type()).thenReturn(Type.string()); // Overridden mapping in Spanner + + SourceColumn sourceCol = mock(SourceColumn.class); + when(sourceCol.name()).thenReturn("bytea_column"); + when(sourceCol.type()).thenReturn("bytea"); + + JSONObject json = new JSONObject("{\"bytea_column\":\"48656c6c6f\"}"); // "Hello" in hex, passed as string + + String res = + PostgreSQLDMLGenerator.getMappedColumnValue(spannerCol, sourceCol, json, "+00:00", null); + assertEquals("decode('48656c6c6f', 'hex')", res); // Wrapped in decode function + } + @Test public void testUuidTypeDML() throws Exception { Column spannerCol = mock(Column.class); From 251580386320e13ef6cb953d4977ea981c9309e6 Mon Sep 17 00:00:00 2001 From: Sandeep Mishra Date: Wed, 26 Aug 2026 08:11:51 +0000 Subject: [PATCH 7/7] spotless check --- .../v2/templates/source/postgres/PostgreSQLDMLGenerator.java | 3 ++- .../templates/source/postgres/PostgreSQLDMLGeneratorTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java index 40aebe51e3..40bf5cd791 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java @@ -302,7 +302,8 @@ private static String getColumnValueByType( if (isStringType(spannerColType)) { response = "decode(" + getQuotedEscapedString(colValue, spannerColType) + ", 'hex')"; } else { - response = colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() + response = + colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() } break; default: diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java index 48c3633303..74e60a4e54 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java @@ -475,7 +475,8 @@ public void testByteaTypeDMLFromSpannerString() throws Exception { when(sourceCol.name()).thenReturn("bytea_column"); when(sourceCol.type()).thenReturn("bytea"); - JSONObject json = new JSONObject("{\"bytea_column\":\"48656c6c6f\"}"); // "Hello" in hex, passed as string + JSONObject json = + new JSONObject("{\"bytea_column\":\"48656c6c6f\"}"); // "Hello" in hex, passed as string String res = PostgreSQLDMLGenerator.getMappedColumnValue(spannerCol, sourceCol, json, "+00:00", null);