diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java index 29736e5895..12e6fdcd48 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java @@ -268,7 +268,7 @@ private void validateResult( // These types are not mapped as expected, ignore them to avoid failing the // test. Set ignoredTypeMappings = - Set.of("bit_to_string", "date_to_string", "set_to_array", "spatial_geometrycollection"); + Set.of("date_to_string", "set_to_array", "spatial_geometrycollection"); // Validate supported data types. for (Map.Entry>> entry : expectedData.entrySet()) { String type = entry.getKey(); @@ -409,7 +409,7 @@ private Map>> getExpectedData() { "NULL")); expectedData.put("bit", createRows("bit", "f/////////8=", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java index 3367a9b831..cb5eb26c0a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java @@ -179,9 +179,7 @@ private Map>> getExpectedData() { expectedData.put("bit8", createRows("bit8", "0", "255", "NULL")); expectedData.put("bit1", createRows("bit1", "false", "true", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the long - // representation of the bits which is unexpected even if it's not necessarily incorrect - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java index 85f2c694e3..548cf1c37b 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java @@ -185,9 +185,7 @@ private Map>> getExpectedData() { expectedData.put("bit8", createRows("bit8", "0", "255", "NULL")); expectedData.put("bit1", createRows("bit1", "false", "true", "NULL")); expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the long - // representation of the bits which is unexpected even if it's not necessarily incorrect - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL")); + expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL")); expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL")); expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL")); expectedData.put( diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java index b71be58f2a..45548a1211 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java @@ -181,13 +181,11 @@ private Map>> getExpectedData() { result.put( "bit", createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the literal - // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]" - // result.put("bit_to_string", createRows("0", "1", "NULL")); + result.put( + "bit_to_string", + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - // bit_varying_to_string is commented out to avoid failing the test case; returned data is the - // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" - // result.put("bit_varying_to_string", createRows("5", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java index da149b5c94..518635543a 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java @@ -188,13 +188,11 @@ private Map>> getExpectedData() { result.put( "bit", createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL")); - // bit_to_string is commented out to avoid failing the test case; returned data is the literal - // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]" - // result.put("bit_to_string", createRows("0", "1", "NULL")); + result.put( + "bit_to_string", + createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL")); result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL")); - // bit_varying_to_string is commented out to avoid failing the test case; returned data is the - // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" - // result.put("bit_varying_to_string", createRows("5", "NULL")); + result.put("bit_varying_to_string", createRows("0101", "NULL")); result.put("bool", createRows("false", "true", "NULL")); result.put("bool_to_string", createRows("false", "true", "NULL")); result.put("boolean", createRows("false", "true", "NULL")); diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java index 28c1ce597c..59cd1a2ed6 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java @@ -331,6 +331,15 @@ static String avroFieldToString(Object recordValue, Schema fieldSchema) { if (recordValue == null) { return null; } + if (recordValue instanceof ByteBuffer) { + ByteBuffer byteBuffer = ((ByteBuffer) recordValue).duplicate(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + return Hex.encodeHexString(bytes); + } + if (recordValue instanceof byte[]) { + return Hex.encodeHexString((byte[]) recordValue); + } return recordValue.toString(); } catch (Exception e) { throw new AvroTypeConvertorException( diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java index 1402886095..526ac28f5a 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java @@ -290,6 +290,14 @@ public void testAvroFieldToString_valid() { result = AvroToValueMapper.avroFieldToString(325.532, SchemaBuilder.builder().doubleType()); assertEquals("325.532", result); + + byte[] byteArray = new byte[] {0x68, 0x65, 0x6c, 0x6c, 0x6f}; + result = AvroToValueMapper.avroFieldToString(byteArray, SchemaBuilder.builder().bytesType()); + assertEquals("68656c6c6f", result); + + ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray); + result = AvroToValueMapper.avroFieldToString(byteBuffer, SchemaBuilder.builder().bytesType()); + assertEquals("68656c6c6f", result); } @Test diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java index 9f8fec4891..75d49ba1e8 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java @@ -293,11 +293,17 @@ static String getColumnValueByType( case "multilinestring": case "polygon": case "multipolygon": + response = getQuotedEscapedString(colValue, spannerColType); + break; case "tinyblob": case "mediumblob": case "blob": case "longblob": - response = getQuotedEscapedString(colValue, spannerColType); + if (isStringType(spannerColType)) { + response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")"; + } else { + response = getQuotedEscapedString(colValue, spannerColType); + } break; case "timestamp": case "datetime": @@ -312,7 +318,11 @@ static String getColumnValueByType( break; case "binary": case "varbinary": - response = getBinaryString(colValue, spannerColType); + if (isStringType(spannerColType)) { + response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")"; + } else { + response = getBinaryString(colValue, spannerColType); + } break; default: response = colValue; @@ -320,6 +330,12 @@ static String getColumnValueByType( return response; } + private static boolean isStringType(String spannerColType) { + return "STRING".equalsIgnoreCase(spannerColType) + || "PG_VARCHAR".equalsIgnoreCase(spannerColType) + || "PG_TEXT".equalsIgnoreCase(spannerColType); + } + private static String escapeString(String input) { String cleanedNullBytes = StringUtils.replace(input, "\u0000", ""); cleanedNullBytes = StringUtils.replace(cleanedNullBytes, "'", "''"); diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java index dbc95192d5..40bf5cd791 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java @@ -299,7 +299,12 @@ private static String getColumnValueByType( case "bytea": case "binary": case "varbinary": - response = colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() + if (isStringType(spannerColType)) { + response = "decode(" + getQuotedEscapedString(colValue, spannerColType) + ", 'hex')"; + } else { + response = + colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex() + } break; default: response = colValue; @@ -317,6 +322,12 @@ private static String escapeString(String input) { return cleanedNullBytes; } + private static boolean isStringType(String spannerColType) { + return "STRING".equalsIgnoreCase(spannerColType) + || "PG_VARCHAR".equalsIgnoreCase(spannerColType) + || "PG_TEXT".equalsIgnoreCase(spannerColType); + } + static String getQuotedEscapedString(String input, String spannerColType) { if ("BYTES".equals(spannerColType) || "PG_BYTEA".equals(spannerColType)) { return input; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java index c2249017e5..4ed8b10d93 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java @@ -176,16 +176,7 @@ private void writeRowsInSpanner(Map> spannerTableData) { private ConditionCheck buildConditionCheck(Map> spannerTableData) { // These tables fail to migrate all expected rows, ignore them to avoid having to wait for the // timeout. - Set ignoredTables = - Set.of( - "binary_to_string", - "bit_to_string", - "set_to_array", - "blob_to_string", - "longblob_to_string", - "mediumblob_to_string", - "tinyblob_to_string", - "varbinary_to_string"); + Set ignoredTables = Set.of("set_to_array"); ConditionCheck combinedCondition = null; for (Map.Entry> entry : spannerTableData.entrySet()) { @@ -302,7 +293,7 @@ private Map> getSpannerTableData() { "bit", List.of(Value.bytesFromBase64("f/////////8="), Value.bytesFromBase64(null))); spannerRowData.put( "bit_to_bool", List.of(Value.bool(false), Value.bool(true), Value.bool(null))); - spannerRowData.put("bit_to_string", List.of(Value.string("7fff"), Value.string(null))); + spannerRowData.put("bit_to_string", List.of(Value.string("32767"), Value.string(null))); spannerRowData.put( "bit_to_int64", List.of(Value.int64(9223372036854775807L), Value.int64(null))); spannerRowData.put( @@ -613,23 +604,16 @@ private Map>> getExpectedData() { "bigint_unsigned", createRows("bigint_unsigned", "42", "0", "18446744073709551615", null)); expectedData.put( "binary", createRows("binary", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "binary_to_string", - // createRows( - // "binary_to_string", - // "7835383030000000000000000000000000000000", - // "ff".repeat(255), - // null)); + expectedData.put( + "binary_to_string", + createRows("binary_to_string", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); expectedData.put("bit", createRows("bit", "f/////////8=", null)); expectedData.put("bit_to_bool", createRows("bit_to_bool", false, true, null)); - // Fails to migrate, ignored to avoid failing the test. - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", null)); + expectedData.put("bit_to_string", createRows("bit_to_string", "f/8=", null)); expectedData.put("bit_to_int64", createRows("bit_to_int64", "f/////////8=", null)); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "blob_to_string", createRows("blob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "blob_to_string", createRows("blob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("bool", createRows("bool", false, true, null)); expectedData.put("bool_to_string", createRows("bool_to_string", false, true, null)); expectedData.put("boolean", createRows("boolean", false, true, null)); @@ -735,17 +719,14 @@ private Map>> getExpectedData() { expectedData.put("integer_unsigned", createRows("integer_unsigned", 0, 42, 4294967295L, null)); expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", null)); expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", null)); - expectedData.put("longblob", createRows("longblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "longblob_to_string", - // createRows("longblob_to_string", "7835383030", "ff".repeat(65535), null)); + expectedData.put( + "longblob_to_string", + createRows("longblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("longtext", createRows("longtext", "longtext", "a".repeat(65535), null)); expectedData.put("mediumblob", createRows("mediumblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "mediumblob_to_string", - // createRows("mediumblob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "mediumblob_to_string", + createRows("mediumblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("mediumint", createRows("mediumint", 20, null)); expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", null)); expectedData.put("mediumint_unsigned", createRows("mediumint_unsigned", 42, 0, 16777215, null)); @@ -800,10 +781,8 @@ private Map>> getExpectedData() { "2038-01-19 03:14:07.0", null)); expectedData.put("tinyblob", createRows("tinyblob", "eDU4MDA=", "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "tinyblob_to_string", - // createRows("tinyblob_to_string", "7835383030", "ff".repeat(255), null)); + expectedData.put( + "tinyblob_to_string", createRows("tinyblob_to_string", "eDU4MDA=", "/".repeat(340), null)); expectedData.put("tinyint", createRows("tinyint", 10, 127, -128, null)); expectedData.put( "tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", null)); @@ -811,10 +790,9 @@ private Map>> getExpectedData() { expectedData.put("tinytext", createRows("tinytext", "tinytext", "a".repeat(255), null)); expectedData.put( "varbinary", createRows("varbinary", "eDU4MDA=", "/".repeat(86666) + "8=", null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "varbinary_to_string", - // createRows("varbinary_to_string", "7835383030", "ff".repeat(65000), null)); + expectedData.put( + "varbinary_to_string", + createRows("varbinary_to_string", "eDU4MDA=", "/".repeat(86666) + "8=", null)); expectedData.put("varchar", createRows("varchar", "abc", "a".repeat(21000), null)); // Year gets read out of the DB as a java.sql.Date, so it includes the month/day (both defaulted // to 1); the actual data in the DB is just the year diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java index c28cb69af1..77ca5bb792 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesPGDialectIT.java @@ -182,16 +182,7 @@ private void writeRowsInSpanner(Map> spannerTableData) { private ConditionCheck buildConditionCheck(Map> spannerTableData) { // These tables fail to migrate all expected rows, ignore them to avoid having to wait for the // timeout. - Set ignoredTables = - Set.of( - "binary_to_string", - "bit_to_string", - "set_to_array", - "blob_to_string", - "largeblob_to_string", - "mediumblob_to_string", - "tinyblob_to_string", - "varbinary_to_string"); + Set ignoredTables = Set.of("set_to_array"); ConditionCheck combinedCondition = null; for (Map.Entry> entry : spannerTableData.entrySet()) { @@ -308,7 +299,7 @@ private Map> getSpannerTableData() { "bit", List.of(Value.bytesFromBase64("f/////////8="), Value.bytesFromBase64(null))); spannerRowData.put( "bit_to_bool", List.of(Value.bool(false), Value.bool(true), Value.bool(null))); - spannerRowData.put("bit_to_string", List.of(Value.string("7fff"), Value.string(null))); + spannerRowData.put("bit_to_string", List.of(Value.string("32767"), Value.string(null))); spannerRowData.put( "bit_to_int64", List.of(Value.int64(9223372036854775807L), Value.int64(null))); spannerRowData.put( @@ -619,23 +610,16 @@ private Map>> getExpectedData() { "bigint_unsigned", createRows("bigint_unsigned", "42", "0", "18446744073709551615", null)); expectedData.put( "binary", createRows("binary", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "binary_to_string", - // createRows( - // "binary_to_string", - // "7835383030000000000000000000000000000000", - // "ff".repeat(255), - // null)); + expectedData.put( + "binary_to_string", + createRows("binary_to_string", "eDU4MD" + "A".repeat(334), "/".repeat(340), null)); expectedData.put("bit", createRows("bit", "f/////////8=", null)); expectedData.put("bit_to_bool", createRows("bit_to_bool", false, true, null)); - // Fails to migrate, ignored to avoid failing the test. - // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", null)); + expectedData.put("bit_to_string", createRows("bit_to_string", "f/8=", null)); expectedData.put("bit_to_int64", createRows("bit_to_int64", "f/////////8=", null)); expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "blob_to_string", createRows("blob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "blob_to_string", createRows("blob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("bool", createRows("bool", false, true, null)); expectedData.put("bool_to_string", createRows("bool_to_string", false, true, null)); expectedData.put("boolean", createRows("boolean", false, true, null)); @@ -741,17 +725,14 @@ private Map>> getExpectedData() { expectedData.put("integer_unsigned", createRows("integer_unsigned", 0, 42, 4294967295L, null)); expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", null)); expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", null)); - expectedData.put("longblob", createRows("longblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "longblob_to_string", - // createRows("longblob_to_string", "7835383030", "ff".repeat(65535), null)); + expectedData.put( + "longblob_to_string", + createRows("longblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("longtext", createRows("longtext", "longtext", "a".repeat(65535), null)); expectedData.put("mediumblob", createRows("mediumblob", "eDU4MDA=", "/".repeat(87380), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "mediumblob_to_string", - // createRows("mediumblob_to_string", "7835383030", "FF".repeat(65535), null)); + expectedData.put( + "mediumblob_to_string", + createRows("mediumblob_to_string", "eDU4MDA=", "/".repeat(87380), null)); expectedData.put("mediumint", createRows("mediumint", 20, null)); expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", null)); expectedData.put("mediumint_unsigned", createRows("mediumint_unsigned", 42, 0, 16777215, null)); @@ -806,10 +787,8 @@ private Map>> getExpectedData() { "2038-01-19 03:14:07.0", null)); expectedData.put("tinyblob", createRows("tinyblob", "eDU4MDA=", "/".repeat(340), null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "tinyblob_to_string", - // createRows("tinyblob_to_string", "7835383030", "ff".repeat(255), null)); + expectedData.put( + "tinyblob_to_string", createRows("tinyblob_to_string", "eDU4MDA=", "/".repeat(340), null)); expectedData.put("tinyint", createRows("tinyint", 10, 127, -128, null)); expectedData.put( "tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", null)); @@ -817,10 +796,9 @@ private Map>> getExpectedData() { expectedData.put("tinytext", createRows("tinytext", "tinytext", "a".repeat(255), null)); expectedData.put( "varbinary", createRows("varbinary", "eDU4MDA=", "/".repeat(86666) + "8=", null)); - // Not mapped as expected, ignored to avoid failing the test. - // expectedData.put( - // "varbinary_to_string", - // createRows("varbinary_to_string", "7835383030", "ff".repeat(65000), null)); + expectedData.put( + "varbinary_to_string", + createRows("varbinary_to_string", "eDU4MDA=", "/".repeat(86666) + "8=", null)); expectedData.put("varchar", createRows("varchar", "abc", "a".repeat(21000), null)); // Year gets read out of the DB as a java.sql.Date, so it includes the month/day (both defaulted // to 1); the actual data in the DB is just the year diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java index 16ed66b1e3..850a27150e 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGeneratorTest.java @@ -1650,4 +1650,54 @@ public void testGetDMLStatement_NullPrimaryKeys() { assertThrows(InvalidDMLGenerationException.class, () -> generator.getDMLStatement(request)); } + + @Test + public void testBinaryAndBlobFromStringDML() throws Exception { + Column spannerStringCol = Mockito.mock(Column.class); + Mockito.when(spannerStringCol.name()).thenReturn("binary_col"); + Mockito.when(spannerStringCol.type()).thenReturn(Type.string()); + + SourceColumn sourceBinaryCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBinaryCol.name()).thenReturn("binary_col"); + Mockito.when(sourceBinaryCol.type()).thenReturn("binary"); + + JSONObject json = new JSONObject("{\"binary_col\":\"7835383030\"}"); + + String binaryRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerStringCol, sourceBinaryCol, json, "+00:00", null); + assertEquals("UNHEX('7835383030')", binaryRes); + + SourceColumn sourceBlobCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBlobCol.name()).thenReturn("blob_col"); + Mockito.when(sourceBlobCol.type()).thenReturn("blob"); + + Column spannerBlobStringCol = Mockito.mock(Column.class); + Mockito.when(spannerBlobStringCol.name()).thenReturn("blob_col"); + Mockito.when(spannerBlobStringCol.type()).thenReturn(Type.string()); + + JSONObject jsonBlob = new JSONObject("{\"blob_col\":\"7835383030\"}"); + String blobRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerBlobStringCol, sourceBlobCol, jsonBlob, "+00:00", null); + assertEquals("UNHEX('7835383030')", blobRes); + } + + @Test + public void testBitFromStringDML() throws Exception { + Column spannerStringCol = Mockito.mock(Column.class); + Mockito.when(spannerStringCol.name()).thenReturn("bit_col"); + Mockito.when(spannerStringCol.type()).thenReturn(Type.string()); + + SourceColumn sourceBitCol = Mockito.mock(SourceColumn.class); + Mockito.when(sourceBitCol.name()).thenReturn("bit_col"); + Mockito.when(sourceBitCol.type()).thenReturn("bit"); + + JSONObject json = new JSONObject("{\"bit_col\":\"32767\"}"); + + String bitRes = + MySQLDMLGenerator.getMappedColumnValue( + spannerStringCol, sourceBitCol, json, "+00:00", null); + assertEquals("32767", bitRes); + } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java index 10f09b8e7b..74e60a4e54 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGeneratorTest.java @@ -465,6 +465,24 @@ public void testByteaTypeDML() throws Exception { assertEquals("'\\x48656c6c6f'", res); // "Hello" in hex is 48656c6c6f } + @Test + public void testByteaTypeDMLFromSpannerString() throws Exception { + Column spannerCol = mock(Column.class); + when(spannerCol.name()).thenReturn("bytea_column"); + when(spannerCol.type()).thenReturn(Type.string()); // Overridden mapping in Spanner + + SourceColumn sourceCol = mock(SourceColumn.class); + when(sourceCol.name()).thenReturn("bytea_column"); + when(sourceCol.type()).thenReturn("bytea"); + + JSONObject json = + new JSONObject("{\"bytea_column\":\"48656c6c6f\"}"); // "Hello" in hex, passed as string + + String res = + PostgreSQLDMLGenerator.getMappedColumnValue(spannerCol, sourceCol, json, "+00:00", null); + assertEquals("decode('48656c6c6f', 'hex')", res); // Wrapped in decode function + } + @Test public void testUuidTypeDML() throws Exception { Column spannerCol = mock(Column.class);