Binary to string mappings

v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java

@@ -268,7 +268,7 @@ private void validateResult(
     // These types are not mapped as expected, ignore them to avoid failing the
     // test.
     Set<String> ignoredTypeMappings =
-        Set.of("bit_to_string", "date_to_string", "set_to_array", "spatial_geometrycollection");
+        Set.of("date_to_string", "set_to_array", "spatial_geometrycollection");
     // Validate supported data types.
     for (Map.Entry<String, List<Map<String, Object>>> entry : expectedData.entrySet()) {
       String type = entry.getKey();
@@ -409,7 +409,7 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
             "NULL"));
     expectedData.put("bit", createRows("bit", "f/////////8=", "NULL"));
     expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL"));
-    expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL"));
+    expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL"));
     expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL"));
     expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), "NULL"));
     expectedData.put(

v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java

@@ -179,9 +179,7 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
     expectedData.put("bit8", createRows("bit8", "0", "255", "NULL"));
     expectedData.put("bit1", createRows("bit1", "false", "true", "NULL"));
     expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL"));
-    // bit_to_string is commented out to avoid failing the test case; returned data is the long
-    // representation of the bits which is unexpected even if it's not necessarily incorrect
-    // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL"));
+    expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL"));
     expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL"));
     expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL"));
     expectedData.put(

v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java

@@ -185,9 +185,7 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
     expectedData.put("bit8", createRows("bit8", "0", "255", "NULL"));
     expectedData.put("bit1", createRows("bit1", "false", "true", "NULL"));
     expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL"));
-    // bit_to_string is commented out to avoid failing the test case; returned data is the long
-    // representation of the bits which is unexpected even if it's not necessarily incorrect
-    // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", "NULL"));
+    expectedData.put("bit_to_string", createRows("bit_to_string", "32767", "NULL"));
     expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL"));
     expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL"));
     expectedData.put(

v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesIT.java

@@ -181,13 +181,11 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
     result.put(
         "bit",
         createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL"));
-    // bit_to_string is commented out to avoid failing the test case; returned data is the literal
-    // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]"
-    // result.put("bit_to_string", createRows("0", "1", "NULL"));
+    result.put(
+        "bit_to_string",
+        createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL"));
     result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL"));
-    // bit_varying_to_string is commented out to avoid failing the test case; returned data is the
-    // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]"
-    // result.put("bit_varying_to_string", createRows("5", "NULL"));
+    result.put("bit_varying_to_string", createRows("0101", "NULL"));
     result.put("bool", createRows("false", "true", "NULL"));
     result.put("bool_to_string", createRows("false", "true", "NULL"));
     result.put("boolean", createRows("false", "true", "NULL"));

v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDataTypesPGDialectIT.java

@@ -188,13 +188,11 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
     result.put(
         "bit",
         createRows(ByteArray.copyFrom("0").toBase64(), ByteArray.copyFrom("1").toBase64(), "NULL"));
-    // bit_to_string is commented out to avoid failing the test case; returned data is the literal
-    // string "java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]"
-    // result.put("bit_to_string", createRows("0", "1", "NULL"));
+    result.put(
+        "bit_to_string",
+        createRows("00000000000000000000000000000000", "00000000000000000000000000000001", "NULL"));
     result.put("bit_varying", createRows(ByteArray.copyFrom("0101").toBase64(), "NULL"));
-    // bit_varying_to_string is commented out to avoid failing the test case; returned data is the
-    // literal string "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]"
-    // result.put("bit_varying_to_string", createRows("5", "NULL"));
+    result.put("bit_varying_to_string", createRows("0101", "NULL"));
     result.put("bool", createRows("false", "true", "NULL"));
     result.put("bool_to_string", createRows("false", "true", "NULL"));
     result.put("boolean", createRows("false", "true", "NULL"));

v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java

@@ -331,6 +331,15 @@ static String avroFieldToString(Object recordValue, Schema fieldSchema) {
       if (recordValue == null) {
         return null;
       }
+      if (recordValue instanceof ByteBuffer) {
+        ByteBuffer byteBuffer = ((ByteBuffer) recordValue).duplicate();
+        byte[] bytes = new byte[byteBuffer.remaining()];
+        byteBuffer.get(bytes);
+        return Hex.encodeHexString(bytes);
+      }
+      if (recordValue instanceof byte[]) {
+        return Hex.encodeHexString((byte[]) recordValue);
+      }
       return recordValue.toString();
     } catch (Exception e) {
       throw new AvroTypeConvertorException(

v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java

@@ -290,6 +290,14 @@ public void testAvroFieldToString_valid() {
 
     result = AvroToValueMapper.avroFieldToString(325.532, SchemaBuilder.builder().doubleType());
     assertEquals("325.532", result);
+
+    byte[] byteArray = new byte[] {0x68, 0x65, 0x6c, 0x6c, 0x6f};
+    result = AvroToValueMapper.avroFieldToString(byteArray, SchemaBuilder.builder().bytesType());
+    assertEquals("68656c6c6f", result);
+
+    ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray);
+    result = AvroToValueMapper.avroFieldToString(byteBuffer, SchemaBuilder.builder().bytesType());
+    assertEquals("68656c6c6f", result);
   }
 
   @Test

v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/mysql/MySQLDMLGenerator.java

@@ -293,11 +293,17 @@ static String getColumnValueByType(
       case "multilinestring":
       case "polygon":
       case "multipolygon":
+        response = getQuotedEscapedString(colValue, spannerColType);
+        break;
       case "tinyblob":
       case "mediumblob":
       case "blob":
       case "longblob":
-        response = getQuotedEscapedString(colValue, spannerColType);
+        if (isStringType(spannerColType)) {
+          response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")";
+        } else {
+          response = getQuotedEscapedString(colValue, spannerColType);
+        }
         break;
       case "timestamp":
       case "datetime":
@@ -312,14 +318,24 @@ static String getColumnValueByType(
         break;
       case "binary":
       case "varbinary":
-        response = getBinaryString(colValue, spannerColType);
+        if (isStringType(spannerColType)) {
+          response = "UNHEX(" + getQuotedEscapedString(colValue, spannerColType) + ")";
+        } else {
+          response = getBinaryString(colValue, spannerColType);
+        }
         break;
       default:
         response = colValue;
     }
     return response;
   }
 
+  private static boolean isStringType(String spannerColType) {
+    return "STRING".equalsIgnoreCase(spannerColType)
+        || "PG_VARCHAR".equalsIgnoreCase(spannerColType)
+        || "PG_TEXT".equalsIgnoreCase(spannerColType);
+  }
+
   private static String escapeString(String input) {
     String cleanedNullBytes = StringUtils.replace(input, "\u0000", "");
     cleanedNullBytes = StringUtils.replace(cleanedNullBytes, "'", "''");

v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/postgres/PostgreSQLDMLGenerator.java

@@ -299,7 +299,12 @@ private static String getColumnValueByType(
       case "bytea":
       case "binary":
       case "varbinary":
-        response = colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex()
+        if (isStringType(spannerColType)) {
+          response = "decode(" + getQuotedEscapedString(colValue, spannerColType) + ", 'hex')";
+        } else {
+          response =
+              colValue; // Handled in getMappedColumnValue via decode() or convertBase64ToHex()
+        }
         break;
       default:
         response = colValue;
@@ -317,6 +322,12 @@ private static String escapeString(String input) {
     return cleanedNullBytes;
   }
 
+  private static boolean isStringType(String spannerColType) {
+    return "STRING".equalsIgnoreCase(spannerColType)
+        || "PG_VARCHAR".equalsIgnoreCase(spannerColType)
+        || "PG_TEXT".equalsIgnoreCase(spannerColType);
+  }
+
   static String getQuotedEscapedString(String input, String spannerColType) {
     if ("BYTES".equals(spannerColType) || "PG_BYTEA".equals(spannerColType)) {
       return input;

v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlDataTypesIT.java

@@ -176,16 +176,7 @@ private void writeRowsInSpanner(Map<String, List<Value>> spannerTableData) {
   private ConditionCheck buildConditionCheck(Map<String, List<Value>> spannerTableData) {
     // These tables fail to migrate all expected rows, ignore them to avoid having to wait for the
     // timeout.
-    Set<String> ignoredTables =
-        Set.of(
-            "binary_to_string",
-            "bit_to_string",
-            "set_to_array",
-            "blob_to_string",
-            "longblob_to_string",
-            "mediumblob_to_string",
-            "tinyblob_to_string",
-            "varbinary_to_string");
+    Set<String> ignoredTables = Set.of("set_to_array");
 
     ConditionCheck combinedCondition = null;
     for (Map.Entry<String, List<Value>> entry : spannerTableData.entrySet()) {
@@ -302,7 +293,7 @@ private Map<String, List<Value>> getSpannerTableData() {
         "bit", List.of(Value.bytesFromBase64("f/////////8="), Value.bytesFromBase64(null)));
     spannerRowData.put(
         "bit_to_bool", List.of(Value.bool(false), Value.bool(true), Value.bool(null)));
-    spannerRowData.put("bit_to_string", List.of(Value.string("7fff"), Value.string(null)));
+    spannerRowData.put("bit_to_string", List.of(Value.string("32767"), Value.string(null)));
     spannerRowData.put(
         "bit_to_int64", List.of(Value.int64(9223372036854775807L), Value.int64(null)));
     spannerRowData.put(
@@ -613,23 +604,16 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
         "bigint_unsigned", createRows("bigint_unsigned", "42", "0", "18446744073709551615", null));
     expectedData.put(
         "binary", createRows("binary", "eDU4MD" + "A".repeat(334), "/".repeat(340), null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //     "binary_to_string",
-    //     createRows(
-    //         "binary_to_string",
-    //         "7835383030000000000000000000000000000000",
-    //         "ff".repeat(255),
-    //         null));
+    expectedData.put(
+        "binary_to_string",
+        createRows("binary_to_string", "eDU4MD" + "A".repeat(334), "/".repeat(340), null));
     expectedData.put("bit", createRows("bit", "f/////////8=", null));
     expectedData.put("bit_to_bool", createRows("bit_to_bool", false, true, null));
-    // Fails to migrate, ignored to avoid failing the test.
-    // expectedData.put("bit_to_string", createRows("bit_to_string", "7fff", null));
+    expectedData.put("bit_to_string", createRows("bit_to_string", "f/8=", null));
     expectedData.put("bit_to_int64", createRows("bit_to_int64", "f/////////8=", null));
     expectedData.put("blob", createRows("blob", "eDU4MDA=", "/".repeat(87380), null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //     "blob_to_string", createRows("blob_to_string", "7835383030", "FF".repeat(65535), null));
+    expectedData.put(
+        "blob_to_string", createRows("blob_to_string", "eDU4MDA=", "/".repeat(87380), null));
     expectedData.put("bool", createRows("bool", false, true, null));
     expectedData.put("bool_to_string", createRows("bool_to_string", false, true, null));
     expectedData.put("boolean", createRows("boolean", false, true, null));
@@ -735,17 +719,14 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
     expectedData.put("integer_unsigned", createRows("integer_unsigned", 0, 42, 4294967295L, null));
     expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", null));
     expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", null));
-    expectedData.put("longblob", createRows("longblob", "eDU4MDA=", "/".repeat(87380), null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //    "longblob_to_string",
-    //     createRows("longblob_to_string", "7835383030", "ff".repeat(65535), null));
+    expectedData.put(
+        "longblob_to_string",
+        createRows("longblob_to_string", "eDU4MDA=", "/".repeat(87380), null));
     expectedData.put("longtext", createRows("longtext", "longtext", "a".repeat(65535), null));
     expectedData.put("mediumblob", createRows("mediumblob", "eDU4MDA=", "/".repeat(87380), null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //     "mediumblob_to_string",
-    //     createRows("mediumblob_to_string", "7835383030", "FF".repeat(65535), null));
+    expectedData.put(
+        "mediumblob_to_string",
+        createRows("mediumblob_to_string", "eDU4MDA=", "/".repeat(87380), null));
     expectedData.put("mediumint", createRows("mediumint", 20, null));
     expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", null));
     expectedData.put("mediumint_unsigned", createRows("mediumint_unsigned", 42, 0, 16777215, null));
@@ -800,21 +781,18 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
             "2038-01-19 03:14:07.0",
             null));
     expectedData.put("tinyblob", createRows("tinyblob", "eDU4MDA=", "/".repeat(340), null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //     "tinyblob_to_string",
-    //     createRows("tinyblob_to_string", "7835383030", "ff".repeat(255), null));
+    expectedData.put(
+        "tinyblob_to_string", createRows("tinyblob_to_string", "eDU4MDA=", "/".repeat(340), null));
     expectedData.put("tinyint", createRows("tinyint", 10, 127, -128, null));
     expectedData.put(
         "tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", null));
     expectedData.put("tinyint_unsigned", createRows("tinyint_unsigned", 0, 255, null));
     expectedData.put("tinytext", createRows("tinytext", "tinytext", "a".repeat(255), null));
     expectedData.put(
         "varbinary", createRows("varbinary", "eDU4MDA=", "/".repeat(86666) + "8=", null));
-    // Not mapped as expected, ignored to avoid failing the test.
-    // expectedData.put(
-    //     "varbinary_to_string",
-    //     createRows("varbinary_to_string", "7835383030", "ff".repeat(65000), null));
+    expectedData.put(
+        "varbinary_to_string",
+        createRows("varbinary_to_string", "eDU4MDA=", "/".repeat(86666) + "8=", null));
     expectedData.put("varchar", createRows("varchar", "abc", "a".repeat(21000), null));
     // Year gets read out of the DB as a java.sql.Date, so it includes the month/day (both defaulted
     // to 1); the actual data in the DB is just the year