diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitor.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitor.java index d2ed372d0f..6165d08937 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitor.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitor.java @@ -37,6 +37,8 @@ public interface IUnifiedVisitor { void visitFloat64(double d); + void visitFloat32(float f); + void visitBool(boolean b); void visitBytes(byte[] b); @@ -64,12 +66,14 @@ static void dispatch(Value value, IUnifiedVisitor visitor) { case UUID -> visitor.visitUuid(value.getUuid()); case INT64 -> visitor.visitInt64(value.getInt64()); case FLOAT64 -> visitor.visitFloat64(value.getFloat64()); + case FLOAT32 -> visitor.visitFloat32(value.getFloat32()); case BOOL -> visitor.visitBool(value.getBool()); case BYTES -> visitor.visitBytes(value.getBytes().toByteArray()); case DATE -> visitor.visitDate(value.getDate()); case NUMERIC, PG_NUMERIC -> visitor.visitNumeric(value.getNumeric()); case TIMESTAMP -> visitor.visitTimestamp(value.getTimestamp()); - case JSON, PG_JSONB -> visitor.visitJson(value.getJson()); + case JSON -> visitor.visitJson(value.getJson()); + case PG_JSONB -> visitor.visitJson(value.getPgJsonb()); default -> visitor.visitDefault(value); } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitor.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitor.java index 9d5f4e6ea7..c67f4c6f0a 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitor.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitor.java @@ -66,6 +66,13 @@ public void visitFloat64(double d) { hasher.putDouble(d); } + @Override + public void visitFloat32(float f) { + // Float32 values are encoded with a sentinel byte 1 followed by the float value + markNonNull(); + hasher.putFloat(f); + } + @Override public void visitBool(boolean b) { // Bool values are encoded with a sentinel byte 1 followed by the boolean value diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitor.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitor.java index 8c9e860d6e..a943b20445 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitor.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitor.java @@ -49,6 +49,11 @@ public void visitFloat64(double d) { result = String.valueOf(d); } + @Override + public void visitFloat32(float f) { + result = String.valueOf(f); + } + @Override public void visitBool(boolean b) { result = String.valueOf(b); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVAvroSetupHelper.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVAvroSetupHelper.java index 7d81b3ee2e..b740009796 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVAvroSetupHelper.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVAvroSetupHelper.java @@ -15,6 +15,8 @@ */ package com.google.cloud.teleport.v2.templates; +import com.google.common.io.Resources; +import java.io.InputStream; import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -29,6 +31,20 @@ */ public class GCSSpannerDVAvroSetupHelper { + /** + * Helper function to load an Avro Schema from a resource file. + * + * @param resourceName The path to the Avro schema file relative to the resources directory + * @return The parsed Avro Schema + */ + public static Schema getSchemaFromAvscFile(String resourceName) { + try (InputStream is = Resources.getResource(resourceName).openStream()) { + return new Schema.Parser().parse(is); + } catch (Exception e) { + throw new RuntimeException("Failed to load Avro schema from resource: " + resourceName, e); + } + } + /** * Defines standard table schemas that are universally used across multiple integration tests. * Centralizing these definitions prevents schema drift across tests and minimizes setup code. @@ -36,13 +52,12 @@ public class GCSSpannerDVAvroSetupHelper { public static class TableDef { public static final TableDef USERS = new TableDef( - GCSSpannerDVITBase.getSchemaFromAvscFile("GCSSpannerDVAvroSetupHelper/users.avsc"), + getSchemaFromAvscFile("GCSSpannerDVAvroSetupHelper/users.avsc"), "Users", Arrays.asList("user_id", "event_id")); public static final TableDef ACCOUNT_ROLES = new TableDef( - GCSSpannerDVITBase.getSchemaFromAvscFile( - "GCSSpannerDVAvroSetupHelper/account_roles.avsc"), + getSchemaFromAvscFile("GCSSpannerDVAvroSetupHelper/account_roles.avsc"), "AccountRoles", Arrays.asList("role_id")); @@ -102,12 +117,13 @@ public GenericRecord build() { * Converts standard Java types into the primitive formats required by Avro logical types. * *

Because this helper uses a generic {@code Map} to dynamically build records, - * standard Java objects (like {@link Instant}) must be manually translated into Avro's expected - * underlying primitives (like {@code Long} for timestamp-micros) before serialization. + * standard Java objects (like {@link Instant}, {@link java.math.BigDecimal}, {@code byte[]}) must + * be manually translated into Avro's expected underlying primitives (like {@code Long} for + * timestamp-micros, {@link java.nio.ByteBuffer} for decimals with scale 9 or raw bytes) before + * serialization. * - *

IMPORTANT: This method currently only supports {@link Instant}. If new test tables - * are introduced that use other complex Avro mappings (e.g., Dates, Decimals, UUIDs, or custom - * Datastream composites like Datetime), this method MUST be updated to coerce those types. + *

NOTE: {@link java.math.BigDecimal} is currently normalized to scale 9 for Avro + * decimal serialization. * * @param value The standard Java object provided in the test map. * @return The Avro-compatible primitive value ready for serialization. @@ -122,6 +138,16 @@ private static Object convertToAvroFormat(Object value) { return (t.getEpochSecond() * 1_000_000L) + (t.getNano() / 1000L); } + if (value instanceof java.math.BigDecimal) { + java.math.BigDecimal bd = (java.math.BigDecimal) value; + return java.nio.ByteBuffer.wrap( + bd.setScale(9, java.math.RoundingMode.HALF_UP).unscaledValue().toByteArray()); + } + + if (value instanceof byte[]) { + return java.nio.ByteBuffer.wrap((byte[]) value); + } + // Default fallback (String, Integer, Long, Double, Float) return value; } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVITBase.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVITBase.java index 51f0ec1bc3..75112f5c85 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVITBase.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVITBase.java @@ -21,7 +21,6 @@ import com.google.common.io.Resources; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; @@ -206,20 +205,6 @@ public void uploadCustomShardJarToGcs(String gcsPathPrefix) throws IOException { "../spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"); } - /** - * Helper function to load an Avro Schema from a resource file. - * - * @param resourceName The path to the Avro schema file relative to the resources directory - * @return The parsed Avro Schema - */ - public static Schema getSchemaFromAvscFile(String resourceName) { - try (InputStream is = Resources.getResource(resourceName).openStream()) { - return new Schema.Parser().parse(is); - } catch (Exception e) { - throw new RuntimeException("Failed to load Avro schema from resource: " + resourceName, e); - } - } - /** * Serializes a list of GenericRecords into a binary .avro file and uploads it to GCS. * diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVSchemaIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVSchemaIT.java index ea0f6d5ac5..bc1906be57 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVSchemaIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVSchemaIT.java @@ -104,7 +104,8 @@ public void validationTestWithEmptyTables() throws Exception { public void validationTestWithReservedKeywords() throws Exception { Schema reservedKeywordsSchema = - getSchemaFromAvscFile("GCSSpannerDVSchemaIT/reserved_keywords.avsc"); + GCSSpannerDVAvroSetupHelper.getSchemaFromAvscFile( + "GCSSpannerDVSchemaIT/reserved_keywords.avsc"); GCSSpannerDVAvroSetupHelper.TableDef reservedTableDef = new GCSSpannerDVAvroSetupHelper.TableDef( reservedKeywordsSchema, "ORDER", Arrays.asList("SELECT")); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax10MibCellIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax10MibCellIT.java index a92f545981..21e9e524ff 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax10MibCellIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax10MibCellIT.java @@ -57,7 +57,9 @@ public void setUp() throws IOException { public void test10MibCell() throws Exception { GCSSpannerDVAvroSetupHelper.TableDef tableDef = new GCSSpannerDVAvroSetupHelper.TableDef( - getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), "Max10MibCellTable", Arrays.asList("id")); + GCSSpannerDVAvroSetupHelper.getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), + "Max10MibCellTable", + Arrays.asList("id")); final int safeBlobSize = (10 * 1024 * 1024) - 1024; // 9.9MB to avoid limit issues byte[] matchBytes = new byte[safeBlobSize]; diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax16KeyTableIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax16KeyTableIT.java index 000b4721ce..a5682d042b 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax16KeyTableIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMax16KeyTableIT.java @@ -55,7 +55,7 @@ public void setUp() throws IOException { public void test16KeyColumns() throws Exception { GCSSpannerDVAvroSetupHelper.TableDef tableDef = new GCSSpannerDVAvroSetupHelper.TableDef( - getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), + GCSSpannerDVAvroSetupHelper.getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), "Max16KeyTable", Arrays.asList( "col_1", "col_2", "col_3", "col_4", "col_5", "col_6", "col_7", "col_8", "col_9", diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMaxColumnNameIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMaxColumnNameIT.java index b5abc76602..e6a7f65f58 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMaxColumnNameIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVWideRowMaxColumnNameIT.java @@ -62,7 +62,9 @@ public void setUp() throws IOException { public void testMaxColumnNameLength() throws Exception { GCSSpannerDVAvroSetupHelper.TableDef maxColTableDef = new GCSSpannerDVAvroSetupHelper.TableDef( - getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), "MaxColumnNameTable", Arrays.asList("id")); + GCSSpannerDVAvroSetupHelper.getSchemaFromAvscFile(AVRO_SCHEMA_RESOURCE), + "MaxColumnNameTable", + Arrays.asList("id")); List records = Arrays.asList( diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/BulkMigrationAndValidationOracleAllDataTypesE2EIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/BulkMigrationAndValidationOracleAllDataTypesE2EIT.java new file mode 100644 index 0000000000..d637ae2189 --- /dev/null +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/BulkMigrationAndValidationOracleAllDataTypesE2EIT.java @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.endtoend; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts; +import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.TableValidationStatsDto; +import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.ValidationSummaryDto; +import java.io.IOException; +import java.util.Collections; +import java.util.TimeZone; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * End-to-End Integration test validating the migration and validation of all supported data types + * from Oracle to Spanner. + * + *

This test verifies the entire lifecycle of data types across two pipelines (Bulk Migration and + * Data Validation). Specifically, it evaluates how the bulk migration pipeline maps each Oracle + * data type to Avro and Spanner, and subsequently, how the validation pipeline uses those Avro + * files to perform end-to-end data validation. + * + *

The test is driven by schemas that reflect real-world mappings: + * + *

+ * + *

To ensure comprehensive boundary coverage, the test injects and validates four distinct rows + * of data: + * + *

+ */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@RunWith(JUnit4.class) +@TemplateIntegrationTest(GCSSpannerDV.class) +public class BulkMigrationAndValidationOracleAllDataTypesE2EIT extends EndToEndTestingITBase { + + private static final String SPANNER_DDL_RESOURCE = + "BulkMigrationAndValidationOracleAllDataTypesE2EIT/spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "BulkMigrationAndValidationOracleAllDataTypesE2EIT/oracle-schema.sql"; + + private CloudOracleResourceManager oracleResourceManager; + private TimeZone originalTimeZone; + + @Before + public void setUp() throws IOException { + originalTimeZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + oracleResourceManager = CloudOracleResourceManager.builder(testName).build(); + spannerResourceManager = setUpSpannerResourceManager(); + bigQueryResourceManager = setUpBigQueryResourceManager(); + bigQueryResourceManager.createDataset(REGION); + } + + @After + public void tearDown() { + if (originalTimeZone != null) { + TimeZone.setDefault(originalTimeZone); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, flexTemplateDataflowJobResourceManager); + // Spanner and BigQuery are automatically cleaned up in tearDownBase() + } + + @Test + public void allDataTypesE2E() throws Exception { + /* + * Creates a table and inserts 4 boundary testing rows (Standard, NULL, Minimum, Maximum). + */ + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // 2. Launch Bulk Pipeline (SourceDbToSpanner) + String gcsOutputDirectory = "gs://" + artifactBucketName + "/" + testId; + + // Launch Bulk Pipeline (SourceDbToSpanner) + LaunchInfo bulkJobInfo = + launchBulkDataflowJob( + testName, spannerResourceManager, gcsClient, oracleResourceManager, null, false); + assertThatPipeline(bulkJobInfo).isRunning(); + pipelineOperator().waitUntilDone(createConfig(bulkJobInfo)); + + // 3. Assert on spanner rows to verify the bulk job was actually successful + assertThat(spannerResourceManager.getRowCount("AllDatatypes")).isEqualTo(4L); + + // 4. Launch Validation Pipeline (GCSSpannerDV) + LaunchConfig.Builder dvOptions = LaunchConfig.builder(testName, specPath); + LaunchInfo validationJobInfo = + launchDataflowJob( + dvOptions, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsOutputDirectory, + null, + null, + null, + null, + null, + null); + + assertThatPipeline(validationJobInfo).isRunning(); + pipelineOperator().waitUntilDone(createConfig(validationJobInfo)); + + // 5. Assert BigQuery Validation Results (Expect PERFECT MATCH) + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Collections.singletonList( + new ValidationSummaryDto( + /* status= */ "MATCH", + /* totalTablesValidated= */ 1L, + /* totalRowsMatched= */ 4L, + /* totalRowsMismatched= */ 0L, + /* tablesWithMismatches= */ ""))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Collections.singletonList( + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "AllDatatypes", + /* status= */ "MATCH", + /* sourceRowCount= */ 4L, + /* destinationRowCount= */ 4L, + /* matchedRowCount= */ 4L, + /* mismatchRowCount= */ 0L))); + } +} diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/EndToEndTestingITBase.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/EndToEndTestingITBase.java index eec24eaa25..fe5c8b5c97 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/EndToEndTestingITBase.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/endtoend/EndToEndTestingITBase.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.cloudsql.CloudPostgresResourceManager; import org.apache.beam.it.gcp.cloudsql.CloudSqlResourceManager; import org.apache.beam.it.gcp.dataflow.FlexTemplateDataflowJobResourceManager; @@ -114,6 +115,10 @@ protected PipelineLauncher.LaunchInfo launchBulkDataflowJob( connectionProps = null; jdbcDriver = "org.postgresql.Driver"; dialect = "POSTGRESQL"; + } else if (cloudSqlResourceManager instanceof CloudOracleResourceManager) { + connectionProps = null; + jdbcDriver = "oracle.jdbc.OracleDriver"; + dialect = "ORACLE"; } if (!multiSharded) { diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitorTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitorTest.java index 7f4bf66e1d..7002375dde 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitorTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/IUnifiedVisitorTest.java @@ -64,6 +64,39 @@ public void testDispatchMatchesFloat64() { verify(visitor).visitFloat64(input); } + @Test + public void testDispatchMatchesFloat32() { + IUnifiedVisitor visitor = mock(IUnifiedVisitor.class); + float input = 123.456f; + Value value = Value.float32(input); + + IUnifiedVisitor.dispatch(value, visitor); + + verify(visitor).visitFloat32(input); + } + + @Test + public void testDispatchMatchesPgNumeric() { + IUnifiedVisitor visitor = mock(IUnifiedVisitor.class); + BigDecimal input = new BigDecimal("123.456"); + Value value = Value.pgNumeric(input.toString()); + + IUnifiedVisitor.dispatch(value, visitor); + + verify(visitor).visitNumeric(input); + } + + @Test + public void testDispatchMatchesPgJsonb() { + IUnifiedVisitor visitor = mock(IUnifiedVisitor.class); + String input = "{\"key\": \"value\"}"; + Value value = Value.pgJsonb(input); + + IUnifiedVisitor.dispatch(value, visitor); + + verify(visitor).visitJson(input); + } + @Test public void testDispatchMatchesBool() { IUnifiedVisitor visitor = mock(IUnifiedVisitor.class); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitorTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitorTest.java index a5ac4fd48a..0e4ff4abe0 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitorTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedHasherVisitorTest.java @@ -103,6 +103,21 @@ public void testVisitFloat64() { assertEquals(expectedHash, actualHash); } + @Test + public void testVisitFloat32() { + Hasher hasher = Hashing.murmur3_128().newHasher(); + UnifiedHasherVisitor visitor = new UnifiedHasherVisitor(hasher); + float input = 123.456f; + + visitor.visitFloat32(input); + HashCode actualHash = hasher.hash(); + + HashCode expectedHash = + Hashing.murmur3_128().newHasher().putByte((byte) 1).putFloat(input).hash(); + + assertEquals(expectedHash, actualHash); + } + @Test public void testVisitBool() { Hasher hasher = Hashing.murmur3_128().newHasher(); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitorTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitorTest.java index 5070759405..8542be344a 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitorTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/visitor/UnifiedStringVisitorTest.java @@ -70,6 +70,16 @@ public void testVisitFloat64() { assertEquals(String.valueOf(input), visitor.getResult()); } + @Test + public void testVisitFloat32() { + UnifiedStringVisitor visitor = new UnifiedStringVisitor(); + float input = 12.3f; + + visitor.visitFloat32(input); + + assertEquals(String.valueOf(input), visitor.getResult()); + } + @Test public void testVisitBool() { UnifiedStringVisitor visitor = new UnifiedStringVisitor(); diff --git a/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/oracle-schema.sql b/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/oracle-schema.sql new file mode 100644 index 0000000000..af473cc667 --- /dev/null +++ b/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/oracle-schema.sql @@ -0,0 +1,111 @@ +CREATE TABLE AllDatatypes ( + id NUMBER(38, 0) PRIMARY KEY, + varchar_col VARCHAR2(2000), + varchar2_col VARCHAR2(2000), + char_col CHAR(255), + nvarchar2_col NVARCHAR2(2000), + nchar_col NCHAR(255), + number_col NUMBER(38, 0), + numeric_col NUMBER(38, 0), + decimal_col NUMBER(38, 10), + float_col FLOAT, + double_precision_col DOUBLE PRECISION, + binary_float_col BINARY_FLOAT, + binary_double_col BINARY_DOUBLE, + integer_col INTEGER, + int_col INT, + smallint_col SMALLINT, + date_col DATE, + timestamp_col TIMESTAMP(6), + timestamp_tz_col TIMESTAMP(6) WITH TIME ZONE, + timestamp_ltz_col TIMESTAMP(6) WITH LOCAL TIME ZONE, + interval_ym_col INTERVAL YEAR TO MONTH, + interval_ds_col INTERVAL DAY TO SECOND, + raw_col RAW(2000), + clob_col CLOB, + nclob_col NCLOB, + blob_col BLOB, + rowid_col ROWID, + json_col VARCHAR2(4000) +); + +-- Row 1 (Standard Values) +INSERT INTO AllDatatypes ( + id, varchar_col, varchar2_col, char_col, nvarchar2_col, nchar_col, + number_col, numeric_col, decimal_col, float_col, double_precision_col, + binary_float_col, binary_double_col, integer_col, int_col, smallint_col, + date_col, timestamp_col, timestamp_tz_col, timestamp_ltz_col, + interval_ym_col, interval_ds_col, raw_col, clob_col, nclob_col, blob_col, + rowid_col, json_col +) VALUES ( + 1, 'varchar', 'varchar2', 'char', 'nvarchar2', 'nchar', + 12345, 12345, 123.456, 123.45, 123.45, + 123.45, 123.45, 12345, 12345, 123, + TO_DATE('2024-01-01 10:00:00', 'YYYY-MM-DD HH24:MI:SS'), + TO_TIMESTAMP('2024-01-01 10:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.FF6'), + TO_TIMESTAMP_TZ('2024-01-01 10:00:00 +00:00', 'YYYY-MM-DD HH24:MI:SS TZH:TZM'), + TO_TIMESTAMP('2024-01-01 10:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.FF6'), + INTERVAL '1-2' YEAR TO MONTH, + INTERVAL '1 02:03:04.555555' DAY TO SECOND, + HEXTORAW('414243'), + 'clob text', + 'nclob text', + HEXTORAW('414243'), + 'AAAB12AADAAAA12AAA', + '{}' +); + +-- Row 2 (All NULL values) +INSERT INTO AllDatatypes (id) VALUES (2); + +-- Row 3 (Minimum Values) +INSERT INTO AllDatatypes ( + id, varchar_col, varchar2_col, char_col, nvarchar2_col, nchar_col, + number_col, numeric_col, decimal_col, float_col, double_precision_col, + binary_float_col, binary_double_col, integer_col, int_col, smallint_col, + date_col, timestamp_col, timestamp_tz_col, timestamp_ltz_col, + interval_ym_col, interval_ds_col, raw_col, clob_col, nclob_col, blob_col, + rowid_col, json_col +) VALUES ( + 3, 'a', 'a', 'a', 'a', 'a', + -999999999999999999, -999999999999999999, -999999999999999.9999999999, -1.0E38, -1.0E308, + -3.402823E+38, -1.7976931348623157E+308, -2147483648, -2147483648, -32768, + TO_DATE('1970-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS'), + TO_TIMESTAMP('1970-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.FF6'), + TO_TIMESTAMP_TZ('1970-01-01 00:00:00 +00:00', 'YYYY-MM-DD HH24:MI:SS TZH:TZM'), + TO_TIMESTAMP('1970-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.FF6'), + INTERVAL '-99-11' YEAR TO MONTH, + INTERVAL '-99 23:59:59.999999' DAY TO SECOND, + HEXTORAW('00'), + 'min clob', + 'min nclob', + HEXTORAW('00'), + 'AAAAAAAAAAAAAAAAAA', + '{}' +); + +-- Row 4 (Maximum Values) +INSERT INTO AllDatatypes ( + id, varchar_col, varchar2_col, char_col, nvarchar2_col, nchar_col, + number_col, numeric_col, decimal_col, float_col, double_precision_col, + binary_float_col, binary_double_col, integer_col, int_col, smallint_col, + date_col, timestamp_col, timestamp_tz_col, timestamp_ltz_col, + interval_ym_col, interval_ds_col, raw_col, clob_col, nclob_col, blob_col, + rowid_col, json_col +) VALUES ( + 4, RPAD('Z', 2000, 'Z'), RPAD('Z', 2000, 'Z'), RPAD('Z', 255, 'Z'), RPAD('Z', 2000, 'Z'), RPAD('Z', 255, 'Z'), + 999999999999999999, 999999999999999999, 999999999999999.9999999999, 1.0E38, 1.0E308, + 3.402823E+38, 1.7976931348623157E+308, 2147483647, 2147483647, 32767, + TO_DATE('9999-12-31 23:59:59', 'YYYY-MM-DD HH24:MI:SS'), + TO_TIMESTAMP('9999-12-31 23:59:59.999999', 'YYYY-MM-DD HH24:MI:SS.FF6'), + TO_TIMESTAMP_TZ('9999-12-31 23:59:59 +00:00', 'YYYY-MM-DD HH24:MI:SS TZH:TZM'), + TO_TIMESTAMP('9999-12-31 23:59:59.999999', 'YYYY-MM-DD HH24:MI:SS.FF6'), + INTERVAL '99-11' YEAR TO MONTH, + INTERVAL '99 23:59:59.999999' DAY TO SECOND, + HEXTORAW('FFFF'), + RPAD('Z', 4000, 'Z'), + RPAD('Z', 4000, 'Z'), + HEXTORAW('FFFF'), + 'ZZZZZZZZZZZZZZZZZZ', + '{}' +); diff --git a/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/spanner-schema.sql b/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/spanner-schema.sql new file mode 100644 index 0000000000..4ab82bdcc5 --- /dev/null +++ b/v2/gcs-spanner-dv/src/test/resources/BulkMigrationAndValidationOracleAllDataTypesE2EIT/spanner-schema.sql @@ -0,0 +1,30 @@ +CREATE TABLE AllDatatypes ( + id INT64, + varchar_col STRING(2000), + varchar2_col STRING(2000), + char_col STRING(255), + nvarchar2_col STRING(2000), + nchar_col STRING(255), + number_col NUMERIC, + numeric_col NUMERIC, + decimal_col NUMERIC, + float_col FLOAT64, + double_precision_col FLOAT64, + binary_float_col FLOAT32, + binary_double_col FLOAT64, + integer_col INT64, + int_col INT64, + smallint_col INT64, + date_col TIMESTAMP, + timestamp_col TIMESTAMP, + timestamp_tz_col TIMESTAMP, + timestamp_ltz_col TIMESTAMP, + interval_ym_col STRING(MAX), + interval_ds_col STRING(MAX), + raw_col BYTES(MAX), + clob_col STRING(MAX), + nclob_col STRING(MAX), + blob_col BYTES(MAX), + rowid_col STRING(MAX), + json_col JSON +) PRIMARY KEY (id); diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java index 3b82c11394..88d2cf1d92 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java @@ -647,7 +647,9 @@ static String handleLogicalFieldType( return timestamp.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); } else if (fieldSchema.getProp(LOGICAL_TYPE) != null && fieldSchema.getProp(LOGICAL_TYPE).equals(CustomAvroTypes.JSON)) { - if (cassandraAnnotations.cassandraType().getKind().equals(Kind.MAP)) { + if (cassandraAnnotations != null + && cassandraAnnotations.cassandraType() != null + && cassandraAnnotations.cassandraType().getKind().equals(Kind.MAP)) { return AvroJsonToCassandraMapConvertor.handleJsonToMap( recordValue, cassandraAnnotations, fieldName, fieldSchema); } else { diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java index d055a7e4a9..3c326c220a 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java @@ -300,6 +300,15 @@ public void testHandleLogicalFieldType() { getTestCassandraAnnotationNone()); assertEquals("Test json_col conversion: ", "{\"k1\":\"476F6F676C65\"}", result); + col = "json_col"; + result = + GenericRecordTypeConvertor.handleLogicalFieldType( + col, + genericRecord.get(col), + genericRecord.getSchema().getField(col).schema(), + null); + assertEquals("Test json_col conversion: ", "{\"k1\":\"476F6F676C65\"}", result); + col = "json_col"; result = GenericRecordTypeConvertor.handleLogicalFieldType(