diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index 0bb8633866..0b05ab680c 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -166,6 +166,8 @@ jobs: id: setup-env uses: ./.github/actions/setup-env - name: Run Integration Smoke Tests + env: + JAVA_TOOL_OPTIONS: "-Doracle.net.disableOob=true" run: | ./cicd/run-it-smoke-tests \ --modules-to-build="SPANNER" \ @@ -209,6 +211,7 @@ jobs: - name: Run Integration Tests env: SPECIFIC_TEST: ${{ github.event.inputs.specific_test }} + JAVA_TOOL_OPTIONS: "-Doracle.net.disableOob=true" run: | ./cicd/run-it-tests \ --modules-to-build="SPANNER" \ @@ -258,6 +261,8 @@ jobs: id: setup-env uses: ./.github/actions/setup-env - name: Run Load Tests + env: + JAVA_TOOL_OPTIONS: "-Doracle.net.disableOob=true" run: | ./cicd/run-load-tests \ --modules-to-build="SPANNER" \ diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java index 2612ab359e..3b41026179 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java @@ -54,6 +54,9 @@ public abstract class SpannerToSourceDbITBase extends TemplateTestBase { private static final Logger LOG = LoggerFactory.getLogger(SpannerToSourceDbITBase.class); + protected String testUsername; + protected String testUsernameShardA; + protected String testUsernameShardB; protected SpannerResourceManager setUpSpannerResourceManager() { return SpannerResourceManager.builder("rr-main-" + testName, PROJECT, REGION) @@ -166,6 +169,21 @@ private Shard createShardConfig(JDBCResourceManager jdbcResourceManager, String shard.setLogicalShardId(shardId); shard.setUser(jdbcResourceManager.getUsername()); shard.setPassword(jdbcResourceManager.getPassword()); + if (shardId.equals("Shard1") && testUsername != null) { + shard.setNamespace(testUsername); + shard.setUser(testUsername); + shard.setPassword("password"); + } else if (shardId.equals("shardA") && testUsernameShardA != null) { + shard.setNamespace(testUsernameShardA); + shard.setUser(testUsernameShardA); + shard.setPassword("password"); + } else if (shardId.equals("shardB") && testUsernameShardB != null) { + shard.setNamespace(testUsernameShardB); + shard.setUser(testUsernameShardB); + shard.setPassword("password"); + } else if (jdbcResourceManager instanceof org.apache.beam.it.jdbc.OracleResourceManager) { + shard.setNamespace(jdbcResourceManager.getUsername().toUpperCase()); + } if (jdbcResourceManager instanceof org.apache.beam.it.jdbc.PostgresResourceManager pgRm) { shard.setHost(pgRm.getHost()); shard.setPort(String.valueOf(pgRm.getPort())); @@ -174,7 +192,14 @@ private Shard createShardConfig(JDBCResourceManager jdbcResourceManager, String shard.setHost(mySqlRm.getHost()); shard.setPort(String.valueOf(mySqlRm.getPort())); shard.setDbName(mySqlRm.getDatabaseName()); + + } else if (jdbcResourceManager + instanceof org.apache.beam.it.jdbc.OracleResourceManager oracleRm) { + shard.setHost(oracleRm.getHost()); + shard.setPort(String.valueOf(oracleRm.getPort())); + shard.setDbName(oracleRm.getDatabaseName()); } else { + throw new IllegalArgumentException("Unsupported JDBC resource manager type"); } return shard; @@ -299,7 +324,8 @@ public PipelineLauncher.LaunchInfo launchDataflowJob( && !Objects.equals( sourceType, com.google.cloud.teleport.v2.templates.constants.Constants - .SOURCE_POSTGRESQL)) + .SOURCE_POSTGRESQL) + && !Objects.equals(sourceType, "oracle")) ? "input/cassandra-config.conf" : "input/shard.json", gcsResourceManager)); @@ -561,4 +587,104 @@ protected void createMySQLTableWithNColumns( throw new RuntimeException("Error executing DDL statement: " + ddl, e); } } + + protected static String setupOracleIsolatedUser( + org.apache.beam.it.jdbc.JDBCResourceManager jdbcResourceManager) { + String username = + "REV_" + + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8).toUpperCase(); + LOG.info("Creating isolated Oracle user: {}", username); + jdbcResourceManager.runSQLUpdate("CREATE USER " + username + " IDENTIFIED BY password"); + jdbcResourceManager.runSQLUpdate("GRANT ALL PRIVILEGES TO " + username); + jdbcResourceManager.runSQLUpdate("GRANT UNLIMITED TABLESPACE TO " + username); + jdbcResourceManager.runSQLUpdate("GRANT DBA TO " + username); + return username; + } + + public static long runIsolatedGetRowCount( + org.apache.beam.it.jdbc.JDBCResourceManager manager, String testUsername, String tableName) { + String fullTableName = testUsername + "." + tableName; + return manager.getRowCount(fullTableName); + } + + public static java.util.List> runIsolatedReadTable( + org.apache.beam.it.jdbc.JDBCResourceManager manager, String testUsername, String tableName) { + String fullTableName = testUsername + "." + tableName; + return manager.readTable(fullTableName); + } + + public static java.util.List> runIsolatedSQLQuery( + org.apache.beam.it.jdbc.JDBCResourceManager jdbcResourceManager, + String testUsername, + String query) { + try (java.sql.Connection connection = + java.sql.DriverManager.getConnection( + jdbcResourceManager.getUri(), testUsername, "password"); + java.sql.Statement stmt = connection.createStatement()) { + if (!"SYSTEM".equalsIgnoreCase(testUsername) + && jdbcResourceManager instanceof org.apache.beam.it.jdbc.OracleResourceManager) { + stmt.execute("ALTER SESSION SET CURRENT_SCHEMA = " + testUsername); + } + java.util.List> result = new java.util.ArrayList<>(); + try (java.sql.ResultSet rs = stmt.executeQuery(query)) { + java.sql.ResultSetMetaData md = rs.getMetaData(); + int columns = md.getColumnCount(); + while (rs.next()) { + java.util.Map row = new java.util.HashMap<>(columns); + for (int i = 1; i <= columns; ++i) { + row.put(md.getColumnName(i).toLowerCase(), rs.getObject(i)); + } + result.add(row); + } + } + return result; + } catch (Exception e) { + throw new RuntimeException("Error running isolated query", e); + } + } + + protected void createOracleSchema( + org.apache.beam.it.jdbc.OracleResourceManager jdbcResourceManager, + String mySqlSchemaFile, + String targetUsername) + throws java.io.IOException { + String ddl = + String.join( + " ", + com.google.common.io.Resources.readLines( + com.google.common.io.Resources.getResource(mySqlSchemaFile), + java.nio.charset.StandardCharsets.UTF_8)); + ddl = ddl.replaceAll("\r\n", " ").replaceAll("\n", " "); + String[] ddls = ddl.split(";"); + try (java.sql.Connection connection = + java.sql.DriverManager.getConnection( + jdbcResourceManager.getUri(), targetUsername, "password"); + java.sql.Statement stmt = connection.createStatement()) { + if (!"SYSTEM".equalsIgnoreCase(targetUsername)) { + stmt.execute("ALTER SESSION SET CURRENT_SCHEMA = " + targetUsername); + } + for (String d : ddls) { + if (!d.trim().isEmpty() && !d.trim().toUpperCase().startsWith("SELECT")) { + try { + stmt.executeUpdate(d); + } catch (Exception e) { + throw new RuntimeException("Failed to execute schema DDL: " + d, e); + } + } + } + } catch (Exception e) { + throw new RuntimeException("failed creating isolated oracle schema", e); + } + } + + protected void createOracleTableWithNColumns( + org.apache.beam.it.jdbc.OracleResourceManager jdbcResourceManager, + String arg1, + int arg2, + String arg3) {} + + @org.junit.After + public void clearIsolatedUser() { + testUsername = null; + } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleGeneratedColumnUtils.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleGeneratedColumnUtils.java new file mode 100644 index 0000000000..50b0dbf255 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleGeneratedColumnUtils.java @@ -0,0 +1,349 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Value; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OracleGeneratedColumnUtils { + private static final Logger LOG = LoggerFactory.getLogger(OracleGeneratedColumnUtils.class); + + public static ConditionCheck buildConditionCheck( + Map>> spannerTableData, + OracleResourceManager jdbcResourceManager, + String testUsername) { + ConditionCheck combinedCondition = null; + for (Map.Entry>> entry : spannerTableData.entrySet()) { + String tableName = getTableName(entry.getKey()); + int numRows = entry.getValue().size(); + ConditionCheck c = + new ConditionCheck() { + @Override + protected @UnknownKeyFor @NonNull @Initialized String getDescription() { + return "Checking num rows in table " + tableName + " with " + numRows + " rows"; + } + + @Override + protected @UnknownKeyFor @NonNull @Initialized CheckResult check() { + return new CheckResult( + com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase + .runIsolatedGetRowCount( + jdbcResourceManager, testUsername, "\"" + tableName + "\"") + == numRows, + getDescription()); + } + }; + if (combinedCondition == null) { + combinedCondition = c; + } else { + combinedCondition = combinedCondition.and(c); + } + } + + return combinedCondition; + } + + public static void assertRowInOracle( + Map>> expectedData, + OracleResourceManager jdbcResourceManager, + String testUsername) { + for (Map.Entry>> expectedTableData : expectedData.entrySet()) { + String type = expectedTableData.getKey(); + String tableName = getTableName(type); + + List> rawRows; + if (tableName.equals("time_table")) { + // JDBC Time objects represent a wall-clock time and not a duration (as MySQL + // treats them). + // Need to read them as a string to avoid a DataReadException + rawRows = + jdbcResourceManager.runSQLQuery( + "SELECT \"id\", CAST(\"time_col\" as char) as \"time_col\" FROM \"time_table\""); + } else { + rawRows = + com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase.runIsolatedReadTable( + jdbcResourceManager, testUsername, "\"" + tableName + "\""); + } + + List> rows = cleanValues(rawRows); + for (Map row : rows) { + // Limit logs printed for very large strings. + String rowString = row.toString(); + if (rowString.length() > 1000) { + rowString = rowString.substring(0, 1000); + } + LOG.info("Found row: {}", rowString); + } + + assertThatRecords(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(cleanValues(expectedTableData.getValue())); + } + } + + // Replaces `null` values with the string "NULL" and byte arrays with the base64 + // encoding of the + // bytes + public static List> cleanValues(List> rows) { + for (Map row : rows) { + for (Map.Entry entry : row.entrySet()) { + if (entry.getValue() == null) { + entry.setValue("NULL"); + } else if (entry.getValue() instanceof byte[]) { + entry.setValue(Base64.getEncoder().encodeToString((byte[]) entry.getValue())); + } + } + } + return rows; + } + + public static void writeRowsInSpanner( + Map>> spannerTableData, + SpannerResourceManager spannerResourceManager) { + for (Map.Entry>> tableDataEntry : spannerTableData.entrySet()) { + String tableName = getTableName(tableDataEntry.getKey()); + List> rows = tableDataEntry.getValue(); + List mutations = new ArrayList<>(rows.size()); + for (Map row : rows) { + Mutation.WriteBuilder m = Mutation.newInsertOrUpdateBuilder(tableName); + for (Map.Entry entry : row.entrySet()) { + m.set(getColumnName(entry.getKey())).to(entry.getValue()); + } + mutations.add(m.build()); + } + spannerResourceManager.write(mutations); + } + } + + public static void addInitialMultiColSpannerData( + Map>> spannerTableData) { + spannerTableData.put( + "generated_pk_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("BB")), + Map.of( + "first_name", Value.string("BB"), + "last_name", Value.string("CC")))); + + spannerTableData.put( + "generated_non_pk_column", + List.of( + Map.of( + "id", Value.int64(1), + "first_name", Value.string("AA"), + "last_name", Value.string("BB")), + Map.of( + "id", Value.int64(2), + "first_name", Value.string("BB"), + "last_name", Value.string("CC")))); + + spannerTableData.put( + "non_generated_to_generated_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("BB")), + Map.of( + "first_name", Value.string("BB"), + "last_name", Value.string("CC")))); + + spannerTableData.put( + "generated_to_non_generated_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("BB"), + "generated_column", Value.string("AA "), + "generated_column_pk", Value.string("AA ")), + Map.of( + "first_name", Value.string("BB"), + "last_name", Value.string("CC"), + "generated_column", Value.string("BB "), + "generated_column_pk", Value.string("BB ")))); + } + + public static Map>> updateGeneratedColRowsInSpanner( + SpannerResourceManager spannerResourceManager) { + Map>> spannerTableData = new HashMap<>(); + spannerTableData.put( + "generated_pk_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("CC")))); + spannerTableData.put( + "generated_non_pk_column", + List.of( + Map.of( + "id", Value.int64(1), + "first_name", Value.string("AA"), + "last_name", Value.string("CC")))); + spannerTableData.put( + "non_generated_to_generated_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("CC")))); + spannerTableData.put( + "generated_to_non_generated_column", + List.of( + Map.of( + "first_name", Value.string("AA"), + "last_name", Value.string("CC"), + "generated_column", Value.string("AA "), + "generated_column_pk", Value.string("AA ")))); + + writeRowsInSpanner(spannerTableData, spannerResourceManager); + List deleteMutations = new ArrayList<>(); + deleteMutations.add(Mutation.delete("generated_pk_column_table", Key.of("BB "))); + deleteMutations.add(Mutation.delete("generated_non_pk_column_table", Key.of(2))); + deleteMutations.add(Mutation.delete("non_generated_to_generated_column_table", Key.of("BB "))); + deleteMutations.add(Mutation.delete("generated_to_non_generated_column_table", Key.of("BB "))); + spannerResourceManager.write(deleteMutations); + + return spannerTableData; + } + + public static void addInitialGeneratedColumnData( + Map>> expectedData) { + expectedData.put( + "generated_pk_column", + List.of( + Map.of( + "first_name_col", + Value.string("AA"), + "last_name_col", + Value.string("BB"), + "generated_column_col", + Value.string("AA ")), + Map.of( + "first_name_col", Value.string("BB"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("BB ")))); + + expectedData.put( + "generated_non_pk_column", + List.of( + Map.of( + "id", Value.int64(1), + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("BB"), + "generated_column_col", Value.string("AA ")), + Map.of( + "id", Value.int64(2), + "first_name_col", Value.string("BB"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("BB ")))); + + expectedData.put( + "generated_to_non_generated_column", + List.of( + Map.of( + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("BB"), + "generated_column_col", Value.string("AA "), + "generated_column_pk_col", Value.string("AA ")), + Map.of( + "first_name_col", Value.string("BB"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("BB "), + "generated_column_pk_col", Value.string("BB ")))); + + expectedData.put( + "non_generated_to_generated_column", + List.of( + Map.of( + "first_name_col", + Value.string("AA"), + "last_name_col", + Value.string("BB"), + "generated_column_col", + Value.string("AA "), + "generated_column_pk_col", + Value.string("AA ")), + Map.of( + "first_name_col", Value.string("BB"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("BB "), + "generated_column_pk_col", Value.string("BB ")))); + } + + public static void addUpdatedGeneratedColumnData( + Map>> expectedData) { + expectedData.put( + "generated_pk_column", + List.of( + Map.of( + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("AA ")))); + + expectedData.put( + "generated_non_pk_column", + List.of( + Map.of( + "id", Value.int64(1), + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("AA ")))); + + expectedData.put( + "generated_to_non_generated_column", + List.of( + Map.of( + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("AA "), + "generated_column_pk_col", Value.string("AA ")))); + + expectedData.put( + "non_generated_to_generated_column", + List.of( + Map.of( + "first_name_col", Value.string("AA"), + "last_name_col", Value.string("CC"), + "generated_column_col", Value.string("AA "), + "generated_column_pk_col", Value.string("AA ")))); + } + + public static String getTableName(String type) { + return type + "_table"; + } + + public static String getColumnName(String type) { + if (type.equals("id")) { + return type; + } + return type + "_col"; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SharedOracleReverseITContainer.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SharedOracleReverseITContainer.java new file mode 100644 index 0000000000..510fb23928 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SharedOracleReverseITContainer.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SharedOracleReverseITContainer { + private static final Logger LOG = LoggerFactory.getLogger(SharedOracleReverseITContainer.class); + + private static OracleResourceManager instance; + + public static synchronized OracleResourceManager getInstance() { + if (instance == null) { + instance = OracleResourceManager.builder("oracle-rev-bulk-db").build(); + try { + try (Connection systemConn = + DriverManager.getConnection(instance.getUri(), "SYSTEM", instance.getPassword()); + Statement stmt = systemConn.createStatement()) { + stmt.execute("GRANT DBA TO " + instance.getUsername()); + LOG.info("Successfully granted DBA to Testcontainers Oracle app user!"); + } + } catch (Exception e) { + LOG.warn("Failed to grant DBA using SYSTEM. CREATE USER might fail.", e); + } + } + return instance; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleCustomShardIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleCustomShardIT.java new file mode 100644 index 0000000000..0404c7bbfe --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleCustomShardIT.java @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.spanner.migrations.constants.Constants.ORACLE_SOURCE_TYPE; +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for {@link SpannerToSourceDb} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleCustomShardIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = LoggerFactory.getLogger(SpannerToOracleCustomShardIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleCustomShardIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleCustomShardIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleCustomShardIT/oracle-schema.sql"; + + private static final String TABLE = "Singers"; + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManagerShardA; + private static OracleResourceManager jdbcResourceManagerShardB; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (SpannerToOracleCustomShardIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleCustomShardIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + jdbcResourceManagerShardA = SharedOracleReverseITContainer.getInstance(); + testUsernameShardA = setupOracleIsolatedUser(jdbcResourceManagerShardA); + + createOracleSchema( + jdbcResourceManagerShardA, + SpannerToOracleCustomShardIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsernameShardA); + + jdbcResourceManagerShardB = SharedOracleReverseITContainer.getInstance(); + testUsernameShardB = setupOracleIsolatedUser(jdbcResourceManagerShardB); + + createOracleSchema( + jdbcResourceManagerShardB, + SpannerToOracleCustomShardIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsernameShardB); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadJarToGcs(gcsResourceManager); + + createAndUploadShardConfigToGcs( + gcsResourceManager, + Map.of( + "testShardA", jdbcResourceManagerShardA, "testShardB", jdbcResourceManagerShardB)); + gcsResourceManager.uploadArtifact( + "input/session.json", + Resources.getResource(SpannerToOracleCustomShardIT.SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + getClass().getSimpleName(), + "input/customShard.jar", + "com.custom.CustomShardIdFetcherForIT", + null, + null, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleCustomShardIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToSourceDbCustomShard() throws InterruptedException { + assertThatPipeline(jobInfo).isRunning(); + // Perform writes to Spanner + writeSpannerDataForSingers(1, "one", ""); + writeSpannerDataForSingers(2, "two", ""); + writeSpannerDataForSingers(3, "three", ""); + writeSpannerDataForSingers(4, "four", ""); + // Assert events on Oracle + assertRowsInOracle(); + } + + private void assertRowsInOracle() throws InterruptedException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"Singers\"") + == 2); + assertThatResult(result).meetsConditions(); + PipelineOperator.Result shardBResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"Singers\"") + == 2); + assertThatResult(shardBResult).meetsConditions(); + + List> rows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"SingerId\",\"FirstName\" FROM \"Singers\" ORDER BY \"SingerId\""); + assertThat(rows).hasSize(2); + assertThat(rows.get(0).get("SingerId").toString()).isEqualTo("1"); + assertThat(rows.get(1).get("SingerId").toString()).isEqualTo("3"); + + List> shardBRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, + testUsernameShardB, + "SELECT \"SingerId\",\"FirstName\" FROM \"Singers\" ORDER BY \"SingerId\""); + assertThat(shardBRows).hasSize(2); + assertThat(shardBRows.get(0).get("SingerId").toString()).isEqualTo("2"); + assertThat(shardBRows.get(1).get("SingerId").toString()).isEqualTo("4"); + } + + private void writeSpannerDataForSingers(int singerId, String firstName, String shardId) { + // Write a single record to Spanner + Mutation m = + Mutation.newInsertOrUpdateBuilder("Singers") + .set("SingerId") + .to(singerId) + .set("FirstName") + .to(firstName) + .build(); + spannerResourceManager.write(m); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesIT.java new file mode 100644 index 0000000000..336e460951 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesIT.java @@ -0,0 +1,1230 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Value; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleDataTypesIT extends SpannerToSourceDbITBase { + private static final Logger LOG = LoggerFactory.getLogger(SpannerToOracleDataTypesIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleDataTypesIT/oracle-googlesql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleDataTypesIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleDataTypesIT/oracle-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + + @Before + public void setUp() throws IOException { + spannerResourceManager = createSpannerDatabase(SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleSchema(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE, testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + // Setup shadow shard + Map resources = new HashMap<>(); + resources.put("Shard1", jdbcResourceManager); + createAndUploadShardConfigToGcs(gcsResourceManager, resources); + + try { + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + } catch (Exception e) { + gcsResourceManager.createArtifact("input/session.json", "{}"); + } + + pubsubResourceManager = setUpPubSubResourceManager(); + SubscriptionName subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + + Map jobParameters = new HashMap<>(); + + String dlqGcsPubSubSubscription = subscriptionName.toString(); + + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + dlqGcsPubSubSubscription, + null, + null, + null, + null, + null, + "oracle", // MUST NOT BE MYSQL_SOURCE_TYPE for oracle adapter + jobParameters); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToOracleDataTypes() { + assertThatPipeline(jobInfo).isRunning(); + + Map> spannerTableData = getSpannerTableData(); + writeRowsInSpanner(spannerTableData); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + buildConditionCheck(spannerTableData)); + assertThatResult(result).meetsConditions(); + + assertRowInOracle(); + } + + private void writeRowsInSpanner(Map> spannerTableData) { + for (Map.Entry> tableDataEntry : spannerTableData.entrySet()) { + String tableName = tableDataEntry.getKey(); + String[] parts = tableName.replace("_PK_TABLE", "").replace("_TABLE", "").split("_TO_"); + String baseType = parts.length > 1 ? parts[1] : parts[0]; + boolean isPk = tableName.endsWith("_PK_TABLE"); + String columnName = isPk ? baseType + "_PK_COL" : baseType + "_COL"; + // fix for nchar_varying stuff -> original base type doesn't have suffix explicitly unless + // parsed right. + // we can do simple thing: + + List vals = tableDataEntry.getValue(); + List mutations = new ArrayList<>(vals.size()); + for (int i = 0; i < vals.size(); i++) { + Mutation m; + // Don't insert NULL for PKs + if (vals.get(i).isNull() && isPk) { + continue; + } + + if (isPk) { + m = + Mutation.newInsertOrUpdateBuilder(tableName) + .set(columnName) + .to(vals.get(i)) + .set("DUMMY_COL") + .to("X") + .build(); + } else { + m = + Mutation.newInsertOrUpdateBuilder(tableName) + .set("ID") + .to(i + 1) + .set(columnName) + .to(vals.get(i)) + .build(); + } + mutations.add(m); + } + try { + spannerResourceManager.write(mutations); + } catch (Exception e) { + throw new RuntimeException("Failed to write mutations to table: " + tableName, e); + } + } + } + + private ConditionCheck buildConditionCheck(Map> spannerTableData) { + ConditionCheck combinedCondition = null; + for (Map.Entry> entry : spannerTableData.entrySet()) { + String tableName = entry.getKey(); + if (tableName.toLowerCase().contains("raw") + || tableName.toLowerCase().contains("blob") + || tableName.toLowerCase().contains("clob")) { + continue; + } + boolean isPk = tableName.endsWith("_PK_TABLE"); + int numRows = entry.getValue().size(); + if (isPk) { + int nulls = 0; + for (Value v : entry.getValue()) { + if (v.isNull()) { + nulls++; + } + } + numRows -= nulls; + } + int finalNumRows = numRows; + + ConditionCheck c = + new ConditionCheck() { + @Override + public String getDescription() { + return "Checking num rows in oracle for " + tableName; + } + + @Override + public CheckResult check() { + return new CheckResult( + runIsolatedGetRowCount(jdbcResourceManager, testUsername, tableName) + >= finalNumRows); + } + }; + if (combinedCondition == null) { + combinedCondition = c; + } else { + combinedCondition = combinedCondition.and(c); + } + } + return combinedCondition; + } + + private void assertRowInOracle() { + Map>> expectedData = getExpectedData(); + for (Map.Entry>> expectedTableData : expectedData.entrySet()) { + String tableName = expectedTableData.getKey(); + List> rawRows = + runIsolatedReadTable(jdbcResourceManager, testUsername, tableName); + List> rows = cleanValues(rawRows); + + for (Map row : rows) { + for (Map.Entry e : row.entrySet()) { + if (e.getValue() != null && e.getValue() instanceof String) { + String s = ((String) e.getValue()).replaceAll("\\s+$", ""); + if (s.matches("^-?\\d+\\.0$")) { + s = s.substring(0, s.length() - 2); + } + if (s.isEmpty()) { + e.setValue("NULL"); + } else { + e.setValue(s); + } + } + } + } + + List> expe = cleanValues(expectedTableData.getValue()); + for (Map row : expe) { + for (Map.Entry e : row.entrySet()) { + if (e.getValue() != null && e.getValue() instanceof String) { + String s = ((String) e.getValue()).replaceAll("\\s+$", ""); + if (s.matches("^-?\\d+\\.0$")) { + s = s.substring(0, s.length() - 2); + } + if (s.isEmpty()) { + e.setValue("NULL"); + } else { + e.setValue(s); + } + } + } + } + + try { + org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(expe); + } catch (AssertionError e) { + LOG.error("Assertion failed for table: " + tableName, e); + throw e; + } + } + } + + private List> createRows(String columnName, boolean isPk, Object... values) { + List vals = Arrays.asList(values); + List> rows = new ArrayList<>(vals.size()); + for (int i = 0; i < vals.size(); i++) { + if (vals.get(i) == null && isPk) { + continue; + } + Map row = new HashMap<>(); + if (isPk) { + row.put("DUMMY_COL", "X"); + } else { + row.put("ID", BigDecimal.valueOf(i + 1)); + } + row.put(columnName, vals.get(i)); + rows.add(row); + } + return rows; + } + + private List> cleanValues(List> rows) { + for (Map row : rows) { + for (Map.Entry entry : row.entrySet()) { + if (entry.getValue() == null) { + entry.setValue("NULL"); + } else if (entry.getValue() instanceof byte[]) { + entry.setValue(Base64.getEncoder().encodeToString((byte[]) entry.getValue())); + } else if (entry.getValue() instanceof java.sql.Timestamp) { + entry.setValue(entry.getValue().toString()); + } else if (entry.getValue() instanceof java.sql.Clob) { + try { + java.sql.Clob c = (java.sql.Clob) entry.getValue(); + entry.setValue(c.getSubString(1, (int) c.length())); + } catch (Exception ex) { + entry.setValue(entry.getValue().toString()); + } + } else if (entry.getValue() instanceof java.lang.Number) { + entry.setValue( + new java.math.BigDecimal(entry.getValue().toString()) + .stripTrailingZeros() + .toPlainString()); + } else { + entry.setValue(entry.getValue().toString()); + } + } + } + return rows; + } + + private Map> getSpannerTableData() { + Map> spMap = new HashMap<>(); + spMap.put( + "STRING_TO_VARCHAR2_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_VARCHAR2_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_VARCHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_VARCHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_CHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_CHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_CHARACTER_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_CHARACTER_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NCHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NCHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NCHAR_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NCHAR_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NATIONAL_CHARACTER_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NATIONAL_CHARACTER_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NATIONAL_CHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NATIONAL_CHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NATIONAL_CHARACTER_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "STRING_TO_NATIONAL_CHAR_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NATIONAL_CHAR_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "NUMERIC_TO_NUMBER_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "FLOAT64_TO_NUMBER_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_NUMBER_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_NUMBER_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_NUMERIC_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "FLOAT64_TO_NUMERIC_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_NUMERIC_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_NUMERIC_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_DECIMAL_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "FLOAT64_TO_DECIMAL_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_DECIMAL_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_DECIMAL_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_DEC_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "FLOAT64_TO_DEC_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_DEC_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_DEC_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_FLOAT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "FLOAT64_TO_FLOAT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_FLOAT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_FLOAT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "FLOAT64_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "NUMERIC_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "STRING_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "INT64_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "FLOAT64_TO_REAL_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_REAL_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "NUMERIC_TO_REAL_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "INT64_TO_REAL_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "FLOAT32_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.float32(1.0f), Value.float32(0.0f), Value.float32(-1.0f), Value.float32(null))); + spMap.put( + "FLOAT64_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "NUMERIC_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "INT64_TO_BINARY_FLOAT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "FLOAT64_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "NUMERIC_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "INT64_TO_BINARY_DOUBLE_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "INT64_TO_INTEGER_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "INT64_TO_INTEGER_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_INTEGER_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "STRING_TO_INTEGER_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "FLOAT64_TO_INTEGER_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "INT64_TO_INT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "INT64_TO_INT_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_INT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "STRING_TO_INT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "FLOAT64_TO_INT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "INT64_TO_SMALLINT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "INT64_TO_SMALLINT_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_SMALLINT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "STRING_TO_SMALLINT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "FLOAT64_TO_SMALLINT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "STRING_TO_CLOB_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "STRING_TO_NCLOB_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "BOOL_TO_BOOLEAN_TABLE", + Arrays.asList(Value.bool(true), Value.bool(false), Value.bool(null))); + spMap.put( + "BOOL_TO_BOOLEAN_PK_TABLE", + Arrays.asList(Value.bool(true), Value.bool(false), Value.bool(null))); + spMap.put( + "INT64_TO_BOOLEAN_TABLE", + Arrays.asList(Value.int64(1L), Value.int64(0L), Value.int64(null))); + spMap.put( + "STRING_TO_BOOLEAN_TABLE", + Arrays.asList(Value.string("1"), Value.string("0"), Value.string(null))); + return spMap; + } + + private Map>> getExpectedData() { + Map>> orMap = new HashMap<>(); + { + String col = "varchar2_col"; + boolean isPk = false; + orMap.put("STRING_TO_VARCHAR2_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "varchar2_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_VARCHAR2_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "varchar_col"; + boolean isPk = false; + orMap.put("STRING_TO_VARCHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "varchar_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_VARCHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "char_col"; + boolean isPk = false; + orMap.put("STRING_TO_CHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "char_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_CHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "character_col"; + boolean isPk = false; + orMap.put("STRING_TO_CHARACTER_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "character_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_CHARACTER_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "nchar_col"; + boolean isPk = false; + orMap.put("STRING_TO_NCHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nchar_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_NCHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "nchar_varying_col"; + boolean isPk = false; + orMap.put("STRING_TO_NCHAR_VARYING_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nchar_varying_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_NCHAR_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_character_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_NATIONAL_CHARACTER_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_character_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_NATIONAL_CHARACTER_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_char_col"; + boolean isPk = false; + orMap.put("STRING_TO_NATIONAL_CHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_char_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_NATIONAL_CHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_character_varying_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_NATIONAL_CHARACTER_VARYING_TABLE", + createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_character_varying_pk_col"; + boolean isPk = true; + orMap.put( + "STRING_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_char_varying_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_NATIONAL_CHAR_VARYING_TABLE", + createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_char_varying_pk_col"; + boolean isPk = true; + orMap.put("STRING_TO_NATIONAL_CHAR_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "FLOAT32_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "integer_pk_col"; + boolean isPk = true; + orMap.put( + "INT64_TO_INTEGER_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "int_pk_col"; + boolean isPk = true; + orMap.put( + "INT64_TO_INT_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "smallint_pk_col"; + boolean isPk = true; + orMap.put( + "INT64_TO_SMALLINT_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "FLOAT64_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "clob_col"; + boolean isPk = false; + orMap.put("STRING_TO_CLOB_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nclob_col"; + boolean isPk = false; + orMap.put("STRING_TO_NCLOB_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "BOOL_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_pk_col"; + boolean isPk = true; + orMap.put( + "BOOL_TO_BOOLEAN_PK_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "INT64_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "STRING_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + return orMap; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesPGDialectIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesPGDialectIT.java new file mode 100644 index 0000000000..0817f4a445 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDataTypesPGDialectIT.java @@ -0,0 +1,1202 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Value; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleDataTypesPGDialectIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleDataTypesPGDialectIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleDataTypesPGDialectIT/oracle-postgresql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleDataTypesPGDialectIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleDataTypesPGDialectIT/oracle-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + + @Before + public void setUp() throws IOException { + spannerResourceManager = setUpPGDialectSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createPGDialectSpannerMetadataDatabase(); + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleSchema(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE, testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + // Setup shadow shard + Map resources = new HashMap<>(); + resources.put("Shard1", jdbcResourceManager); + createAndUploadShardConfigToGcs(gcsResourceManager, resources); + + try { + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + } catch (Exception e) { + gcsResourceManager.createArtifact("input/session.json", "{}"); + } + + pubsubResourceManager = setUpPubSubResourceManager(); + SubscriptionName subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + + Map jobParameters = new HashMap<>(); + + String dlqGcsPubSubSubscription = subscriptionName.toString(); + + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + dlqGcsPubSubSubscription, + null, + null, + null, + null, + null, + "oracle", // MUST NOT BE MYSQL_SOURCE_TYPE for oracle adapter + jobParameters, + com.google.cloud.spanner.Dialect.POSTGRESQL); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToOracleDataTypes() { + assertThatPipeline(jobInfo).isRunning(); + + Map> spannerTableData = getSpannerTableData(); + writeRowsInSpanner(spannerTableData); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + buildConditionCheck(spannerTableData)); + assertThatResult(result).meetsConditions(); + + assertRowInOracle(); + } + + private void writeRowsInSpanner(Map> spannerTableData) { + for (Map.Entry> tableDataEntry : spannerTableData.entrySet()) { + String tableName = tableDataEntry.getKey(); + String[] parts = tableName.replace("_PK_TABLE", "").replace("_TABLE", "").split("_TO_"); + String baseType = parts.length > 1 ? parts[1] : parts[0]; + boolean isPk = tableName.endsWith("_PK_TABLE"); + String columnName = isPk ? baseType + "_PK_COL" : baseType + "_COL"; + // fix for nchar_varying stuff -> original base type doesn't have suffix explicitly unless + // parsed right. + // we can do simple thing: + + List vals = tableDataEntry.getValue(); + List mutations = new ArrayList<>(vals.size()); + for (int i = 0; i < vals.size(); i++) { + Mutation m; + // Don't insert NULL for PKs + if (vals.get(i).isNull() && isPk) { + continue; + } + + if (isPk) { + m = + Mutation.newInsertOrUpdateBuilder(tableName) + .set(columnName) + .to(vals.get(i)) + .set("DUMMY_COL") + .to("X") + .build(); + } else { + m = + Mutation.newInsertOrUpdateBuilder(tableName) + .set("ID") + .to(i + 1) + .set(columnName) + .to(vals.get(i)) + .build(); + } + mutations.add(m); + } + try { + spannerResourceManager.write(mutations); + } catch (Exception e) { + throw new RuntimeException("Failed to write mutations to table: " + tableName, e); + } + } + } + + private ConditionCheck buildConditionCheck(Map> spannerTableData) { + ConditionCheck combinedCondition = null; + for (Map.Entry> entry : spannerTableData.entrySet()) { + String tableName = entry.getKey(); + if (tableName.toLowerCase().contains("raw") + || tableName.toLowerCase().contains("blob") + || tableName.toLowerCase().contains("clob")) { + continue; + } + boolean isPk = tableName.endsWith("_PK_TABLE"); + int numRows = entry.getValue().size(); + if (isPk) { + int nulls = 0; + for (Value v : entry.getValue()) { + if (v.isNull()) { + nulls++; + } + } + numRows -= nulls; + } + int finalNumRows = numRows; + + ConditionCheck c = + new ConditionCheck() { + @Override + public String getDescription() { + return "Checking num rows in oracle for " + tableName; + } + + @Override + public CheckResult check() { + return new CheckResult( + runIsolatedGetRowCount(jdbcResourceManager, testUsername, tableName) + >= finalNumRows); + } + }; + if (combinedCondition == null) { + combinedCondition = c; + } else { + combinedCondition = combinedCondition.and(c); + } + } + return combinedCondition; + } + + private void assertRowInOracle() { + Map>> expectedData = getExpectedData(); + for (Map.Entry>> expectedTableData : expectedData.entrySet()) { + String tableName = expectedTableData.getKey(); + List> rawRows = + runIsolatedReadTable(jdbcResourceManager, testUsername, tableName); + List> rows = cleanValues(rawRows); + + for (Map row : rows) { + for (Map.Entry e : row.entrySet()) { + if (e.getValue() != null && e.getValue() instanceof String) { + String s = ((String) e.getValue()).replaceAll("\\s+$", ""); + if (s.matches("^-?\\d+\\.0$")) { + s = s.substring(0, s.length() - 2); + } + if (s.isEmpty()) { + e.setValue("NULL"); + } else { + e.setValue(s); + } + } + } + } + + List> expe = cleanValues(expectedTableData.getValue()); + for (Map row : expe) { + for (Map.Entry e : row.entrySet()) { + if (e.getValue() != null && e.getValue() instanceof String) { + String s = ((String) e.getValue()).replaceAll("\\s+$", ""); + if (s.matches("^-?\\d+\\.0$")) { + s = s.substring(0, s.length() - 2); + } + if (s.isEmpty()) { + e.setValue("NULL"); + } else { + e.setValue(s); + } + } + } + } + + try { + org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(expe); + } catch (AssertionError e) { + LOG.error("Assertion failed for table: " + tableName, e); + throw e; + } + } + } + + private List> createRows(String columnName, boolean isPk, Object... values) { + List vals = Arrays.asList(values); + List> rows = new ArrayList<>(vals.size()); + for (int i = 0; i < vals.size(); i++) { + if (vals.get(i) == null && isPk) { + continue; + } + Map row = new HashMap<>(); + if (isPk) { + row.put("DUMMY_COL", "X"); + } else { + row.put("ID", BigDecimal.valueOf(i + 1)); + } + row.put(columnName, vals.get(i)); + rows.add(row); + } + return rows; + } + + private List> cleanValues(List> rows) { + for (Map row : rows) { + for (Map.Entry entry : row.entrySet()) { + if (entry.getValue() == null) { + entry.setValue("NULL"); + } else if (entry.getValue() instanceof byte[]) { + entry.setValue(Base64.getEncoder().encodeToString((byte[]) entry.getValue())); + } else if (entry.getValue() instanceof java.sql.Timestamp) { + entry.setValue(entry.getValue().toString()); + } else if (entry.getValue() instanceof java.sql.Clob) { + try { + java.sql.Clob c = (java.sql.Clob) entry.getValue(); + entry.setValue(c.getSubString(1, (int) c.length())); + } catch (Exception ex) { + entry.setValue(entry.getValue().toString()); + } + } else if (entry.getValue() instanceof java.lang.Number) { + entry.setValue( + new java.math.BigDecimal(entry.getValue().toString()) + .stripTrailingZeros() + .toPlainString()); + } else { + entry.setValue(entry.getValue().toString()); + } + } + } + return rows; + } + + private Map> getSpannerTableData() { + Map> spMap = new HashMap<>(); + spMap.put( + "VARCHAR_TO_VARCHAR2_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_VARCHAR2_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_VARCHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_VARCHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_CHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_CHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_CHARACTER_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_CHARACTER_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NCHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NCHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NCHAR_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NCHAR_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHAR_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHAR_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHAR_VARYING_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NATIONAL_CHAR_VARYING_PK_TABLE", + Arrays.asList(Value.string("A"), Value.string("B"), Value.string("C"))); + spMap.put( + "DOUBLE_PRECISION_TO_NUMBER_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "NUMERIC_TO_NUMBER_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_NUMBER_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_NUMBER_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_NUMERIC_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "DOUBLE_PRECISION_TO_NUMERIC_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_NUMERIC_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_NUMERIC_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_DECIMAL_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "DOUBLE_PRECISION_TO_DECIMAL_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_DECIMAL_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_DECIMAL_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_DEC_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "DOUBLE_PRECISION_TO_DEC_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_DEC_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_DEC_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "DOUBLE_PRECISION_TO_FLOAT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "NUMERIC_TO_FLOAT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_FLOAT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "DOUBLE_PRECISION_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "NUMERIC_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_DOUBLE_PRECISION_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "DOUBLE_PRECISION_TO_REAL_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "NUMERIC_TO_REAL_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_REAL_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "BIGINT_TO_REAL_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "REAL_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.float32(1.0f), Value.float32(0.0f), Value.float32(-1.0f), Value.float32(null))); + spMap.put( + "DOUBLE_PRECISION_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "NUMERIC_TO_BINARY_FLOAT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "DOUBLE_PRECISION_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "NUMERIC_TO_BINARY_DOUBLE_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "BIGINT_TO_INTEGER_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "BIGINT_TO_INTEGER_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_INTEGER_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_INTEGER_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "DOUBLE_PRECISION_TO_INTEGER_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "BIGINT_TO_INT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "BIGINT_TO_INT_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_INT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_INT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "DOUBLE_PRECISION_TO_INT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "BIGINT_TO_SMALLINT_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "BIGINT_TO_SMALLINT_PK_TABLE", + Arrays.asList(Value.int64(100L), Value.int64(-100L), Value.int64(0L), Value.int64(null))); + spMap.put( + "NUMERIC_TO_SMALLINT_TABLE", + Arrays.asList( + Value.numeric(new BigDecimal("1")), + Value.numeric(new BigDecimal("0")), + Value.numeric(new BigDecimal("-1")), + Value.numeric(null))); + spMap.put( + "VARCHAR_TO_SMALLINT_TABLE", + Arrays.asList( + Value.string("1"), Value.string("0"), Value.string("-1"), Value.string(null))); + spMap.put( + "DOUBLE_PRECISION_TO_SMALLINT_TABLE", + Arrays.asList( + Value.float64(1.0), Value.float64(0.0), Value.float64(-1.0), Value.float64(null))); + spMap.put( + "VARCHAR_TO_CLOB_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "VARCHAR_TO_NCLOB_TABLE", + Arrays.asList( + Value.string(""), Value.string(" "), Value.string("A"), Value.string("DROP TABLE"))); + spMap.put( + "BOOLEAN_TO_BOOLEAN_TABLE", + Arrays.asList(Value.bool(true), Value.bool(false), Value.bool(null))); + spMap.put( + "BOOLEAN_TO_BOOLEAN_PK_TABLE", + Arrays.asList(Value.bool(true), Value.bool(false), Value.bool(null))); + spMap.put( + "BIGINT_TO_BOOLEAN_TABLE", + Arrays.asList(Value.int64(1L), Value.int64(0L), Value.int64(null))); + spMap.put( + "VARCHAR_TO_BOOLEAN_TABLE", + Arrays.asList(Value.string("1"), Value.string("0"), Value.string(null))); + return spMap; + } + + private Map>> getExpectedData() { + Map>> orMap = new HashMap<>(); + { + String col = "varchar2_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_VARCHAR2_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "varchar2_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_VARCHAR2_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "varchar_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_VARCHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "varchar_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_VARCHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "char_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_CHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "char_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_CHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "character_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_CHARACTER_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "character_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_CHARACTER_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "nchar_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_NCHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nchar_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_NCHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "nchar_varying_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NCHAR_VARYING_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nchar_varying_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_NCHAR_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_character_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_character_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_NATIONAL_CHARACTER_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_char_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NATIONAL_CHAR_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_char_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_NATIONAL_CHAR_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_character_varying_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_TABLE", + createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_character_varying_pk_col"; + boolean isPk = true; + orMap.put( + "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "national_char_varying_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NATIONAL_CHAR_VARYING_TABLE", + createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "national_char_varying_pk_col"; + boolean isPk = true; + orMap.put("VARCHAR_TO_NATIONAL_CHAR_VARYING_PK_TABLE", createRows(col, isPk, "A", "B", "C")); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "number_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_NUMBER_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "numeric_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_NUMERIC_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "decimal_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_DECIMAL_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "dec_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_DEC_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "float_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "double_precision_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_DOUBLE_PRECISION_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "real_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_REAL_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "REAL_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_float_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_BINARY_FLOAT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "binary_double_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_BINARY_DOUBLE_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "integer_pk_col"; + boolean isPk = true; + orMap.put( + "BIGINT_TO_INTEGER_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "integer_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_INTEGER_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "int_pk_col"; + boolean isPk = true; + orMap.put( + "BIGINT_TO_INT_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "int_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_INT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "smallint_pk_col"; + boolean isPk = true; + orMap.put( + "BIGINT_TO_SMALLINT_PK_TABLE", + createRows( + col, isPk, new BigDecimal("100"), new BigDecimal("-100"), new BigDecimal("0"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "NUMERIC_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "smallint_col"; + boolean isPk = false; + orMap.put( + "DOUBLE_PRECISION_TO_SMALLINT_TABLE", + createRows( + col, isPk, new BigDecimal("1"), new BigDecimal("0"), new BigDecimal("-1"), null)); + } + { + String col = "clob_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_CLOB_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "nclob_col"; + boolean isPk = false; + orMap.put("VARCHAR_TO_NCLOB_TABLE", createRows(col, isPk, "", " ", "A", "DROP TABLE")); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "BOOLEAN_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_pk_col"; + boolean isPk = true; + orMap.put( + "BOOLEAN_TO_BOOLEAN_PK_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "BIGINT_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + { + String col = "boolean_col"; + boolean isPk = false; + orMap.put( + "VARCHAR_TO_BOOLEAN_TABLE", + createRows(col, isPk, new BigDecimal("1"), new BigDecimal("0"), null)); + } + return orMap; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDbCustomTransformationIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDbCustomTransformationIT.java new file mode 100644 index 0000000000..ce23f9cbf1 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleDbCustomTransformationIT.java @@ -0,0 +1,391 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.ByteArray; +import com.google.cloud.Date; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Value; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleDbCustomTransformationIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleDbCustomTransformationIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleDbCustomTransformationIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleDbCustomTransformationIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleDbCustomTransformationIT/oracle-schema.sql"; + + private static final String TABLE = "Users1"; + private static final String TABLE2 = "AllDatatypeTransformation"; + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (SpannerToOracleDbCustomTransformationIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleDbCustomTransformationIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleSchema( + jdbcResourceManager, + SpannerToOracleDbCustomTransformationIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + createAndUploadJarToGcs(gcsResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + CustomTransformation customTransformation = + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationWithOracleForIT") + .build(); + createAndUploadJarToGcs(gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + customTransformation, + "oracle", + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleDbCustomTransformationIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testCustomTransformation() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + writeRowInSpanner(); + assertRowInOracle(); + } + + private void writeRowInSpanner() { + Mutation m = + Mutation.newInsertOrUpdateBuilder("Users1").set("id").to(1).set("name").to("AA BB").build(); + spannerResourceManager.write(m); + m = + Mutation.newInsertOrUpdateBuilder("AllDatatypeTransformation") + .set("varchar_column") + .to("example2") + .set("bigint_column") + .to(1000) + .set("binary_column") + .to(Value.bytes(ByteArray.copyFrom("bin_column"))) + .set("bit_column") + .to(Value.bytes(ByteArray.copyFrom("1"))) + .set("blob_column") + .to(Value.bytes(ByteArray.copyFrom("blob_column"))) + .set("bool_column") + .to(Value.bool(Boolean.TRUE)) + .set("date_column") + .to(Value.date(Date.fromYearMonthDay(2024, 01, 01))) + .set("datetime_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("decimal_column") + .to(new BigDecimal("99999.99")) + .set("double_column") + .to(123456.123) + .set("enum_column") + .to("1") + .set("float_column") + .to(12345.67) + .set("int_column") + .to(100) + .set("text_column") + .to("Sample text for entry 2") + .set("time_column") + .to("14:30:00") + .set("timestamp_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("tinyint_column") + .to(2) + .set("year_column") + .to("2024") + .build(); + spannerResourceManager.write(m); + m = + Mutation.newUpdateBuilder("AllDatatypeTransformation") + .set("varchar_column") + .to("example2") + .set("bigint_column") + .to(1000) + .set("binary_column") + .to(Value.bytes(ByteArray.copyFrom("bin_column"))) + .set("bit_column") + .to(Value.bytes(ByteArray.copyFrom("1"))) + .set("blob_column") + .to(Value.bytes(ByteArray.copyFrom("blob_column"))) + .set("bool_column") + .to(Value.bool(Boolean.TRUE)) + .set("date_column") + .to(Value.date(Date.fromYearMonthDay(2024, 01, 01))) + .set("datetime_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("decimal_column") + .to(new BigDecimal("99999.99")) + .set("double_column") + .to(123456.123) + .set("enum_column") + .to("1") + .set("float_column") + .to(12345.67) + .set("int_column") + .to(100) + .set("text_column") + .to("Sample text for entry 2") + .set("time_column") + .to("14:30:00") + .set("timestamp_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("tinyint_column") + .to(2) + .set("year_column") + .to("2024") + .build(); + spannerResourceManager.write(m); + m = Mutation.delete("AllDatatypeTransformation", Key.of("example2")); + spannerResourceManager.write(m); + m = + Mutation.newInsertBuilder("AllDatatypeTransformation") + .set("varchar_column") + .to("example1") + .set("bigint_column") + .to(1000) + .set("binary_column") + .to(Value.bytes(ByteArray.copyFrom("examplebinary1"))) + .set("bit_column") + .to(Value.bytes(ByteArray.copyFrom("1"))) + .set("blob_column") + .to(Value.bytes(ByteArray.copyFrom("exampleblob1"))) + .set("bool_column") + .to(Value.bool(Boolean.TRUE)) + .set("date_column") + .to(Value.date(Date.fromYearMonthDay(2024, 01, 01))) + .set("datetime_column") + .to(Timestamp.parseTimestamp("2024-01-01T12:34:56Z")) + .set("decimal_column") + .to(new BigDecimal("99999.99")) + .set("double_column") + .to(123456.123) + .set("enum_column") + .to("1") + .set("float_column") + .to(12345.67) + .set("int_column") + .to(100) + .set("text_column") + .to("Sample text for entry 1") + .set("time_column") + .to("14:30:00") + .set("timestamp_column") + .to(Timestamp.parseTimestamp("2024-01-01T12:34:56Z")) + .set("tinyint_column") + .to(1) + .set("year_column") + .to("2024") + .build(); + spannerResourceManager.write(m); + m = + Mutation.newInsertBuilder("AllDatatypeTransformation") + .set("varchar_column") + .to("example") + .set("bigint_column") + .to(12345) + .set("binary_column") + .to(Value.bytes(ByteArray.copyFrom("Some binary data"))) + .set("bit_column") + .to(Value.bytes(ByteArray.copyFrom("1"))) + .set("blob_column") + .to(Value.bytes(ByteArray.copyFrom("Some blob data"))) + .set("bool_column") + .to(Value.bool(Boolean.TRUE)) + .set("date_column") + .to(Value.date(Date.fromYearMonthDay(2024, 01, 01))) + .set("datetime_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("decimal_column") + .to(new BigDecimal("12345.67")) + .set("double_column") + .to(123.456) + .set("enum_column") + .to("1") + .set("float_column") + .to(123.45) + .set("int_column") + .to(123) + .set("text_column") + .to("Sample text") + .set("time_column") + .to("14:30:00") + .set("timestamp_column") + .to(Value.timestamp(Timestamp.parseTimestamp("2024-01-01T12:34:56Z"))) + .set("tinyint_column") + .to(1) + .set("year_column") + .to("2024") + .build(); + spannerResourceManager.write(m); + } + + private String readClob(Object clobObj) throws Exception { + if (clobObj == null) { + return null; + } + if (clobObj instanceof java.sql.Clob) { + java.sql.Clob clob = (java.sql.Clob) clobObj; + return clob.getSubString(1, (int) clob.length()); + } + return clobObj.toString(); + } + + private void assertRowInOracle() throws Exception { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + () -> + runIsolatedGetRowCount(jdbcResourceManager, testUsername, "\"" + TABLE + "\"") + == 1); + + assertThatResult(result).meetsConditions(); + + result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + () -> + runIsolatedGetRowCount(jdbcResourceManager, testUsername, "\"" + TABLE2 + "\"") + == 2); + + assertThatResult(result).meetsConditions(); + + List> rows = + runIsolatedReadTable(jdbcResourceManager, testUsername, "\"" + TABLE + "\""); + assertThat(rows).hasSize(1); + assertThat(((Number) rows.get(0).get("id")).longValue()).isEqualTo(1L); + assertThat(rows.get(0).get("first_name")).isEqualTo("AA"); + assertThat(rows.get(0).get("last_name")).isEqualTo("BB"); + + rows = + jdbcResourceManager.runSQLQuery( + String.format("select * from \"%s\" order by \"%s\"", TABLE2, "varchar_column")); + assertThat(rows).hasSize(2); + assertThat(rows.get(1).get("varchar_column")).isEqualTo("example2"); + assertThat(((Number) rows.get(1).get("bigint_column")).longValue()).isEqualTo(1001L); + assertThat(((Number) rows.get(1).get("int_column")).intValue()).isEqualTo(101); + assertThat(readClob(rows.get(1).get("text_column"))) + .isEqualTo("Sample text for entry 2 append"); + + assertThat(rows.get(0).get("varchar_column")).isEqualTo("example"); + assertThat(((Number) rows.get(0).get("bigint_column")).longValue()).isEqualTo(12346L); + assertThat(((Number) rows.get(0).get("int_column")).intValue()).isEqualTo(124); + assertThat(readClob(rows.get(0).get("text_column"))).isEqualTo("Sample text append"); + + rows = + jdbcResourceManager.runSQLQuery( + String.format( + "select * from \"%s\" where \"%s\" like '%s'", + TABLE2, "varchar_column", "example1")); + assertThat(rows).hasSize(0); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleFileOverridesSchemaMapperIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleFileOverridesSchemaMapperIT.java new file mode 100644 index 0000000000..6dcdd11393 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleFileOverridesSchemaMapperIT.java @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for SpannerToSourceDb Flex template using file-based schema overrides for + * Oracle. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleFileOverridesSchemaMapperIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleFileOverridesSchemaMapperIT.class); + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager oracleResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-schema.sql"; + private static final String SCHEMA_OVERRIDE_FILE_RESOURCE = + "oracle/SpannerToOracleFileOverridesSchemaMapperIT/file-overrides.json"; + private static final String SCHEMA_OVERRIDE_GCS_PREFIX = "SpannerToOracleFileOverridesIT"; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToOracleFileOverridesSchemaMapperIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = createSpannerDatabase(SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + oracleResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(oracleResourceManager); + createOracleSchema(oracleResourceManager, ORACLE_SCHEMA_FILE_RESOURCE, testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, oracleResourceManager); + + gcsResourceManager.uploadArtifact( + SCHEMA_OVERRIDE_GCS_PREFIX + "/file-overrides.json", + Resources.getResource(SCHEMA_OVERRIDE_FILE_RESOURCE).getPath()); + + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + + Map jobParameters = new HashMap<>(); + jobParameters.put( + "schemaOverridesFilePath", + getGcsPath(SCHEMA_OVERRIDE_GCS_PREFIX + "/file-overrides.json", gcsResourceManager)); + + // For Oracle, we might need jdbcDriverJars, let's see if base class does this or if we add + // it. + // SKILL.md mentions doing it if needed. For now assuming not needed unless it fails or we + // find it. + // wait, Oracle uses proprietary driver usually, let me add a dummy check or leave it for + // compilation to complain. + + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + "oracle", // source type + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleFileOverridesSchemaMapperIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testSpannerToOracleWithFileOverrides() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + // Insert data into Spanner tables matching the override scenario + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("Target_Table_1") + .set("id_col1") + .to(1) + .set("Target_Name_Col_1") + .to("Name One") + .set("data_col1") + .to("Data for one") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("Target_Table_1") + .set("id_col1") + .to(2) + .set("Target_Name_Col_1") + .to("Name Two") + .set("data_col1") + .to("Data for two") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("source_table2") + .set("key_col2") + .to("K1") + .set("Target_Category_Col_2") + .to("Category Alpha") + .set("value_col2") + .to("Value Alpha") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("source_table2") + .set("key_col2") + .to("K2") + .set("Target_Category_Col_2") + .to("Category Beta") + .set("value_col2") + .to("Value Beta") + .build()); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> + (runIsolatedGetRowCount( + oracleResourceManager, testUsername, "\"source_table1\"") + == 2 + && runIsolatedGetRowCount( + oracleResourceManager, testUsername, "\"source_table2\"") + == 2)); + assertThatResult(result).meetsConditions(); + + // Assert Oracle table1 (should be source_table1, with column name_col1 renamed) + List> oracleTable1 = + runIsolatedSQLQuery( + oracleResourceManager, + testUsername, + "SELECT \"id_col1\", \"name_col1\", TO_CHAR(\"data_col1\") AS \"data_col1\" FROM \"source_table1\" ORDER BY \"id_col1\" ASC"); + assertThat(oracleTable1).hasSize(2); + // Note: Oracle might return BigDecimal for INT columns depending on JDBC mapping, using + // toString(). + assertThat(oracleTable1.get(0).get("id_col1").toString()).isEqualTo("1"); + assertThat(oracleTable1.get(0).get("name_col1")).isEqualTo("Name One"); + assertThat(oracleTable1.get(0).get("data_col1")).isEqualTo("Data for one"); + assertThat(oracleTable1.get(1).get("id_col1").toString()).isEqualTo("2"); + assertThat(oracleTable1.get(1).get("name_col1")).isEqualTo("Name Two"); + assertThat(oracleTable1.get(1).get("data_col1")).isEqualTo("Data for two"); + + // Assert Oracle table2 (should be source_table2, with column category_col2 renamed) + List> oracleTable2 = + runIsolatedSQLQuery( + oracleResourceManager, + testUsername, + "SELECT \"key_col2\", \"category_col2\", TO_CHAR(\"value_col2\") AS \"value_col2\" FROM \"source_table2\" ORDER BY \"key_col2\" ASC"); + assertThat(oracleTable2).hasSize(2); + assertThat(oracleTable2.get(0).get("key_col2")).isEqualTo("K1"); + assertThat(oracleTable2.get(0).get("category_col2")).isEqualTo("Category Alpha"); + assertThat(oracleTable2.get(0).get("value_col2")).isEqualTo("Value Alpha"); + assertThat(oracleTable2.get(1).get("key_col2")).isEqualTo("K2"); + assertThat(oracleTable2.get(1).get("category_col2")).isEqualTo("Category Beta"); + assertThat(oracleTable2.get(1).get("value_col2")).isEqualTo("Value Beta"); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleIT.java new file mode 100644 index 0000000000..aeeef60ebf --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleIT.java @@ -0,0 +1,524 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.spanner.migrations.constants.Constants.ORACLE_SOURCE_TYPE; +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Options; +import com.google.cloud.spanner.TransactionRunner.TransactionCallable; +import com.google.cloud.spanner.Value; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.apache.beam.sdk.io.gcp.spanner.SpannerAccessor; +import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for {@link SpannerToSourceDb} Flex template for basic run including new spanner + * tables and column rename use-case. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleIT extends SpannerToSourceDbITBase { + private static final Logger LOG = LoggerFactory.getLogger(SpannerToOracleIT.class); + // Test timeout configuration - can be adjusted if tests need more time + private static final Duration TEST_TIMEOUT = Duration.ofMinutes(15); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = "oracle/SpannerToOracleIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleIT/oracle-schema.sql"; + private static final String TABLE = "Users"; + private static final String TABLE_WITH_VIRTUAL_GEN_COL = "TableWithVirtualGeneratedColumn"; + private static final String TABLE_WITH_STORED_GEN_COL = "TableWithStoredGeneratedColumn"; + private static final String TABLE_WITH_IDENTITY_COL = "TableWithIdentityColumn"; + private static final String BOUNDARY_CHECK_TABLE = + "testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO"; + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + private static String classLevelTestUsername = null; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + try { + Class.forName("oracle.jdbc.OracleDriver"); + } catch (Exception e) { + LOG.warn("Failed to manually register Oracle driver", e); + } + skipBaseCleanup = true; + synchronized (SpannerToOracleIT.class) { + testInstances.add(this); + if (classLevelTestUsername == null) { + classLevelTestUsername = + setupOracleIsolatedUser(SharedOracleReverseITContainer.getInstance()); + } + this.testUsername = classLevelTestUsername; + if (jobInfo == null) { + spannerResourceManager = createSpannerDatabase(SpannerToOracleIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + + createOracleSchema( + jdbcResourceManager, SpannerToOracleIT.ORACLE_SCHEMA_FILE_RESOURCE, testUsername); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + classLevelTestUsername = null; + } + + @Test + @Ignore("Skipping spannerToSourceDbBasic test") + public void spannerToSourceDbBasic() throws InterruptedException, IOException { + assertThatPipeline(jobInfo).isRunning(); + // Write row in Spanner + writeRowInSpanner(); + // Assert events on Oracle + assertRowInOracle(); + } + + private void writeRowInSpanner() { + // Write a single record to Spanner + Mutation m1 = + Mutation.newInsertOrUpdateBuilder("Users") + .set("id") + .to(1) + .set("full_name") + .to("FF") + .set("from") + .to("AA") + .build(); + spannerResourceManager.write(m1); + Mutation m2 = + Mutation.newInsertOrUpdateBuilder("Users2").set("id").to(2).set("name").to("B").build(); + spannerResourceManager.write(m2); + // Write a single record to Spanner for the given logical shard + // Add the record with the transaction tag as txBy= + SpannerConfig spannerConfig = + SpannerConfig.create() + .withProjectId(PROJECT) + .withInstanceId(spannerResourceManager.getInstanceId()) + .withDatabaseId(spannerResourceManager.getDatabaseId()); + SpannerAccessor spannerAccessor = SpannerAccessor.getOrCreate(spannerConfig); + spannerAccessor + .getDatabaseClient() + .readWriteTransaction( + Options.tag("txBy=forwardMigration"), + Options.priority(spannerConfig.getRpcPriority().get())) + .run( + (TransactionCallable) + transaction -> { + Mutation m3 = + Mutation.newInsertOrUpdateBuilder("Users") + .set("id") + .to(2) + .set("full_name") + .to("GG") + .build(); + transaction.buffer(m3); + return null; + }); + } + + private void assertRowInOracle() throws InterruptedException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + () -> + runIsolatedGetRowCount(jdbcResourceManager, testUsername, "\"" + TABLE + "\"") + == 1); // only one row is inserted + assertThatResult(result).meetsConditions(); + List> rows = + runIsolatedReadTable(jdbcResourceManager, testUsername, "\"" + TABLE + "\""); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).get("id")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("name")).isEqualTo("FF"); + assertThat(rows.get(0).get("from")).isEqualTo("AA"); + } + + @Test + public void spannerToSourceDbWithGeneratedColumns() { + assertThatPipeline(jobInfo).isRunning(); + // INSERT + writeRowsWithGenColInSpanner(); + assertThatPipeline(jobInfo).isRunning(); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + () -> + (runIsolatedGetRowCount( + jdbcResourceManager, + testUsername, + "\"" + TABLE_WITH_STORED_GEN_COL + "\"") + == 2) + && (runIsolatedGetRowCount( + jdbcResourceManager, + testUsername, + "\"" + TABLE_WITH_VIRTUAL_GEN_COL + "\"") + == 2)); // only two rows is inserted + assertGenColRowsInOracleAfterInsert(result); + updateRowsWithGenColsInSpanner(); + result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), this::checkGenColRowsInOracleAfterUpdate); + // Delete rows in spanner. + deleteGenColRowsInSpanner(); + PipelineOperator.Result deleteResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + () -> allGenColRowsDeleted()); // all rows should be deleted + assertThatResult(deleteResult).meetsConditions(); + } + + @Test + public void spannerToOracleSourceDbMaxColAndTableNameTest() + throws IOException, InterruptedException { + assertThatPipeline(jobInfo).isRunning(); + // Write row in Spanner + writeMaxColRowsInSpanner(); + // Assert events on Oracle + assertBoundaryRowInOracle(); + } + + @Test + public void spannerToSourceDbWithIdentityColumns() { + assertThatPipeline(jobInfo).isRunning(); + // INSERT + writeRowsWithIdentityColInSpanner(); + assertThatPipeline(jobInfo).isRunning(); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + () -> + runIsolatedGetRowCount( + jdbcResourceManager, + testUsername, + "\"" + TABLE_WITH_IDENTITY_COL + "\"") + == 2); + assertIdentityColRowsInOracleAfterInsert(result); + } + + @Test + public void spannerToOracleGeneratedColumns() { + LOG.info("Starting Spanner to Oracle Generated Columns IT"); + assertThatPipeline(jobInfo).isRunning(); + Map>> spannerTableData = new HashMap<>(); + OracleGeneratedColumnUtils.addInitialMultiColSpannerData(spannerTableData); + OracleGeneratedColumnUtils.writeRowsInSpanner(spannerTableData, spannerResourceManager); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + OracleGeneratedColumnUtils.buildConditionCheck( + spannerTableData, jdbcResourceManager, testUsername)); + Map>> expectedData = new HashMap<>(); + OracleGeneratedColumnUtils.addInitialGeneratedColumnData(expectedData); + // Assert events on Oracle + OracleGeneratedColumnUtils.assertRowInOracle(expectedData, jdbcResourceManager, testUsername); + // Validating update and delete events. + Map>> updateSpannerTableData = + OracleGeneratedColumnUtils.updateGeneratedColRowsInSpanner(spannerResourceManager); + spannerTableData.putAll(updateSpannerTableData); + result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + OracleGeneratedColumnUtils.buildConditionCheck( + spannerTableData, jdbcResourceManager, testUsername)); + expectedData = new HashMap<>(); + OracleGeneratedColumnUtils.addUpdatedGeneratedColumnData(expectedData); + OracleGeneratedColumnUtils.assertRowInOracle(expectedData, jdbcResourceManager, testUsername); + } + + private void writeMaxColRowsInSpanner() { + List mutations = new ArrayList<>(); + Mutation.WriteBuilder mutationBuilder = + Mutation.newInsertOrUpdateBuilder(BOUNDARY_CHECK_TABLE).set("id").to(1); + mutationBuilder + .set("col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY") + .to("SampleTestValue"); + mutations.add(mutationBuilder.build()); + spannerResourceManager.write(mutations); + LOG.info("Inserted row into Spanner using Mutations"); + } + + private void assertBoundaryRowInOracle() { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + () -> + runIsolatedGetRowCount( + jdbcResourceManager, testUsername, "\"" + BOUNDARY_CHECK_TABLE + "\"") + == 1); + } + + private void writeRowsWithGenColInSpanner() { + List mutations = new ArrayList<>(); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_STORED_GEN_COL) + .set("id") + .to(1) + .set("column1") + .to(1) + .build()); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_STORED_GEN_COL) + .set("id") + .to(2) + .set("column1") + .to(2) + .build()); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_VIRTUAL_GEN_COL) + .set("id") + .to(1) + .set("column1") + .to(1) + .build()); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_VIRTUAL_GEN_COL) + .set("id") + .to(2) + .set("column1") + .to(2) + .build()); + spannerResourceManager.write(mutations); + } + + private void assertGenColRowsInOracleAfterInsert(PipelineOperator.Result result) { + assertThatResult(result).meetsConditions(); + List> rows = + runIsolatedReadTable( + jdbcResourceManager, testUsername, "\"" + TABLE_WITH_VIRTUAL_GEN_COL + "\""); + assertThat(rows).hasSize(2); + assertThat(rows.get(0).get("id")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("column1")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("virtual_generated_column")) + .isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("id")).isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("column1")).isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("virtual_generated_column")) + .isEqualTo(new java.math.BigDecimal("4")); + rows = + runIsolatedReadTable( + jdbcResourceManager, testUsername, "\"" + TABLE_WITH_STORED_GEN_COL + "\""); + assertThat(rows).hasSize(2); + assertThat(rows.get(0).get("id")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("column1")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("stored_generated_column")).isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("id")).isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("column1")).isEqualTo(new java.math.BigDecimal("2")); + assertThat(rows.get(1).get("stored_generated_column")).isEqualTo(new java.math.BigDecimal("4")); + } + + private void updateRowsWithGenColsInSpanner() { + List mutations = new ArrayList<>(); + mutations.add( + Mutation.newUpdateBuilder(TABLE_WITH_STORED_GEN_COL) + .set("id") + .to(1) + .set("column1") + .to(3) + .build()); + mutations.add( + Mutation.newUpdateBuilder(TABLE_WITH_VIRTUAL_GEN_COL) + .set("id") + .to(1) + .set("column1") + .to(4) + .build()); + spannerResourceManager.write(mutations); + } + + private boolean checkGenColRowsInOracleAfterUpdate() { + List> rows = + runIsolatedSQLQuery( + jdbcResourceManager, + testUsername, + "select * from \"TableWithVirtualGeneratedColumn\" where \"id\"=1"); + if (rows.size() != 1) { + return false; + } + if (!rows.get(0).get("id").equals(new java.math.BigDecimal("1"))) { + return false; + } + if (!rows.get(0).get("column1").equals(new java.math.BigDecimal("4"))) { + return false; + } + rows = + runIsolatedSQLQuery( + jdbcResourceManager, + testUsername, + "select * from \"TableWithStoredGeneratedColumn\" where \"id\"=1"); + if (rows.size() != 1) { + return false; + } + if (!rows.get(0).get("id").equals(new java.math.BigDecimal("1"))) { + return false; + } + if (!rows.get(0).get("column1").equals(new java.math.BigDecimal("3"))) { + return false; + } + return true; + } + + private void deleteGenColRowsInSpanner() { + Mutation m1 = Mutation.delete(TABLE_WITH_VIRTUAL_GEN_COL, Key.newBuilder().append(1).build()); + spannerResourceManager.write(m1); + Mutation m2 = Mutation.delete(TABLE_WITH_VIRTUAL_GEN_COL, Key.newBuilder().append(2).build()); + spannerResourceManager.write(m2); + Mutation m3 = Mutation.delete(TABLE_WITH_STORED_GEN_COL, Key.newBuilder().append(1).build()); + spannerResourceManager.write(m3); + Mutation m4 = Mutation.delete(TABLE_WITH_STORED_GEN_COL, Key.newBuilder().append(2).build()); + spannerResourceManager.write(m4); + } + + private boolean allGenColRowsDeleted() { + long rowCountTable1 = + runIsolatedGetRowCount( + jdbcResourceManager, testUsername, "\"" + TABLE_WITH_STORED_GEN_COL + "\""); + long rowCountTable2 = + runIsolatedGetRowCount( + jdbcResourceManager, testUsername, "\"" + TABLE_WITH_VIRTUAL_GEN_COL + "\""); + return (rowCountTable1 == 0) && (rowCountTable2 == 0); + } + + private void writeRowsWithIdentityColInSpanner() { + List mutations = new ArrayList<>(); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_IDENTITY_COL) + .set("id") + .to(1) + .set("column1") + .to("id1") + .build()); + mutations.add( + Mutation.newInsertBuilder(TABLE_WITH_IDENTITY_COL) + .set("id") + .to(2) + .set("column1") + .to("id2") + .build()); + spannerResourceManager.write(mutations); + } + + private void assertIdentityColRowsInOracleAfterInsert(PipelineOperator.Result result) { + assertThatResult(result).meetsConditions(); + List> rows = + runIsolatedReadTable( + jdbcResourceManager, testUsername, "\"" + TABLE_WITH_IDENTITY_COL + "\""); + assertThat(rows).hasSize(2); + assertThat(rows.get(0).get("id").toString()).isEqualTo("1"); + assertThat(rows.get(0).get("column1")).isEqualTo("id1"); + assertThat(rows.get(1).get("id").toString()).isEqualTo("2"); + assertThat(rows.get(1).get("column1")).isEqualTo("id2"); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleInterleaveMultiShardIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleInterleaveMultiShardIT.java new file mode 100644 index 0000000000..29928affe1 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleInterleaveMultiShardIT.java @@ -0,0 +1,427 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.spanner.migrations.constants.Constants.ORACLE_SOURCE_TYPE; +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for {@link SpannerToSourceDb} Flex template for multiple shards. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleInterleaveMultiShardIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleInterleaveMultiShardIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleInterleaveMultiShardIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURSE = + "oracle/SpannerToOracleInterleaveMultiShardIT/session.json"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/SpannerToOracleInterleaveMultiShardIT/oracle-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManagerShardA; + private static OracleResourceManager jdbcResourceManagerShardB; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToOracleInterleaveMultiShardIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleInterleaveMultiShardIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManagerShardA = SharedOracleReverseITContainer.getInstance(); + testUsernameShardA = setupOracleIsolatedUser(jdbcResourceManagerShardA); + createOracleSchema( + jdbcResourceManagerShardA, + SpannerToOracleInterleaveMultiShardIT.ORACLE_DDL_RESOURCE, + testUsernameShardA); + + jdbcResourceManagerShardB = SharedOracleReverseITContainer.getInstance(); + testUsernameShardB = setupOracleIsolatedUser(jdbcResourceManagerShardB); + createOracleSchema( + jdbcResourceManagerShardB, + SpannerToOracleInterleaveMultiShardIT.ORACLE_DDL_RESOURCE, + testUsernameShardB); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs( + gcsResourceManager, + Map.of("shardA", jdbcResourceManagerShardA, "shardB", jdbcResourceManagerShardB)); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURSE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + put("dlqRetryMinutes", "1"); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleInterleaveMultiShardIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToSourceFKTest() throws IOException, InterruptedException { + assertThatPipeline(jobInfo).isRunning(); + + doInsertsInSpanner(); + assertInsertedRowsInOracle(); + + doUpdatesInSpanner(); + assertUpdatedRowsInOracle(); + + doDeletesInSpanner(); + assertDeletedRowsInOracle(); + } + + private void doInsertsInSpanner() { + // Insert records + List mutations = new ArrayList<>(); + Mutation p1 = + Mutation.newInsertOrUpdateBuilder("parent1") + .set("id") + .to(1) + .set("migration_shard_id") + .to("shardA") + .build(); + spannerResourceManager.write(p1); + + Mutation p2 = + Mutation.newInsertOrUpdateBuilder("parent2") + .set("id") + .to(2) + .set("migration_shard_id") + .to("shardB") + .build(); + spannerResourceManager.write(p2); + + Mutation c1 = + Mutation.newInsertOrUpdateBuilder("child11") + .set("child_id") + .to(11) + .set("parent_id") + .to(1) + .set("migration_shard_id") + .to("shardA") + .build(); + Mutation c2 = + Mutation.newInsertOrUpdateBuilder("child21") + .set("child_id") + .to(22) + .set("id") + .to(2) + .set("migration_shard_id") + .to("shardB") + .build(); + Mutation c3 = + Mutation.newInsertOrUpdateBuilder("child31") + .set("child_id") + .to(33) + .set("id") + .to(2) + .set("migration_shard_id") + .to("shardB") + .build(); + mutations.add(c1); + mutations.add(c2); + mutations.add(c3); + spannerResourceManager.write(mutations); + } + + private void assertInsertedRowsInOracle() throws InterruptedException { + PipelineOperator.Result parent1Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"parent1\"") + == 1); + assertThatResult(parent1Result).meetsConditions(); + + PipelineOperator.Result child1Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"child11\"") + == 1); + assertThatResult(child1Result).meetsConditions(); + + PipelineOperator.Result parent2Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"parent2\"") + == 1); + assertThatResult(parent2Result).meetsConditions(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"child21\"") + == 1); + assertThatResult(result).meetsConditions(); + + PipelineOperator.Result result2 = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"child31\"") + == 1); + assertThatResult(result2).meetsConditions(); + + List> rows = + runIsolatedReadTable(jdbcResourceManagerShardA, testUsernameShardA, "\"parent1\""); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).get("id")).isEqualTo(new java.math.BigDecimal("1")); + + List> rows1 = + runIsolatedReadTable(jdbcResourceManagerShardB, testUsernameShardB, "\"parent2\""); + assertThat(rows1).hasSize(1); + assertThat(rows1.get(0).get("id")).isEqualTo(new java.math.BigDecimal("2")); + + List> rows2 = + runIsolatedReadTable(jdbcResourceManagerShardA, testUsernameShardA, "\"child11\""); + assertThat(rows2).hasSize(1); + assertThat(rows2.get(0).get("child_id")).isEqualTo(new java.math.BigDecimal("11")); + + List> rows3 = + runIsolatedReadTable(jdbcResourceManagerShardB, testUsernameShardB, "\"child21\""); + assertThat(rows3).hasSize(1); + assertThat(rows3.get(0).get("child_id")).isEqualTo(new java.math.BigDecimal("22")); + + List> rows4 = + runIsolatedReadTable(jdbcResourceManagerShardB, testUsernameShardB, "\"child31\""); + assertThat(rows4).hasSize(1); + assertThat(rows4.get(0).get("child_id")).isEqualTo(new java.math.BigDecimal("33")); + } + + private void doUpdatesInSpanner() { + List mutations = new ArrayList<>(); + Mutation p1 = + Mutation.newUpdateBuilder("parent1") + .set("id") + .to(1) + .set("migration_shard_id") + .to("shardA") + .set("update_ts") + .to(Timestamp.parseTimestamp("1980-01-01T00:00:00Z")) + .build(); + Mutation c1 = + Mutation.newUpdateBuilder("child11") + .set("child_id") + .to(11) + .set("parent_id") + .to(1) + .set("migration_shard_id") + .to("shardA") + .set("update_ts") + .to(Timestamp.parseTimestamp("1980-01-01T00:00:00Z")) + .build(); + // This extra insert will help us in validation + Mutation c2 = + Mutation.newInsertOrUpdateBuilder("child11") + .set("child_id") + .to(12) + .set("parent_id") + .to(1) + .set("migration_shard_id") + .to("shardA") + .build(); + mutations.add(p1); + mutations.add(c1); + mutations.add(c2); + spannerResourceManager.write(mutations); + } + + private void assertUpdatedRowsInOracle() throws InterruptedException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"child11\"") + == 2); + assertThatResult(result).meetsConditions(); + + List> rows = + runIsolatedReadTable(jdbcResourceManagerShardA, testUsernameShardA, "\"parent1\""); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).get("id")).isEqualTo(new java.math.BigDecimal("1")); + assertThat(rows.get(0).get("update_ts").toString()).isEqualTo("1980-01-01 00:00:00.0"); + + List> rows2 = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"child_id\",\"update_ts\" FROM \"child11\" ORDER BY \"child_id\""); + assertThat(rows2).hasSize(2); + assertThat(rows2.get(0).get("child_id")).isEqualTo(new java.math.BigDecimal("11")); + assertThat(rows.get(0).get("update_ts").toString()).isEqualTo("1980-01-01 00:00:00.0"); + } + + private void doDeletesInSpanner() { + // Delete records + List mutations = new ArrayList<>(); + Mutation c1 = Mutation.delete("child11", Key.of(11)); + Mutation c2 = Mutation.delete("child11", Key.of(12)); + Mutation p1 = Mutation.delete("parent1", Key.of(1)); + Mutation p2 = Mutation.delete("parent2", Key.of(2)); + mutations.add(c1); + mutations.add(c2); + mutations.add(p1); + mutations.add(p2); // this should cause child22 delete as well + spannerResourceManager.write(mutations); + } + + private void assertDeletedRowsInOracle() throws InterruptedException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"parent2\"") + == 0); + assertThatResult(result).meetsConditions(); + + PipelineOperator.Result parent1Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"parent1\"") + == 0); + assertThatResult(parent1Result).meetsConditions(); + PipelineOperator.Result child1Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofSeconds(1)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardA, testUsernameShardA, "\"child11\"") + == 0); + assertThatResult(child1Result).meetsConditions(); + PipelineOperator.Result child2Result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofSeconds(1)), + () -> + runIsolatedGetRowCount( + jdbcResourceManagerShardB, testUsernameShardB, "\"child22\"") + == 0); + assertThatResult(child2Result).meetsConditions(); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleReservedKeywordsIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleReservedKeywordsIT.java new file mode 100644 index 0000000000..615b3f7206 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleReservedKeywordsIT.java @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * An integration test for {@link SpannerToSourceDb} Flex template which tests a basic migration on + * a simple schema with reserved keywords, targeting Oracle. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleReservedKeywordsIT extends SpannerToSourceDbITBase { + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleReservedKeywordsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/SpannerToOracleReservedKeywordsIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleReservedKeywordsIT/session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager oracleResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + @Before + public void setUp() throws IOException { + spannerResourceManager = createSpannerDatabase(SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + oracleResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(oracleResourceManager); + createOracleSchema(oracleResourceManager, ORACLE_DDL_RESOURCE, testUsername); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, oracleResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + "oracle", + jobParameters); + } + + @AfterClass + public static void cleanUp() throws IOException { + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testSpannerToOracleReservedKeywords() throws InterruptedException { + assertThatPipeline(jobInfo).isRunning(); + spannerResourceManager.write(generateData()); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> runIsolatedGetRowCount(oracleResourceManager, testUsername, "\"true\"") == 2); + assertThatResult(result).meetsConditions(); + + List> actualData = + runIsolatedReadTable(oracleResourceManager, testUsername, "\"true\""); + for (Map row : actualData) { + if (row.get("COLUMN") instanceof Number) { + row.put("COLUMN", ((Number) row.get("COLUMN")).longValue()); + } + } + List> expectedData = getExpectedOracleRows(); + + // Sort both lists by the primary key for deterministic comparison + actualData.sort(Comparator.comparing(m -> ((Number) m.get("COLUMN")).longValue())); + expectedData.sort(Comparator.comparing(m -> ((Number) m.get("COLUMN")).longValue())); + + assertThat(actualData).isEqualTo(expectedData); + } + + private List generateData() { + List mutations = new ArrayList<>(); + mutations.add( + Mutation.newInsertOrUpdateBuilder("true") + .set("COLUMN") + .to(1) + .set("TABLE") + .to("value1") + .set("WITH") + .to("value1") + .build()); + mutations.add( + Mutation.newInsertOrUpdateBuilder("true") + .set("COLUMN") + .to(2) + .set("TABLE") + .to("value2") + .set("WITH") + .to("value2") + .build()); + return mutations; + } + + private List> getExpectedOracleRows() { + List> rows = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("COLUMN", 1L); + row1.put("TABLE", "value1"); + row1.put("WITH", "value1"); + rows.add(row1); + Map row2 = new HashMap<>(); + row2.put("COLUMN", 2L); + row2.put("TABLE", "value2"); + row2.put("WITH", "value2"); + rows.add(row2); + return rows; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRow10MbIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRow10MbIT.java new file mode 100644 index 0000000000..5d75eae04b --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRow10MbIT.java @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.ByteArray; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.junit.runners.model.MultipleFailureException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for {@link SpannerToSourceDb} Flex template for column of size 10MB into Oracle. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleSourceDbWideRow10MbIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleSourceDbWideRow10MbIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-col-mb-session.json"; + private static final String TABLE1 = "large_data"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-10mb-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToOracleSourceDbWideRow10MbIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleSourceDbWideRow10MbIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + createOracleSchema( + jdbcResourceManager, + SpannerToOracleSourceDbWideRow10MbIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + + gcsResourceManager = + GcsResourceManager.builder(artifactBucketName, getClass().getSimpleName(), credentials) + .build(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager).replace("gs://" + artifactBucketName, ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + "oracle", + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleSourceDbWideRow10MbIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToOracleSourceDB10MBTest() + throws IOException, InterruptedException, MultipleFailureException { + assertThatPipeline(jobInfo).isRunning(); + // Write row in Spanner + writeBasicRowInSpanner(); + // Assert events on Oracle + assertRowInOracle(); + } + + private void writeBasicRowInSpanner() { + LOG.info("Writing a basic row to Spanner..."); + + final int maxBlobSize = 10 * 1024 * 1024; // 10MB + final int safeBlobSize = maxBlobSize - 1024; // 9.9MB to avoid limit issues + + try { + byte[] blobData = new byte[safeBlobSize]; + Mutation mutation = + Mutation.newInsertBuilder("large_data") + .set("id") + .to(UUID.randomUUID().toString()) + .set("large_blob") + .to(ByteArray.copyFrom(blobData)) // Ensures ≤10MB limit + .build(); + + spannerResourceManager.write(mutation); + LOG.info("✅ Successfully inserted a 9.9MB row into Spanner."); + } catch (Exception e) { + LOG.error("❌ Failed to insert BLOB in Spanner: {}", e.getMessage(), e); + } + } + + private final List assertionErrors = new ArrayList<>(); + + private void assertRowInOracle() throws MultipleFailureException { + LOG.info("Validating row in Oracle..."); + final int maxBlobSize = 10 * 1024 * 1024; // 10MB + final int safeBlobSize = maxBlobSize - 1024; // 9.9MB to avoid limit issues + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> { + try { + return runIsolatedGetRowCount(jdbcResourceManager, testUsername, TABLE1) == 1; + } catch (Exception e) { + LOG.error("Error while getting row count from Oracle", e); + return false; + } + }); + + assertThatResult(result).meetsConditions(); + + try { + List> rows = + runIsolatedReadTable(jdbcResourceManager, testUsername, TABLE1); + assertThat(rows).hasSize(1); + + Map row = rows.get(0); + Object idValue = null; + Object blobValue = null; + for (Map.Entry e : row.entrySet()) { + if ("id".equalsIgnoreCase(e.getKey())) { + idValue = e.getValue(); + } + if ("large_blob".equalsIgnoreCase(e.getKey())) { + blobValue = e.getValue(); + } + } + + assertThat(idValue).isNotNull(); + assertThat(idValue.toString()).isNotEmpty(); + + assertThat(blobValue).isNotNull(); + byte[] bytes; + if (blobValue instanceof java.sql.Blob) { + java.sql.Blob blob = (java.sql.Blob) blobValue; + bytes = blob.getBytes(1, (int) blob.length()); + } else { + bytes = (byte[]) blobValue; + } + assertThat(bytes.length).isEqualTo(safeBlobSize); + + } catch (Exception e) { + assertionErrors.add(new AssertionError("Oracle validation failed", e)); + } + + if (!assertionErrors.isEmpty()) { + throw new MultipleFailureException(assertionErrors); + } + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRowMaxColumnsIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRowMaxColumnsIT.java new file mode 100644 index 0000000000..b96e5b9ba0 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleSourceDbWideRowMaxColumnsIT.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.junit.runners.model.MultipleFailureException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for {@link SpannerToSourceDb} Flex template for max number of columns in Oracle. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleSourceDbWideRowMaxColumnsIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToOracleSourceDbWideRowMaxColumnsIT.class); + private static final String SESSION_FILE_RESOURCE = + "SpannerToSourceDbWideRowIT/max-col-session.json"; + private static final String TABLE1 = "testtable"; + private static final int NUM_NON_KEY_COLS = 100; + private static final String COLUMN_SIZE = "100"; + private static final String ORACLE_SOURCE_TYPE = "oracle"; + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws Exception + */ + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (SpannerToOracleSourceDbWideRowMaxColumnsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDBAndTableWithNColumns(TABLE1, NUM_NON_KEY_COLS, COLUMN_SIZE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleTableWithNColumns(jdbcResourceManager, TABLE1, NUM_NON_KEY_COLS, COLUMN_SIZE); + + gcsResourceManager = + GcsResourceManager.builder(artifactBucketName, getClass().getSimpleName(), credentials) + .build(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager).replace("gs://" + artifactBucketName, ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleSourceDbWideRowMaxColumnsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToOracleSourceDbMaxColTest() + throws IOException, InterruptedException, MultipleFailureException { + assertThatPipeline(jobInfo).isRunning(); + // Write row in Spanner + writeRowsInSpanner(); + // Assert events on Oracle + assertRowInOracle(); + } + + private void writeRowsInSpanner() { + List mutations = new ArrayList<>(); + Mutation.WriteBuilder mutationBuilder = + Mutation.newInsertOrUpdateBuilder(TABLE1).set("id").to("SampleTest"); + + for (int i = 1; i <= 100; i++) { + mutationBuilder.set("col_" + i).to("TestValue_" + i); + } + + mutations.add(mutationBuilder.build()); + spannerResourceManager.write(mutations); + LOG.info("Inserted row with 100 columns into Spanner using Mutations"); + } + + private final List assertionErrors = new ArrayList<>(); + + private void assertRowInOracle() throws MultipleFailureException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> + runIsolatedGetRowCount(jdbcResourceManager, testUsername, "\"" + TABLE1 + "\"") + == 1); + assertThatResult(result).meetsConditions(); + + List> rows = + runIsolatedReadTable(jdbcResourceManager, testUsername, "\"" + TABLE1 + "\""); + assertThat(rows).hasSize(1); + Map row = rows.get(0); + + Map lowerCaseRow = new HashMap<>(); + for (Map.Entry entry : row.entrySet()) { + lowerCaseRow.put(entry.getKey().toLowerCase(), entry.getValue()); + } + + for (int i = 1; i <= 100; i++) { + String columnName = "col_" + i; + String expectedValue = "TestValue_" + i; + + try { + assertThat(lowerCaseRow.get(columnName)).isEqualTo(expectedValue); + } catch (Throwable e) { + assertionErrors.add(e); + } + } + if (!assertionErrors.isEmpty()) { + throw new MultipleFailureException(assertionErrors); + } + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleTimezoneIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleTimezoneIT.java new file mode 100644 index 0000000000..0072841eb2 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleTimezoneIT.java @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.spanner.migrations.constants.Constants.ORACLE_SOURCE_TYPE; +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for checking the timezone conversion. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleTimezoneIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = LoggerFactory.getLogger(SpannerToOracleTimezoneIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToOracleTimezoneIT/session.json"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleTimezoneIT/oracle-schema.sql"; + + private static final String TABLE = "Users"; + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToOracleTimezoneIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleTimezoneIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleSchema( + jdbcResourceManager, + SpannerToOracleTimezoneIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + gcsResourceManager.uploadArtifact( + "input/session.json", + Resources.getResource(SpannerToOracleTimezoneIT.SESSION_FILE_RESOURCE).getPath()); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + "+10:00", + null, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleTimezoneIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void timezoneTest() throws IOException, InterruptedException { + assertThatPipeline(jobInfo).isRunning(); + // Write row in Spanner + writeRowInSpanner(); + // Assert events on Oracle + assertRowInOracle(); + } + + private void writeRowInSpanner() { + Mutation m = + Mutation.newInsertOrUpdateBuilder("Users") + .set("id") + .to(1) + .set("time_colm") + .to(Timestamp.parseTimestamp("2024-02-02T00:00:00Z")) + .build(); + spannerResourceManager.write(m); + Mutation m2 = + Mutation.newInsertOrUpdateBuilder("Users") + .set("id") + .to(2) + .set("time_colm") + .to(Timestamp.parseTimestamp("2024-02-02T10:00:00Z")) + .build(); + spannerResourceManager.write(m2); + Mutation m3 = + Mutation.newInsertOrUpdateBuilder("Users") + .set("id") + .to(3) + .set("time_colm") + .to(Timestamp.parseTimestamp("2024-02-02T20:00:00Z")) + .build(); + spannerResourceManager.write(m3); + } + + private void assertRowInOracle() throws InterruptedException { + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> runIsolatedGetRowCount(jdbcResourceManager, testUsername, "\"Users\"") == 3); + assertThatResult(result).meetsConditions(); + List> rows = + runIsolatedSQLQuery( + jdbcResourceManager, + testUsername, + "SELECT \"id\",\"time_colm\" FROM \"Users\" ORDER BY \"id\""); + assertThat(rows).hasSize(3); + assertThat(rows.get(0).get("id")).isEqualTo(java.math.BigDecimal.valueOf(1)); + assertThat(rows.get(0).get("time_colm").toString()).isEqualTo("2024-02-02 10:00:00.0"); + assertThat(rows.get(1).get("id")).isEqualTo(java.math.BigDecimal.valueOf(2)); + assertThat(rows.get(1).get("time_colm").toString()).isEqualTo("2024-02-02 20:00:00.0"); + assertThat(rows.get(2).get("id")).isEqualTo(java.math.BigDecimal.valueOf(3)); + assertThat(rows.get(2).get("time_colm").toString()).isEqualTo("2024-02-03 06:00:00.0"); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleWithoutSessionIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleWithoutSessionIT.java new file mode 100644 index 0000000000..a81db5f71d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToOracleWithoutSessionIT.java @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Value; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToOracleWithoutSessionIT extends SpannerToSourceDbITBase { + @Rule public Timeout timeout = new Timeout(25, TimeUnit.MINUTES); + + private static final Logger LOG = LoggerFactory.getLogger(SpannerToOracleWithoutSessionIT.class); + + private static final Duration TEST_TIMEOUT = Duration.ofMinutes(10); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToOracleWithoutSessionIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToOracleWithoutSessionIT/oracle-schema.sql"; + + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + private static OracleResourceManager jdbcResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToOracleWithoutSessionIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToOracleWithoutSessionIT.SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + try { + createOracleSchema( + jdbcResourceManager, + SpannerToOracleWithoutSessionIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + } catch (Exception e) { + throw new IOException("Failed to create Oracle Schema", e); + } + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = new HashMap<>(); + + // If your target source database relies on a proprietary JDBC driver that is excluded from + // the main template deployment + // Not specifically called out if oracle driver is staged dynamically... wait I should check + // if we need `--jdbcDriverJars` + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + "oracle", + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToOracleWithoutSessionIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void spannerToOracleGeneratedColumns() { + LOG.info("Starting Spanner to Oracle Generated Columns IT"); + assertThatPipeline(jobInfo).isRunning(); + Map>> spannerTableData = new HashMap<>(); + OracleGeneratedColumnUtils.addInitialMultiColSpannerData(spannerTableData); + + OracleGeneratedColumnUtils.writeRowsInSpanner(spannerTableData, spannerResourceManager); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + OracleGeneratedColumnUtils.buildConditionCheck( + spannerTableData, jdbcResourceManager, testUsername)); + assertThatResult(result).meetsConditions(); + + Map>> expectedData = new HashMap<>(); + OracleGeneratedColumnUtils.addInitialGeneratedColumnData(expectedData); + OracleGeneratedColumnUtils.assertRowInOracle(expectedData, jdbcResourceManager, testUsername); + + Map>> updateSpannerTableData = + OracleGeneratedColumnUtils.updateGeneratedColRowsInSpanner(spannerResourceManager); + spannerTableData.putAll(updateSpannerTableData); + result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, TEST_TIMEOUT), + OracleGeneratedColumnUtils.buildConditionCheck( + spannerTableData, jdbcResourceManager, testUsername)); + assertThatResult(result).meetsConditions(); + + expectedData = new HashMap<>(); + OracleGeneratedColumnUtils.addUpdatedGeneratedColumnData(expectedData); + OracleGeneratedColumnUtils.assertRowInOracle(expectedData, jdbcResourceManager, testUsername); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryAllDLQIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..65ed74408c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryAllDLQIT.java @@ -0,0 +1,696 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.spanner.migrations.constants.Constants.ORACLE_SOURCE_TYPE; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.apache.beam.it.jdbc.conditions.JDBCRowsCheck; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for reverse replication from Spanner to Oracle using the retryAllDLQ mode. + * + *

Objective: Verify that the retryAllDLQ batch job correctly processes and retries ALL Dead + * Letter Queue (DLQ) events when the main pipeline is stopped. + * + *

Edge cases covered in this test include: + * + *

    + *
  • Handling retriable errors such as check constraint and foreign key violations via the + * retryAllDLQ pipeline. + *
  • Processing severe errors introduced by custom transformation failures via the retryAllDLQ + * pipeline. + *
  • Retrying fixed items successfully in both retry/ and severe/ buckets: e.g. fixing a foreign + * key violation by inserting a missing parent row, and using a corrected transformation file. + *
  • Ensuring non-fixable items remain correctly logged under their respective error buckets. + *
  • Validating schema complexities between Source and Spanner, including mismatched primary + * keys, added, deleted, and renamed columns, as well as all datatypes. + *
  • Utilizing the schema overrides file to reconcile schema differences. + *
  • Utilizing the static DLQ file-based consumer (instead of the Pub/Sub consumer flow). + *
+ */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToSourceDBOracleRetryAllDLQIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToSourceDBOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryAllDLQIT/overrides.json"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (SpannerToSourceDBOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToSourceDBOracleRetryAllDLQIT.SPANNER_DDL_RESOURCE); + + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleSchema( + jdbcResourceManager, + SpannerToSourceDBOracleRetryAllDLQIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + + // Upload overrides file + gcsResourceManager.uploadArtifact( + "input/overrides.json", Resources.getResource(OVERRIDES_FILE_RESOURCE).getPath()); + + CustomTransformation customTransformation = + CustomTransformation.builder( + "input/customShard.jar", // Use relative path! + "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=bad") + .build(); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + Map jobParameters = + new HashMap<>() { + { + put( + "schemaOverridesFilePath", + getGcsPath("input/overrides.json", gcsResourceManager)); + put("dlqMaxRetryCount", "20"); + put( + "dlqRetryMinutes", + "60"); // keeping these high so that the test can comfortably read the static + // retry/ bucket + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, // Passing null disables Pub/Sub consumer, leaving retry DLQ items statically + // in the bucket + null, + null, + null, + null, + customTransformation, + ORACLE_SOURCE_TYPE, + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToSourceDBOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, spannerMetadataResourceManager, gcsResourceManager); + } + + @Test + public void testSpannerToSrcDBRetryAllDLQ() throws Exception { + LOG.info("Starting testSpannerToSrcDBRetryAllDLQ"); + assertThatPipeline(jobInfo).isRunning(); + + // 1. Insert parent rows directly into Oracle. This prevents out-of-order Dataflow failures + // since Dataflow processes asynchronously and might process child rows before parent rows. + LOG.info("Inserting parent rows directly into Oracle"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"" + + testUsername + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (2, 'Customer 2', 1500, 'Silver')"); + + // 2. Insert all test data into the source Spanner database. This will generate: + // - 2 severe errors (for id=999 and id=888) due to the custom transformation throwing exception + // in "bad" mode. + // - 1 retryable error (for order101) due to missing parent customer (FK violation: Customer 3 + // does not exist). + // - 1 retryable error (for customer1) due to check constraint violation (CreditLimit is 500, + // must be > 1000). + insertDataInSpanner(); + LOG.info("Data inserted into Spanner successfully"); + + // 3. Wait for DLQ events to appear in corresponding buckets. + LOG.info("Waiting for DLQ events to appear in retry and severe buckets"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(2) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"Orders\"") + .setMinRows(1) // id=102 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"AllDataTypes\"") + .setMinRows(1) // id=1 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"Customers\"") + .setMinRows(1) // id=2 + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + LOG.info("DLQ events successfully generated in corresponding buckets"); + + // 4. Stop the regular pipeline. The retry pipeline must run independently. + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(15))); + LOG.info("Regular pipeline stopped successfully"); + + // 5. Apply partial fixes to simulate user intervention correcting data before DLQ retry. + // Insert parent for Orders to fix the foreign key violation. + LOG.info("Applying partial fixes in Oracle (inserting missing parent row for Orders)"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"" + + testUsername + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (3, 'Parent Customer', 2000, 'Gold')"); + + // 6. Launch a new Dataflow job in retryAllDLQ mode to process the DLQ items. + LOG.info("Launching retryAllDLQ job with schema overrides to process DLQ"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryAllDLQ"); + retryParams.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryParams.put("dlqMaxRetryCount", "20"); + retryParams.put("dlqRetryMinutes", "60"); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, + null, + null, + null, + null, + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters( + "mode=semi-fixed") // This fixes one of our severe errors simulated in the + // transformer + .build(), + ORACLE_SOURCE_TYPE, + retryParams); + LOG.info("RetryAllDLQ job launched: {}", retryJobInfo.jobId()); + + assertThatPipeline(retryJobInfo).isRunning(); + + // 7. Wait for the retry job to process events and ensure they reach the DLQ correctly BEFORE + // cancelling + // The buckets should now have exactly 1 entry each (for Customers 1 in retry and id=888 in + // severe). + // The other entries were fixed: + // - Orders 101 FK violation fixed by inserting missing parent row. + // - AllDataTypes 999 severe error fixed by updating custom transformation to mode="semi-fixed". + // Remaining rows: + // - Customers 1 remains in retry because the check constraint violation was not fixed. + // - AllDataTypes 888 remains in severe because mode='semi-fixed' only fixes row 999, not 888. + LOG.info("Waiting for DLQ events to appear in retry and severe buckets after retry"); + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/retry/") + .setMinEvents(1) + .setMaxEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(1) + .setMaxEvents(1) + .build()) + .and( + JDBCRowsCheck.builder(jdbcResourceManager, "\"" + testUsername + "\".\"Orders\"") + .setMinRows(2) // id = 102 and 101 + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"AllDataTypes\"") + .setMinRows(2) // id = 1 and 999 + .build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(15)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + LOG.info("Retry job completed processing successfully"); + + // 8. Verify target Oracle database has the correct updated state. + LOG.info("Verifying target Oracle database contents"); + + assertTrue( + JDBCRowsCheck.builder(jdbcResourceManager, "\"" + testUsername + "\".\"AllDataTypes\"") + .setMinRows(2) + .setMaxRows(2) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder(jdbcResourceManager, "\"" + testUsername + "\".\"Customers\"") + .setMinRows(2) + .setMaxRows(2) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder(jdbcResourceManager, "\"" + testUsername + "\".\"Orders\"") + .setMinRows(2) + .setMaxRows(2) + .build() + .get()); + + // AllDataTypes: + // id=1 should exist + // id=999 should exist (fixed) + // id=888 should NOT exist (written back since the transformation error wasn't fixed) + List> allDataTypesRows = + runIsolatedSQLQuery(jdbcResourceManager, testUsername, "SELECT * FROM \"AllDataTypes\""); + List allDataTypesIds = + allDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=1 should exist", allDataTypesIds.contains(1)); + assertTrue("id=999 should exist", allDataTypesIds.contains(999)); + assertTrue("id=888 should NOT exist", !allDataTypesIds.contains(888)); + + // Assert contents of AllDataTypes rows to ensure data integrity + Map row999 = + allDataTypesRows.stream() + .filter(r -> getIntValueCaseInsensitive(r, "id") == 999) + .findFirst() + .orElse(null); + assertTrue("Row with id=999 should be found", row999 != null); + Map expectedRow999 = createExpectedRow999(); + assertRowMatchesExpected(row999, expectedRow999); + + // Customers: + // id=2 should exist (inserted directly) + // id=3 should exist (inserted as a partial fix) + // id=1 should NOT exist (check constraint violation wasn't fixed: CreditLimit was 500 but must + // be > 1000) + List> customersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"CustomerId\" FROM \"Customers\""); + List customersIds = + customersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=1 should NOT exist", !customersIds.contains(1)); + assertTrue("id=2 should exist", customersIds.contains(2)); + assertTrue("id=3 should exist", customersIds.contains(3)); + + // Orders: + // id=101 should exist (FK issue fixed by inserting parent) + // id=102 should exist (parent was seeded originally) + List> ordersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"OrderId\" FROM \"Orders\""); + List ordersIds = + ordersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=101 should exist", ordersIds.contains(101)); + assertTrue("id=102 should exist", ordersIds.contains(102)); + + LOG.info("Verified target Oracle database contents successfully"); + } + + private Integer getIntValueCaseInsensitive(Map map, String key) { + for (String k : map.keySet()) { + if (k.equalsIgnoreCase(key)) { + Object val = map.get(k); + if (val instanceof Number) { + return ((Number) val).intValue(); + } + } + } + return null; + } + + private void insertDataInSpanner() { + com.google.cloud.spanner.Mutation customer1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(1) + .set("CustomerName") + .to("Customer 1") + .set("CreditLimit") + .to(500) // this will fail due to check constraint at source + .set("LoyaltyTier") + .to("Bronze") + .build(); + + com.google.cloud.spanner.Mutation order101 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(101) + .set("CustomerId") + .to(3) // fails due to no parent row in Customers + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("Website") + .build(); + com.google.cloud.spanner.Mutation order102 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(102) + .set("CustomerId") + .to(2) + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("AppStore") + .build(); + + com.google.cloud.spanner.Mutation allTypes1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(1) + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test1") + .set("bit8_col") + .to(11) + .set("bit1_col") + .to(true) + .build(); + com.google.cloud.spanner.Mutation allTypes999 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(999) + .set("varchar_col") + .to("test999") + .set("tinyint_col") + .to(9) + .set("tinyint_unsigned_col") + .to(9) + .set("text_col") + .to("text999") + .set("date_col") + .to(com.google.cloud.Date.parseDate("2023-01-09")) + .set("smallint_col") + .to(99) + .set("smallint_unsigned_col") + .to(99) + .set("mediumint_col") + .to(999) + .set("mediumint_unsigned_col") + .to(999) + .set("bigint_col") + .to(9999L) + .set("bigint_unsigned_col") + .to(new java.math.BigDecimal("9999")) + .set("float_col") + .to(9.9f) + .set("double_col") + .to(99.9d) + .set("decimal_col") + .to(new java.math.BigDecimal("99.9")) + .set("datetime_col") + .to(com.google.cloud.Timestamp.parseTimestamp("2023-01-09T12:00:00Z")) + .set("time_col") + .to("12:00:09") + .set("year_col") + .to("2023") + .set("char_col") + .to("c") + .set("tinyblob_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("blob9".getBytes()))) + .set("tinytext_col") + .to("tinytext9") + .set("blob_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("blob9".getBytes()))) + .set("mediumblob_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("mediumblob9".getBytes()))) + .set("mediumtext_col") + .to("mediumtext9") + .set("test_json_col") + .to(com.google.cloud.spanner.Value.json("{\"k\":\"v9\"}")) + .set("longblob_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("longblob9".getBytes()))) + .set("longtext_col") + .to("longtext9") + .set("enum_col") + .to("1") + .set("bool_col") + .to(true) + .set("binary_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("bin".getBytes()))) + .set("varbinary_col") + .to( + com.google.cloud.ByteArray.fromBase64( + java.util.Base64.getEncoder().encodeToString("varbin".getBytes()))) + .set("bit_col") + .to(com.google.cloud.ByteArray.copyFrom(new byte[] {(byte) 1})) + .set("bit8_col") + .to(255) + .set("bit1_col") + .to(true) + .set("boolean_col") + .to(false) + .set("int_col") + .to(9999) + .set("integer_unsigned_col") + .to(9999) + .set("timestamp_col") + .to(com.google.cloud.Timestamp.parseTimestamp("2023-01-09T12:00:00Z")) + .set("set_col") + .to("v1") + .build(); + com.google.cloud.spanner.Mutation allTypes888 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(888) // Bad and good transformer fail on purpose + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test888") + .set("bit8_col") + .to(33) + .set("bit1_col") + .to(true) + .build(); + + spannerResourceManager.write( + List.of(customer1, order101, order102, allTypes1, allTypes999, allTypes888)); + } + + private String getCustomShardJarPath() { + String userDir = System.getProperty("user.dir"); + if (userDir.endsWith("v2/spanner-to-sourcedb")) { + return "../spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + return "v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private Map createExpectedRow999() { + Map row = new java.util.HashMap<>(); + row.put("id", 999); + row.put("varchar_col", "test999"); + row.put("tinyint_col", 9); + row.put("text_col", "text999"); + row.put("date_col", "2023-01-09"); // Expected as String + row.put("smallint_col", 99); + row.put("mediumint_col", 999); + row.put("bigint_col", 9999L); + row.put("decimal_col", new java.math.BigDecimal("99.9")); + row.put("float_col", 9.9f); + row.put("double_col", 99.9d); + row.put("char_col", "c"); + row.put("tinytext_col", "tinytext9"); + row.put("mediumtext_col", "mediumtext9"); + row.put("longtext_col", "longtext9"); + row.put("enum_col", "1"); + row.put("bool_col", true); + row.put("boolean_col", false); + row.put("int_col", 9999); + row.put("set_col", "v1"); + row.put("tinyblob_col", "blob9".getBytes()); + row.put("blob_col", "blob9".getBytes()); + row.put("mediumblob_col", "mediumblob9".getBytes()); + row.put("longblob_col", "longblob9".getBytes()); + row.put("binary_col", "bin".getBytes()); // RAW(255) is NOT right-padded with zeros + row.put("varbinary_col", "varbin".getBytes()); + row.put("bit_col", new byte[] {1}); // Oracle RAW(8) stores exactly what is pushed + row.put("bit8_col", 255); // BIT(8) expected as Integer 255 + row.put("bit1_col", true); + row.put("tinyint_unsigned_col", 9); + row.put("smallint_unsigned_col", 99); + row.put("mediumint_unsigned_col", 999); + row.put("bigint_unsigned_col", 9999L); + row.put("integer_unsigned_col", 9999); + row.put( + "datetime_col", + "2023-01-09 12:00:00"); // Expected as String, handles missing seconds in actual + row.put("time_col", "12:00:09"); + row.put("year_col", "2023"); // Expected as String, handles Date return in actual + row.put("test_json_col", "{\"k\":\"v9\"}"); // JSON expected as String + row.put("timestamp_col", "2023-01-09 12:00:00"); + + return row; + } + + private void assertRowMatchesExpected( + Map actualRow, Map expectedRow) { + expectedRow.forEach( + (key, expectedValue) -> { + Object actualValue = actualRow.get(key); + + LOG.info("Field '{}': expectedValue={}, actualValue={}", key, expectedValue, actualValue); + + if (expectedValue == null) { + assertTrue("Field " + key + " should be null", actualValue == null); + } else if (actualValue instanceof java.sql.Clob) { + try { + java.sql.Clob clob = (java.sql.Clob) actualValue; + String actualString = clob.getSubString(1, (int) clob.length()); + assertTrue( + "Field " + key + " mismatch", String.valueOf(expectedValue).equals(actualString)); + } catch (java.sql.SQLException e) { + throw new RuntimeException("Failed to read Clob", e); + } + } else if (expectedValue instanceof byte[] && actualValue instanceof java.sql.Blob) { + try { + java.sql.Blob blob = (java.sql.Blob) actualValue; + byte[] actualBytes = blob.getBytes(1, (int) blob.length()); + assertTrue( + "Field " + key + " mismatch", + java.util.Arrays.equals((byte[]) expectedValue, actualBytes)); + } catch (java.sql.SQLException e) { + throw new RuntimeException("Failed to read Blob", e); + } + + } else if (expectedValue instanceof Boolean && actualValue instanceof java.lang.Number) { + boolean expectedBool = (Boolean) expectedValue; + boolean actualBool = ((java.lang.Number) actualValue).intValue() == 1; + assertTrue( + "Field " + key + " mismatch: expected " + expectedValue + " but got " + actualValue, + expectedBool == actualBool); + } else if (expectedValue instanceof byte[] && actualValue instanceof byte[]) { + assertTrue( + "Field " + key + " mismatch", + java.util.Arrays.equals((byte[]) expectedValue, (byte[]) actualValue)); + } else if (expectedValue instanceof Number + && actualValue instanceof byte[] + && ((byte[]) actualValue).length == 1) { + assertTrue( + "Field " + key + " mismatch", + ((Number) expectedValue).intValue() == (((byte[]) actualValue)[0] & 0xFF)); + } else if (expectedValue instanceof Number && actualValue instanceof Number) { + assertTrue( + "Field " + key + " mismatch", + Math.abs( + ((Number) expectedValue).doubleValue() + - ((Number) actualValue).doubleValue()) + < 0.001); + } else { + String exp = expectedValue.toString().replace(" ", "").replace("T", ""); + String act = + actualValue != null ? actualValue.toString().replace(" ", "").replace("T", "") : ""; + if (act.endsWith(".0")) { + act = act.substring(0, act.length() - 2); + } + + boolean isDatePrefix = + act.length() > exp.length() + && act.startsWith(exp) + && act.substring(exp.length()) + .replace("0", "") + .replace(":", "") + .replace(".", "") + .isEmpty(); + + boolean isTimePrefix = + exp.length() > act.length() + && exp.startsWith(act) + && exp.charAt(act.length()) == ':'; + + assertTrue( + "Field " + key + " mismatch: expected " + exp + " but got " + act, + exp.equals(act) || isDatePrefix || isTimePrefix); + } + }); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryDLQIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryDLQIT.java new file mode 100644 index 0000000000..6e51fd0374 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBOracleRetryDLQIT.java @@ -0,0 +1,466 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.cloud.teleport.v2.templates.constants.Constants.SOURCE_ORACLE; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.apache.beam.it.jdbc.conditions.JDBCRowsCheck; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for reverse replication from Spanner to Oracle using the retryDLQ mode. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToSourceDBOracleRetryDLQIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToSourceDBOracleRetryDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/SpannerToSourceDBOracleRetryDLQIT/overrides.json"; + + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + public static PubsubResourceManager pubsubResourceManager; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (SpannerToSourceDBOracleRetryDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToSourceDBOracleRetryDLQIT.SPANNER_DDL_RESOURCE); + + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManager = SharedOracleReverseITContainer.getInstance(); + testUsername = setupOracleIsolatedUser(jdbcResourceManager); + + createOracleTableWithNColumns(jdbcResourceManager, "test", 1, "25"); + + createOracleSchema( + jdbcResourceManager, + SpannerToSourceDBOracleRetryDLQIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsername); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, jdbcResourceManager); + + // Upload overrides file + gcsResourceManager.uploadArtifact( + "input/overrides.json", Resources.getResource(OVERRIDES_FILE_RESOURCE).getPath()); + + CustomTransformation customTransformation = + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=bad") + .build(); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + pubsubResourceManager = setUpPubSubResourceManager(); + SubscriptionName subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + + Map jobParameters = + new HashMap<>() { + { + put( + "schemaOverridesFilePath", + getGcsPath("input/overrides.json", gcsResourceManager)); + put("dlqRetryMinutes", "1"); + put("dlqMaxRetryCount", "1000"); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + customTransformation, + SOURCE_ORACLE, + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToSourceDBOracleRetryDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testSpannerToSrcDBRetryDLQ() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + // Insert parent rows directly into Oracle to prevent out-of-order Dataflow failures. + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"" + + testUsername + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (2, 'Customer 2', 1500, 'Silver')"); + + jdbcResourceManager.runSQLUpdate("COMMIT"); + insertDataInSpanner(); + + LOG.info("Waiting for DLQ events to appear in severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(2) + .build() + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"Orders\"") + .setMinRows(1) // id = 102 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"AllDataTypes\"") + .setMinRows(1) // id = 1 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"Customers\"") + .setMinRows(1) // id = 2 + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + LOG.info("Verifying Oracle state before retry job runs"); + List> customersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"CustomerId\" FROM \"Customers\""); + List customersIds = + customersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=1 should NOT exist yet", !customersIds.contains(1)); + + List> ordersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"OrderId\" FROM \"Orders\""); + List ordersIds = + ordersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=101 should NOT exist yet", !ordersIds.contains(101)); + assertTrue("id=102 should exist", ordersIds.contains(102)); + + List> allDataTypesRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"id\" FROM \"AllDataTypes\""); + List allDataTypesIds = + allDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=1 should exist", allDataTypesIds.contains(1)); + + LOG.info("Launching retryDLQ job with schema overrides to process DLQ"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryDLQ"); + retryParams.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, + null, + null, + null, + null, + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=semi-fixed") + .build(), + SOURCE_ORACLE, + retryParams); + + assertThatPipeline(retryJobInfo).isRunning(); + + LOG.info("Applying partial fixes in Oracle (inserting missing parent row for Orders)"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"" + + testUsername + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (3, 'Parent Customer', 2000, 'Gold')"); + + jdbcResourceManager.runSQLUpdate("COMMIT"); + LOG.info("Waiting for the retryDLQ job to complete automatically"); + PipelineOperator.Result retryJobResult = + pipelineOperator().waitUntilDone(createConfig(retryJobInfo, Duration.ofMinutes(15))); + assertThatResult(retryJobResult).isLaunchFinished(); + + LOG.info("Verifying that severe bucket has exactly 1 entry after retryDLQ job completes"); + assertTrue( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(1) + .setMaxEvents(1) + .build() + .get()); + + LOG.info("Waiting for fixed rows to appear in Oracle"); + PipelineOperator.Result finalWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + JDBCRowsCheck.builder(jdbcResourceManager, "\"" + testUsername + "\".\"Orders\"") + .setMinRows(2) + .build() + .and( + JDBCRowsCheck.builder( + jdbcResourceManager, "\"" + testUsername + "\".\"AllDataTypes\"") + .setMinRows(2) + .build())); + assertThatResult(finalWaitResult).meetsConditions(); + + LOG.info("Verifying final target Oracle database contents"); + + customersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"CustomerId\" FROM \"Customers\""); + customersIds = + customersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=1 should NOT exist", !customersIds.contains(1)); + assertTrue("id=2 should exist", customersIds.contains(2)); + assertTrue("id=3 should exist", customersIds.contains(3)); + + ordersRows = + runIsolatedSQLQuery( + jdbcResourceManager, testUsername, "SELECT \"OrderId\" FROM \"Orders\""); + ordersIds = ordersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=101 should exist", ordersIds.contains(101)); + assertTrue("id=102 should exist", ordersIds.contains(102)); + + allDataTypesRows = + runIsolatedSQLQuery(jdbcResourceManager, testUsername, "SELECT * FROM \"AllDataTypes\""); + allDataTypesIds = + allDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=1 should exist", allDataTypesIds.contains(1)); + assertTrue("id=999 should exist", allDataTypesIds.contains(999)); + assertTrue("id=888 should NOT exist", !allDataTypesIds.contains(888)); + + Map row999 = + allDataTypesRows.stream() + .filter(r -> getIntValueCaseInsensitive(r, "id") == 999) + .findFirst() + .orElse(null); + assertTrue("Row with id=999 should be found", row999 != null); + + Map expectedRow999 = createExpectedRow999(); + assertRowMatchesExpected(row999, expectedRow999); + + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineLauncher.cancelJob(PROJECT, REGION, jobInfo.jobId()); + } + + private Integer getIntValueCaseInsensitive(Map map, String key) { + for (String k : map.keySet()) { + if (k.equalsIgnoreCase(key) || k.equalsIgnoreCase("\"" + key + "\"")) { + Object val = map.get(k); + if (val instanceof Number) { + return ((Number) val).intValue(); + } + } + } + return null; + } + + private void insertDataInSpanner() { + com.google.cloud.spanner.Mutation customer1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(1) + .set("CustomerName") + .to("Customer 1") + .set("CreditLimit") + .to(500) // this will fail due to check constraint at source + .set("LoyaltyTier") + .to("Bronze") + .build(); + com.google.cloud.spanner.Mutation order101 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(101) + .set("CustomerId") + .to(3) // fails due to no parent row in Customers + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("Website") + .build(); + com.google.cloud.spanner.Mutation order102 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(102) + .set("CustomerId") + .to(2) + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("AppStore") + .build(); + + com.google.cloud.spanner.Mutation allTypes1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(1) + .set("varchar_col") + .to("test1") + .build(); + com.google.cloud.spanner.Mutation allTypes999 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(999) + .set("varchar_col") + .to("test999") + .build(); + com.google.cloud.spanner.Mutation allTypes888 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(888) /* Bad and semi-fixed transformer fail on purpose */ + .set("varchar_col") + .to("test888") + .build(); + + spannerResourceManager.write( + List.of(customer1, order101, order102, allTypes1, allTypes999, allTypes888)); + } + + private String getCustomShardJarPath() { + String userDir = System.getProperty("user.dir"); + if (userDir.endsWith("v2/spanner-to-sourcedb")) { + return "../spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + return "v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private Map createExpectedRow999() { + Map row = new java.util.HashMap<>(); + row.put("id", 999); + row.put("varchar_col", "test999"); + return row; + } + + private void assertRowMatchesExpected( + Map actualRow, Map expectedRow) { + expectedRow.forEach( + (key, expectedValue) -> { + Object actualValue = actualRow.get(key); + if (actualValue == null && actualRow.containsKey(key.toUpperCase())) { + actualValue = actualRow.get(key.toUpperCase()); + } + + LOG.info("Field '{}': expectedValue={}, actualValue={}", key, expectedValue, actualValue); + + if (expectedValue == null) { + assertTrue("Field " + key + " should be null", actualValue == null); + } else if (expectedValue instanceof byte[] && actualValue instanceof byte[]) { + assertTrue( + "Field " + key + " mismatch", + java.util.Arrays.equals((byte[]) expectedValue, (byte[]) actualValue)); + } else if (expectedValue instanceof Number + && actualValue instanceof byte[] + && ((byte[]) actualValue).length == 1) { + assertTrue( + "Field " + key + " mismatch", + ((Number) expectedValue).intValue() == (((byte[]) actualValue)[0] & 0xFF)); + } else if (expectedValue instanceof Number && actualValue instanceof Number) { + assertTrue( + "Field " + key + " mismatch", + Math.abs( + ((Number) expectedValue).doubleValue() + - ((Number) actualValue).doubleValue()) + < 0.001); + } else { + String exp = expectedValue.toString().replace(" ", "").replace("T", ""); + String act = + actualValue != null ? actualValue.toString().replace(" ", "").replace("T", "") : ""; + if (act.endsWith(".0")) { + act = act.substring(0, act.length() - 2); + } + + boolean isDatePrefix = + act.length() > exp.length() + && act.startsWith(exp) + && act.charAt(exp.length()) == '-'; + boolean isTimePrefix = + exp.length() > act.length() + && exp.startsWith(act) + && exp.charAt(act.length()) == ':'; + + assertTrue( + "Field " + key + " mismatch: expected " + exp + " but got " + act, + exp.equals(act) || isDatePrefix || isTimePrefix); + } + }); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..fb55afe39e --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT.java @@ -0,0 +1,514 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.apache.beam.it.jdbc.conditions.JDBCRowsCheck; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for reverse replication from Spanner to MySQL using the retryAllDLQ mode. + * + *

Objective: Verify that the retryAllDLQ batch job correctly processes and retries ALL Dead + * Letter Queue (DLQ) events when the main pipeline is stopped inside a sharded topology using + * custom shard logic. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToSourceDBShardedOracleRetryAllDLQIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToSourceDBShardedOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String MYSQL_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/overrides.json"; + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManagerShardA; + public static OracleResourceManager jdbcResourceManagerShardB; + public static GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (SpannerToSourceDBShardedOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToSourceDBShardedOracleRetryAllDLQIT.SPANNER_DDL_RESOURCE); + + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManagerShardA = SharedOracleReverseITContainer.getInstance(); + testUsernameShardA = setupOracleIsolatedUser(jdbcResourceManagerShardA); + createOracleSchema( + jdbcResourceManagerShardA, + SpannerToSourceDBShardedOracleRetryAllDLQIT.MYSQL_SCHEMA_FILE_RESOURCE, + testUsernameShardA); + + jdbcResourceManagerShardB = SharedOracleReverseITContainer.getInstance(); + testUsernameShardB = setupOracleIsolatedUser(jdbcResourceManagerShardB); + createOracleSchema( + jdbcResourceManagerShardB, + SpannerToSourceDBShardedOracleRetryAllDLQIT.MYSQL_SCHEMA_FILE_RESOURCE, + testUsernameShardB); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + // Use generic multi-shard logic instead of base IT helper + createAndUploadShardConfigToGcs( + gcsResourceManager, + Map.of( + "testShardA", jdbcResourceManagerShardA, "testShardB", jdbcResourceManagerShardB)); + ; + + // Upload overrides file + gcsResourceManager.uploadArtifact( + "input/overrides.json", Resources.getResource(OVERRIDES_FILE_RESOURCE).getPath()); + + CustomTransformation customTransformation = + CustomTransformation.builder( + "input/customShard.jar", // Use relative path! + "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=bad") + .build(); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + Map jobParameters = + new HashMap<>() { + { + put( + "schemaOverridesFilePath", + getGcsPath("input/overrides.json", gcsResourceManager)); + put("dlqMaxRetryCount", "20"); + put("dlqRetryMinutes", "60"); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, // Passing null disables Pub/Sub consumer, leaving retry DLQ items statically + getClass().getSimpleName(), + "input/customShard.jar", + "com.custom.CustomShardIdFetcherForRetryIT", + null, + customTransformation, + "oracle", + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToSourceDBShardedOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, spannerMetadataResourceManager, gcsResourceManager); + } + + @Test + public void testSpannerToSrcDBRetryAllDLQ() throws Exception { + LOG.info("Starting testSpannerToSrcDBRetryAllDLQ for sharded execution"); + assertThatPipeline(jobInfo).isRunning(); + + // 1. Insert parent rows directly into MySQL. + LOG.info("Inserting parent rows directly into MySQL"); + // customer2 routes to ShardB (2%2==0) + jdbcResourceManagerShardB.runSQLUpdate( + "INSERT INTO \"" + + testUsernameShardB + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (2, 'Customer 2', 1500, 'Silver')"); + + // 2. Insert test data into the source Spanner database. This will generate: + // - 2 severe errors (for id=999 and id=888) due to the custom transformation throwing exception + // in "bad" mode. + // - 1 retryable error (for order101) due to missing parent customer (FK violation: Customer 3 + // does not exist). + // - 1 retryable error (for customer1) due to check constraint violation (CreditLimit is 500, + // must be > 1000). + insertDataInSpanner(); + LOG.info("Data inserted into Spanner successfully"); + + // 3. Wait for DLQ events to appear in corresponding buckets. + // Total events expected: + // - Customers: 1 + // - Orders: 1 + // - AllDataTypes: 2 + LOG.info("Waiting for DLQ events to appear in retry and severe buckets"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(2) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, + "\"" + testUsernameShardB + "\".\"Orders\"") + .setMinRows(1) // id = 102 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, + "\"" + testUsernameShardA + "\".\"AllDataTypes\"") + .setMinRows(1) // id = 1 + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, + "\"" + testUsernameShardB + "\".\"Customers\"") + .setMinRows(1) // id = 2 + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + LOG.info("DLQ events appeared in corresponding buckets"); + + // 4. Stop the regular pipeline. + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(15))); + LOG.info("Regular pipeline stopped successfully"); + + // 5. Apply partial fixes to simulate user intervention correcting data before DLQ retry. + LOG.info("Applying partial fixes in MySQL (inserting missing parent row for Orders)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"" + + testUsernameShardA + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (3, 'Parent Customer A', 2000, 'Gold')"); + + // 6. Launch a new Dataflow job in retryAllDLQ mode. + LOG.info("Launching retryAllDLQ job with schema overrides to process DLQ"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryAllDLQ"); + retryParams.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryParams.put("dlqMaxRetryCount", "20"); + retryParams.put("dlqRetryMinutes", "60"); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, + getClass().getSimpleName(), + "input/customShard.jar", + "com.custom.CustomShardIdFetcherForRetryIT", + null, + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=semi-fixed") + .build(), + "oracle", + retryParams); + LOG.info("RetryAllDLQ job launched: {}", retryJobInfo.jobId()); + + assertThatPipeline(retryJobInfo).isRunning(); + + // 7. Wait for the retry job to process events and ensure they reach the DLQ correctly BEFORE + // cancelling. + // The buckets should now have exactly 1 entry each (for Customers 1 in retry and id=888 in + // severe). + // The other entries were fixed: + // - Orders 101 FK violation fixed by inserting missing parent row on Shard A. + // - AllDataTypes 999 severe error fixed by updating custom transformation to mode="semi-fixed". + // Remaining rows: + // - Customers 1 remains in retry because the check constraint violation was not fixed. + // - AllDataTypes 888 remains in severe because mode='semi-fixed' only fixes row 999, not 888. + LOG.info("Waiting for DLQ events to appear in retry and severe buckets after retry"); + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/retry/") + .setMinEvents(1) + .setMaxEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(1) + .setMaxEvents(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, "\"" + testUsernameShardA + "\".\"Orders\"") + .setMinRows(1) // id = 101 + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, + "\"" + testUsernameShardA + "\".\"AllDataTypes\"") + .setMinRows(2) // id = 1 and 999 + .build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(15)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + LOG.info("Retry job completed processing successfully"); + + // 8. Verify target MySQL database has the correct updated state. + LOG.info("Verifying MySQL data across logical shards"); + + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, "\"" + testUsernameShardA + "\".\"AllDataTypes\"") + .setMinRows(2) + .setMaxRows(2) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, "\"" + testUsernameShardB + "\".\"AllDataTypes\"") + .setMinRows(0) + .setMaxRows(0) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, "\"" + testUsernameShardB + "\".\"Customers\"") + .setMinRows(1) + .setMaxRows(1) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, "\"" + testUsernameShardA + "\".\"Customers\"") + .setMinRows(1) + .setMaxRows(1) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, "\"" + testUsernameShardA + "\".\"Orders\"") + .setMinRows(1) + .setMaxRows(1) + .build() + .get()); + assertTrue( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, "\"" + testUsernameShardB + "\".\"Orders\"") + .setMinRows(1) + .setMaxRows(1) + .build() + .get()); + + List> shardAAllTypes = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, testUsernameShardA, "SELECT \"id\" FROM \"AllDataTypes\""); + List shardAAllTypesIds = + shardAAllTypes.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + + List> shardBAllTypes = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, testUsernameShardB, "SELECT \"id\" FROM \"AllDataTypes\""); + List shardBAllTypesIds = + shardBAllTypes.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + + // 1(mod2!=0) -> ShardA + assertTrue("id=1 should exist on Shard A", shardAAllTypesIds.contains(1)); + // 999(mod2!=0) -> ShardA + assertTrue("id=999 should exist on Shard A", shardAAllTypesIds.contains(999)); + // 888(mod2==0) -> ShardB (Fails repeatedly, so shouldn't exist anywhere) + assertTrue("id=888 should NOT exist on Shard B", !shardBAllTypesIds.contains(888)); + + List> shardACust = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"CustomerId\" FROM \"Customers\""); + List shardACustIds = + shardACust.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + + List> shardBCust = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, + testUsernameShardB, + "SELECT \"CustomerId\" FROM \"Customers\""); + List shardBCustIds = + shardBCust.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + + // 1(mod2!=0) -> ShardA (failed check constraint: CreditLimit was 500 but must be > 1000) + assertTrue("id=1 should NOT exist on Shard A", !shardACustIds.contains(1)); + // 3(mod2!=0) -> ShardA + assertTrue("id=3 should exist on Shard A", shardACustIds.contains(3)); + // 2(mod2==0) -> ShardB + assertTrue("id=2 should exist on Shard B", shardBCustIds.contains(2)); + + List> shardAOrders = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, testUsernameShardA, "SELECT \"OrderId\" FROM \"Orders\""); + List shardAOrderIds = + shardAOrders.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + + List> shardBOrders = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, testUsernameShardB, "SELECT \"OrderId\" FROM \"Orders\""); + List shardBOrderIds = + shardBOrders.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + + // order101 (Cust 3, mod2!=0) -> ShardA + assertTrue("id=101 should exist on Shard A", shardAOrderIds.contains(101)); + // order102 (Cust 2, mod2==0) -> ShardB + assertTrue("id=102 should exist on Shard B", shardBOrderIds.contains(102)); + } + + private Integer getIntValueCaseInsensitive(Map map, String key) { + for (String k : map.keySet()) { + if (k.equalsIgnoreCase(key)) { + Object val = map.get(k); + if (val instanceof Number) { + return ((Number) val).intValue(); + } + } + } + return null; + } + + private void insertDataInSpanner() { + com.google.cloud.spanner.Mutation customer1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(1) + .set("CustomerName") + .to("Customer 1") + .set("CreditLimit") + .to(500) // this will fail due to check constraint at source + .set("LoyaltyTier") + .to("Bronze") + .build(); + + com.google.cloud.spanner.Mutation order101 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(101) + .set("CustomerId") + .to(3) // fails due to no parent row in Customers + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("Website") + .build(); + com.google.cloud.spanner.Mutation order102 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(102) + .set("CustomerId") + .to(2) + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("AppStore") + .build(); + + com.google.cloud.spanner.Mutation allTypes1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(1) + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test1") + .set("bit8_col") + .to(11) + .set("bit1_col") + .to(true) + .build(); + com.google.cloud.spanner.Mutation allTypes999 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(999) // Bad transformer fails on purpose, good transformer doesnt + .set("boolean_col") + .to(false) + .set("varchar_col") + .to("test999") + .set("bit8_col") + .to(22) + .set("bit1_col") + .to(false) + .build(); + com.google.cloud.spanner.Mutation allTypes888 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(888) // Bad and good transformer fail on purpose + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test888") + .set("bit8_col") + .to(33) + .set("bit1_col") + .to(true) + .build(); + + spannerResourceManager.write( + List.of(customer1, order101, order102, allTypes1, allTypes999, allTypes888)); + } + + private String getCustomShardJarPath() { + String userDir = System.getProperty("user.dir"); + if (userDir.endsWith("v2/spanner-to-sourcedb")) { + return "../spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + return "v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryDLQIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryDLQIT.java new file mode 100644 index 0000000000..da4f20643f --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDBShardedOracleRetryDLQIT.java @@ -0,0 +1,465 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.common.io.Resources; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.apache.beam.it.jdbc.conditions.JDBCRowsCheck; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToSourceDBShardedOracleRetryDLQIT extends SpannerToSourceDbITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToSourceDBShardedOracleRetryDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/SpannerToSourceDBShardedOracleRetryDLQIT/session.json"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager jdbcResourceManagerShardA; + public static OracleResourceManager jdbcResourceManagerShardB; + public static GcsResourceManager gcsResourceManager; + public static PubsubResourceManager pubsubResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (SpannerToSourceDBShardedOracleRetryDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = + createSpannerDatabase(SpannerToSourceDBShardedOracleRetryDLQIT.SPANNER_DDL_RESOURCE); + + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + + jdbcResourceManagerShardA = SharedOracleReverseITContainer.getInstance(); + testUsernameShardA = setupOracleIsolatedUser(jdbcResourceManagerShardA); + createOracleSchema( + jdbcResourceManagerShardA, + SpannerToSourceDBShardedOracleRetryDLQIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsernameShardA); + + jdbcResourceManagerShardB = SharedOracleReverseITContainer.getInstance(); + testUsernameShardB = setupOracleIsolatedUser(jdbcResourceManagerShardB); + createOracleSchema( + jdbcResourceManagerShardB, + SpannerToSourceDBShardedOracleRetryDLQIT.ORACLE_SCHEMA_FILE_RESOURCE, + testUsernameShardB); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs( + gcsResourceManager, + Map.of( + "testShardA", jdbcResourceManagerShardA, "testShardB", jdbcResourceManagerShardB)); + + // Upload session file + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + + CustomTransformation customTransformation = + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=bad") + .build(); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + pubsubResourceManager = setUpPubSubResourceManager(); + SubscriptionName subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + + Map jobParameters = + new HashMap<>() { + { + put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + put("dlqMaxRetryCount", "1000"); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + getClass().getSimpleName(), + null, + null, + null, + customTransformation, + "oracle", + jobParameters); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToSourceDBShardedOracleRetryDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testSpannerToSrcDBRetryDLQ() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + jdbcResourceManagerShardB.runSQLUpdate( + "INSERT INTO \"" + + testUsernameShardB + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (2, 'Customer 2', 1500, 'Silver')"); + + insertDataInSpanner(); + LOG.info("Data inserted into Spanner successfully"); + + LOG.info("Waiting for DLQ events to appear in severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(2) + .build() + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, + "\"" + testUsernameShardB + "\".\"Orders\"") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, + "\"" + testUsernameShardA + "\".\"AllDataTypes\"") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardB, + "\"" + testUsernameShardB + "\".\"Customers\"") + .setMinRows(1) + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + LOG.info("Verifying Oracle state before retry job runs"); + List> shardACustomersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"CustomerId\" FROM \"Customers\""); + List shardACustomersIds = + shardACustomersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=1 should NOT exist yet on Shard A", !shardACustomersIds.contains(1)); + + List> shardAOrdersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, testUsernameShardA, "SELECT \"OrderId\" FROM \"Orders\""); + List shardAOrdersIds = + shardAOrdersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=101 should NOT exist yet on Shard A", !shardAOrdersIds.contains(101)); + + List> shardBOrdersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, testUsernameShardB, "SELECT \"OrderId\" FROM \"Orders\""); + List shardBOrdersIds = + shardBOrdersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=102 should exist on Shard B", shardBOrdersIds.contains(102)); + + List> shardAAllDataTypesRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, testUsernameShardA, "SELECT \"id\" FROM \"AllDataTypes\""); + List shardAAllDataTypesIds = + shardAAllDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=1 should exist on Shard A", shardAAllDataTypesIds.contains(1)); + + LOG.info("Launching retryDLQ job with session file to process DLQ"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryDLQ"); + retryParams.put("sessionFilePath", getGcsPath("input/session.json", gcsResourceManager)); + + // CustomTransformationImplFetcher.clearInstance(); + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + null, + getClass().getSimpleName(), + null, + null, + null, + CustomTransformation.builder( + "input/customShard.jar", "com.custom.CustomTransformationForDLQIT") + .setCustomParameters("mode=semi-fixed") + .build(), + "oracle", + retryParams); + + assertThatPipeline(retryJobInfo).isRunning(); + + LOG.info("Applying partial fixes in Oracle (inserting missing parent row for Orders)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"" + + testUsernameShardA + + "\".\"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LegacyRegion\") VALUES (3, 'Parent Customer A', 2000, 'Gold')"); + + LOG.info("Waiting for the retryDLQ job to complete automatically"); + PipelineOperator.Result retryJobResult = + pipelineOperator().waitUntilDone(createConfig(retryJobInfo, Duration.ofMinutes(15))); + assertThatResult(retryJobResult).isLaunchFinished(); + + LOG.info("Verifying that severe bucket has exactly 1 entry after retryDLQ job completes"); + assertTrue( + DlqEventsCountCheck.builder(gcsResourceManager, "dlq/severe/") + .setMinEvents(1) + .build() + .get()); + + LOG.info("Waiting for fixed rows to appear in Oracle"); + PipelineOperator.Result finalWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, "\"" + testUsernameShardA + "\".\"Orders\"") + .setMinRows(1) + .build() + .and( + JDBCRowsCheck.builder( + jdbcResourceManagerShardA, + "\"" + testUsernameShardA + "\".\"AllDataTypes\"") + .setMinRows(2) + .build())); + assertThatResult(finalWaitResult).meetsConditions(); + + LOG.info("Verifying final target Oracle database contents across shards"); + + shardACustomersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"CustomerId\" FROM \"Customers\""); + shardACustomersIds = + shardACustomersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=1 should NOT exist on Shard A", !shardACustomersIds.contains(1)); + assertTrue("id=3 should exist on Shard A", shardACustomersIds.contains(3)); + + List> shardBCustomersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, + testUsernameShardB, + "SELECT \"CustomerId\" FROM \"Customers\""); + List shardBCustomersIds = + shardBCustomersRows.stream().map(r -> getIntValueCaseInsensitive(r, "CustomerId")).toList(); + assertTrue("id=2 should exist on Shard B", shardBCustomersIds.contains(2)); + + shardAOrdersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, testUsernameShardA, "SELECT \"OrderId\" FROM \"Orders\""); + shardAOrdersIds = + shardAOrdersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=101 should exist on Shard A", shardAOrdersIds.contains(101)); + + shardBOrdersRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, testUsernameShardB, "SELECT \"OrderId\" FROM \"Orders\""); + shardBOrdersIds = + shardBOrdersRows.stream().map(r -> getIntValueCaseInsensitive(r, "OrderId")).toList(); + assertTrue("id=102 should exist on Shard B", shardBOrdersIds.contains(102)); + + shardAAllDataTypesRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardA, + testUsernameShardA, + "SELECT \"id\", \"varchar_col\" FROM \"AllDataTypes\""); + shardAAllDataTypesIds = + shardAAllDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=1 should exist on Shard A", shardAAllDataTypesIds.contains(1)); + assertTrue("id=999 should exist on Shard A", shardAAllDataTypesIds.contains(999)); + + List> shardBAllDataTypesRows = + runIsolatedSQLQuery( + jdbcResourceManagerShardB, testUsernameShardB, "SELECT \"id\" FROM \"AllDataTypes\""); + List shardBAllDataTypesIds = + shardBAllDataTypesRows.stream().map(r -> getIntValueCaseInsensitive(r, "id")).toList(); + assertTrue("id=888 should NOT exist on Shard B", !shardBAllDataTypesIds.contains(888)); + + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineLauncher.cancelJob(PROJECT, REGION, jobInfo.jobId()); + } + + private Integer getIntValueCaseInsensitive(Map map, String key) { + for (String k : map.keySet()) { + if (k.equalsIgnoreCase(key)) { + Object val = map.get(k); + if (val instanceof Number) { + return ((Number) val).intValue(); + } + } + } + return null; + } + + private void insertDataInSpanner() { + com.google.cloud.spanner.Mutation customer1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(1) + .set("CustomerName") + .to("Customer 1") + .set("CreditLimit") + .to(500) + .set("LoyaltyTier") + .to("Bronze") + .set("migration_shard_id") + .to("testShardA") + .build(); + com.google.cloud.spanner.Mutation order101 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(101) + .set("CustomerId") + .to(3) + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("Website") + .set("migration_shard_id") + .to("testShardA") + .build(); + com.google.cloud.spanner.Mutation order102 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Orders") + .set("OrderId") + .to(102) + .set("CustomerId") + .to(2) + .set("OrderValue") + .to(1000) + .set("OrderSource") + .to("AppStore") + .set("migration_shard_id") + .to("testShardB") + .build(); + + com.google.cloud.spanner.Mutation allTypes1 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(1) + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test1") + .set("bit8_col") + .to(11) + .set("bit1_col") + .to(true) + .set("migration_shard_id") + .to("testShardA") + .build(); + com.google.cloud.spanner.Mutation allTypes999 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(999) + .set("boolean_col") + .to(false) + .set("varchar_col") + .to("test999") + .set("bit8_col") + .to(22) + .set("bit1_col") + .to(false) + .set("migration_shard_id") + .to("testShardA") + .build(); + com.google.cloud.spanner.Mutation allTypes888 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("AllDataTypes") + .set("id") + .to(888) + .set("boolean_col") + .to(true) + .set("varchar_col") + .to("test888") + .set("bit8_col") + .to(33) + .set("bit1_col") + .to(true) + .set("migration_shard_id") + .to("testShardB") + .build(); + + spannerResourceManager.write( + List.of(customer1, order101, order102, allTypes1, allTypes999, allTypes888)); + } + + private String getCustomShardJarPath() { + String userDir = System.getProperty("user.dir"); + if (userDir.endsWith("v2/spanner-to-sourcedb")) { + return "../spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + return "v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT.java new file mode 100644 index 0000000000..78cd821a36 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDb; +import com.google.cloud.teleport.v2.templates.SpannerToSourceDbITBase; +import com.google.pubsub.v1.SubscriptionName; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.OracleResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for SpannerToSourceDb Flex template using string-based schema overrides for + * Oracle. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(SpannerToSourceDb.class) +@RunWith(JUnit4.class) +public class SpannerToSourceDbOracleStringOverridesSchemaMapperIT extends SpannerToSourceDbITBase { + private static final Logger LOG = + LoggerFactory.getLogger(SpannerToSourceDbOracleStringOverridesSchemaMapperIT.class); + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + private static SpannerResourceManager spannerMetadataResourceManager; + public static OracleResourceManager oracleResourceManager; + public static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private SubscriptionName subscriptionName; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/oracle-schema.sql"; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (SpannerToSourceDbOracleStringOverridesSchemaMapperIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = createSpannerDatabase(SPANNER_DDL_RESOURCE); + spannerMetadataResourceManager = createSpannerMetadataDatabase(); + oracleResourceManager = OracleResourceManager.builder(testName).build(); + createOracleSchema(oracleResourceManager, ORACLE_SCHEMA_FILE_RESOURCE, testUsername); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createAndUploadShardConfigToGcs(gcsResourceManager, oracleResourceManager); + pubsubResourceManager = setUpPubSubResourceManager(); + subscriptionName = + createPubsubResources( + getClass().getSimpleName(), + pubsubResourceManager, + getGcsPath("dlq", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""), + gcsResourceManager); + Map jobParameters = + new HashMap<>() { + { + put("tableOverrides", "[{source_table1, Target_Table_1}]"); + put( + "columnOverrides", + "[{source_table1.name_col1, source_table1.Target_Name_Col_1}, {source_table2.category_col2, source_table2.Target_Category_Col_2}]"); + } + }; + jobInfo = + launchDataflowJob( + gcsResourceManager, + spannerResourceManager, + spannerMetadataResourceManager, + subscriptionName.toString(), + null, + null, + null, + null, + null, + "oracle", + jobParameters); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (SpannerToSourceDbOracleStringOverridesSchemaMapperIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + oracleResourceManager, + spannerMetadataResourceManager, + gcsResourceManager, + pubsubResourceManager); + } + + @Test + public void testSpannerToOracleWithStringOverrides() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + // Insert data into Spanner tables matching the override scenario + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("Target_Table_1") + .set("id_col1") + .to(1) + .set("Target_Name_Col_1") + .to("Name One") + .set("data_col1") + .to("Data for one") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("Target_Table_1") + .set("id_col1") + .to(2) + .set("Target_Name_Col_1") + .to("Name Two") + .set("data_col1") + .to("Data for two") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("source_table2") + .set("key_col2") + .to("K1") + .set("Target_Category_Col_2") + .to("Category Alpha") + .set("value_col2") + .to("Value Alpha") + .build()); + spannerResourceManager.write( + Mutation.newInsertOrUpdateBuilder("source_table2") + .set("key_col2") + .to("K2") + .set("Target_Category_Col_2") + .to("Category Beta") + .set("value_col2") + .to("Value Beta") + .build()); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + () -> + (oracleResourceManager.getRowCount("\"source_table1\"") == 2 + && oracleResourceManager.getRowCount("\"source_table2\"") == 2)); + assertThatResult(result).meetsConditions(); + + // Assert Oracle table1 (should be source_table1, with column name_col1 renamed) + // Note: getRowCount returns the count. runSQLQuery returns a list of maps. + List> oracleTable1 = + runIsolatedSQLQuery( + oracleResourceManager, + testUsername, + "SELECT \"id_col1\", \"name_col1\", TO_CHAR(\"data_col1\") AS \"data_col1\" FROM \"source_table1\" ORDER BY \"id_col1\""); + assertThat(oracleTable1).hasSize(2); + // Integer type from Oracle might come back as BigDecimal depending on driver, so let's convert + // to int to be safe + assertThat(((Number) oracleTable1.get(0).get("id_col1")).intValue()).isEqualTo(1); + assertThat(oracleTable1.get(0).get("name_col1")).isEqualTo("Name One"); + assertThat(oracleTable1.get(0).get("data_col1")).isEqualTo("Data for one"); + assertThat(((Number) oracleTable1.get(1).get("id_col1")).intValue()).isEqualTo(2); + assertThat(oracleTable1.get(1).get("name_col1")).isEqualTo("Name Two"); + assertThat(oracleTable1.get(1).get("data_col1")).isEqualTo("Data for two"); + + // Assert Oracle table2 (should be source_table2, with column category_col2 renamed) + List> oracleTable2 = + runIsolatedSQLQuery( + oracleResourceManager, + testUsername, + "SELECT \"key_col2\", \"category_col2\", TO_CHAR(\"value_col2\") AS \"value_col2\" FROM \"source_table2\" ORDER BY \"key_col2\""); + assertThat(oracleTable2).hasSize(2); + assertThat(oracleTable2.get(0).get("key_col2")).isEqualTo("K1"); + assertThat(oracleTable2.get(0).get("category_col2")).isEqualTo("Category Alpha"); + assertThat(oracleTable2.get(0).get("value_col2")).isEqualTo("Value Alpha"); + assertThat(oracleTable2.get(1).get("key_col2")).isEqualTo("K2"); + assertThat(oracleTable2.get(1).get("category_col2")).isEqualTo("Category Beta"); + assertThat(oracleTable2.get(1).get("value_col2")).isEqualTo("Value Beta"); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerGeneratedColumnUtils.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerGeneratedColumnUtils.java index e4735ca912..79f01e202e 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerGeneratedColumnUtils.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerGeneratedColumnUtils.java @@ -27,7 +27,7 @@ import java.util.Map; import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.spanner.SpannerResourceManager; -import org.apache.beam.it.jdbc.MySQLResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -39,7 +39,7 @@ public class SpannerGeneratedColumnUtils { public static ConditionCheck buildConditionCheck( Map>> spannerTableData, - MySQLResourceManager jdbcResourceManager) { + JDBCResourceManager jdbcResourceManager) { ConditionCheck combinedCondition = null; for (Map.Entry>> entry : spannerTableData.entrySet()) { String tableName = getTableName(entry.getKey()); @@ -69,7 +69,7 @@ public static ConditionCheck buildConditionCheck( public static void assertRowInMySQL( Map>> expectedData, - MySQLResourceManager jdbcResourceManager) { + JDBCResourceManager jdbcResourceManager) { for (Map.Entry>> expectedTableData : expectedData.entrySet()) { String type = expectedTableData.getKey(); String tableName = getTableName(type); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-google_standard_sql-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..fb999e0af2 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE Singers ( + SingerId INT64 NOT NULL, + FirstName STRING(MAX), +) PRIMARY KEY(SingerId); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-schema.sql new file mode 100644 index 0000000000..a71c5db080 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/oracle-schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE "Singers" ( + "SingerId" INTEGER NOT NULL, + "FirstName" VARCHAR2(50), + PRIMARY KEY("SingerId") +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/session.json new file mode 100644 index 0000000000..317487d30a --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleCustomShardIT/session.json @@ -0,0 +1,141 @@ +{ + "SpSchema": { + "t1": { + "Name": "Singers", + "ColIds": [ + "c3", + "c4" + ], + "ShardIdColumn": "", + "ColDefs": { + "c3": { + "Name": "SingerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: SingerId bigint(19)", + "Id": "c3" + }, + "c4": { + "Name": "FirstName", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: FirstName text(65535)", + "Id": "c4" + } + }, + "PrimaryKeys": [ + { + "ColId": "c3", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Singers", + "Id": "t1" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "Singers", + "Schema": "alltypes", + "ColIds": [ + "c3", + "c4" + ], + "ColDefs": { + "c3": { + "Name": "SingerId", + "Type": { + "Name": "INTEGER", + "Mods": [ + 19 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c3" + }, + "c4": { + "Name": "FirstName", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4" + } + }, + "PrimaryKeys": [ + { + "ColId": "c3", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t1" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c3": [], + "c4": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r38", + "Name": "r38", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true + } \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-googlesql-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-googlesql-spanner-schema.sql new file mode 100644 index 0000000000..eab678b8b1 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-googlesql-spanner-schema.sql @@ -0,0 +1,79 @@ +CREATE TABLE `STRING_TO_VARCHAR2_TABLE` (`id` INT64, `varchar2_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_VARCHAR2_PK_TABLE` (`varchar2_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`varchar2_pk_col`); +CREATE TABLE `STRING_TO_VARCHAR_TABLE` (`id` INT64, `varchar_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_VARCHAR_PK_TABLE` (`varchar_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`varchar_pk_col`); +CREATE TABLE `STRING_TO_CHAR_TABLE` (`id` INT64, `char_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_CHAR_PK_TABLE` (`char_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`char_pk_col`); +CREATE TABLE `STRING_TO_CHARACTER_TABLE` (`id` INT64, `character_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_CHARACTER_PK_TABLE` (`character_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`character_pk_col`); +CREATE TABLE `STRING_TO_NCHAR_TABLE` (`id` INT64, `nchar_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NCHAR_PK_TABLE` (`nchar_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`nchar_pk_col`); +CREATE TABLE `STRING_TO_NCHAR_VARYING_TABLE` (`id` INT64, `nchar_varying_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NCHAR_VARYING_PK_TABLE` (`nchar_varying_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`nchar_varying_pk_col`); +CREATE TABLE `STRING_TO_NATIONAL_CHARACTER_TABLE` (`id` INT64, `national_character_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NATIONAL_CHARACTER_PK_TABLE` (`national_character_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`national_character_pk_col`); +CREATE TABLE `STRING_TO_NATIONAL_CHAR_TABLE` (`id` INT64, `national_char_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NATIONAL_CHAR_PK_TABLE` (`national_char_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`national_char_pk_col`); +CREATE TABLE `STRING_TO_NATIONAL_CHARACTER_VARYING_TABLE` (`id` INT64, `national_character_varying_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE` (`national_character_varying_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`national_character_varying_pk_col`); +CREATE TABLE `STRING_TO_NATIONAL_CHAR_VARYING_TABLE` (`id` INT64, `national_char_varying_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NATIONAL_CHAR_VARYING_PK_TABLE` (`national_char_varying_pk_col` STRING(MAX), `dummy_col` STRING(MAX)) PRIMARY KEY (`national_char_varying_pk_col`); +CREATE TABLE `NUMERIC_TO_NUMBER_TABLE` (`id` INT64, `number_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_NUMBER_TABLE` (`id` INT64, `number_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NUMBER_TABLE` (`id` INT64, `number_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_NUMBER_TABLE` (`id` INT64, `number_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_NUMERIC_TABLE` (`id` INT64, `numeric_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_NUMERIC_TABLE` (`id` INT64, `numeric_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NUMERIC_TABLE` (`id` INT64, `numeric_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_NUMERIC_TABLE` (`id` INT64, `numeric_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_DECIMAL_TABLE` (`id` INT64, `decimal_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_DECIMAL_TABLE` (`id` INT64, `decimal_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_DECIMAL_TABLE` (`id` INT64, `decimal_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_DECIMAL_TABLE` (`id` INT64, `decimal_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_DEC_TABLE` (`id` INT64, `dec_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_DEC_TABLE` (`id` INT64, `dec_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_DEC_TABLE` (`id` INT64, `dec_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_DEC_TABLE` (`id` INT64, `dec_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_FLOAT_TABLE` (`id` INT64, `float_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_FLOAT_TABLE` (`id` INT64, `float_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_FLOAT_TABLE` (`id` INT64, `float_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_FLOAT_TABLE` (`id` INT64, `float_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_DOUBLE_PRECISION_TABLE` (`id` INT64, `double_precision_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_DOUBLE_PRECISION_TABLE` (`id` INT64, `double_precision_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_DOUBLE_PRECISION_TABLE` (`id` INT64, `double_precision_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_DOUBLE_PRECISION_TABLE` (`id` INT64, `double_precision_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_REAL_TABLE` (`id` INT64, `real_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_REAL_TABLE` (`id` INT64, `real_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_REAL_TABLE` (`id` INT64, `real_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_REAL_TABLE` (`id` INT64, `real_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT32_TO_BINARY_FLOAT_TABLE` (`id` INT64, `binary_float_col` FLOAT32) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_BINARY_FLOAT_TABLE` (`id` INT64, `binary_float_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_BINARY_FLOAT_TABLE` (`id` INT64, `binary_float_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_BINARY_FLOAT_TABLE` (`id` INT64, `binary_float_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_BINARY_FLOAT_TABLE` (`id` INT64, `binary_float_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_BINARY_DOUBLE_TABLE` (`id` INT64, `binary_double_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_BINARY_DOUBLE_TABLE` (`id` INT64, `binary_double_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `NUMERIC_TO_BINARY_DOUBLE_TABLE` (`id` INT64, `binary_double_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_BINARY_DOUBLE_TABLE` (`id` INT64, `binary_double_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_INTEGER_TABLE` (`id` INT64, `integer_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_INTEGER_PK_TABLE` (`integer_pk_col` INT64, `dummy_col` STRING(MAX)) PRIMARY KEY (`integer_pk_col`); +CREATE TABLE `NUMERIC_TO_INTEGER_TABLE` (`id` INT64, `integer_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_INTEGER_TABLE` (`id` INT64, `integer_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_INTEGER_TABLE` (`id` INT64, `integer_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_INT_TABLE` (`id` INT64, `int_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_INT_PK_TABLE` (`int_pk_col` INT64, `dummy_col` STRING(MAX)) PRIMARY KEY (`int_pk_col`); +CREATE TABLE `NUMERIC_TO_INT_TABLE` (`id` INT64, `int_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_INT_TABLE` (`id` INT64, `int_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_INT_TABLE` (`id` INT64, `int_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_SMALLINT_TABLE` (`id` INT64, `smallint_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `INT64_TO_SMALLINT_PK_TABLE` (`smallint_pk_col` INT64, `dummy_col` STRING(MAX)) PRIMARY KEY (`smallint_pk_col`); +CREATE TABLE `NUMERIC_TO_SMALLINT_TABLE` (`id` INT64, `smallint_col` NUMERIC) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_SMALLINT_TABLE` (`id` INT64, `smallint_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `FLOAT64_TO_SMALLINT_TABLE` (`id` INT64, `smallint_col` FLOAT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_CLOB_TABLE` (`id` INT64, `clob_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_NCLOB_TABLE` (`id` INT64, `nclob_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE TABLE `BOOL_TO_BOOLEAN_TABLE` (`id` INT64, `boolean_col` BOOL) PRIMARY KEY (`id`); +CREATE TABLE `BOOL_TO_BOOLEAN_PK_TABLE` (`boolean_pk_col` BOOL, `dummy_col` STRING(MAX)) PRIMARY KEY (`boolean_pk_col`); +CREATE TABLE `INT64_TO_BOOLEAN_TABLE` (`id` INT64, `boolean_col` INT64) PRIMARY KEY (`id`); +CREATE TABLE `STRING_TO_BOOLEAN_TABLE` (`id` INT64, `boolean_col` STRING(MAX)) PRIMARY KEY (`id`); +CREATE CHANGE STREAM allstream FOR ALL OPTIONS (value_capture_type = 'NEW_ROW', retention_period = '7d'); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-schema.sql new file mode 100644 index 0000000000..bfcf3602cd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/oracle-schema.sql @@ -0,0 +1,78 @@ +CREATE TABLE "STRING_TO_VARCHAR2_TABLE" ("id" NUMBER PRIMARY KEY, "varchar2_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_VARCHAR2_PK_TABLE" ("varchar2_pk_col" VARCHAR2(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_VARCHAR_TABLE" ("id" NUMBER PRIMARY KEY, "varchar_col" VARCHAR(255)); +CREATE TABLE "STRING_TO_VARCHAR_PK_TABLE" ("varchar_pk_col" VARCHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_CHAR_TABLE" ("id" NUMBER PRIMARY KEY, "char_col" CHAR(255)); +CREATE TABLE "STRING_TO_CHAR_PK_TABLE" ("char_pk_col" CHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_CHARACTER_TABLE" ("id" NUMBER PRIMARY KEY, "character_col" CHARACTER(255)); +CREATE TABLE "STRING_TO_CHARACTER_PK_TABLE" ("character_pk_col" CHARACTER(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NCHAR_TABLE" ("id" NUMBER PRIMARY KEY, "nchar_col" NCHAR(255)); +CREATE TABLE "STRING_TO_NCHAR_PK_TABLE" ("nchar_pk_col" NCHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NCHAR_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "nchar_varying_col" NCHAR VARYING(255)); +CREATE TABLE "STRING_TO_NCHAR_VARYING_PK_TABLE" ("nchar_varying_pk_col" NCHAR VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHARACTER_TABLE" ("id" NUMBER PRIMARY KEY, "national_character_col" NATIONAL CHARACTER(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHARACTER_PK_TABLE" ("national_character_pk_col" NATIONAL CHARACTER(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHAR_TABLE" ("id" NUMBER PRIMARY KEY, "national_char_col" NATIONAL CHAR(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHAR_PK_TABLE" ("national_char_pk_col" NATIONAL CHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHARACTER_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "national_character_varying_col" NATIONAL CHARACTER VARYING(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE" ("national_character_varying_pk_col" NATIONAL CHARACTER VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHAR_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "national_char_varying_col" NATIONAL CHAR VARYING(255)); +CREATE TABLE "STRING_TO_NATIONAL_CHAR_VARYING_PK_TABLE" ("national_char_varying_pk_col" NATIONAL CHAR VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "FLOAT64_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "STRING_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "INT64_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "NUMERIC_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "FLOAT64_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "STRING_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "INT64_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "NUMERIC_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "FLOAT64_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "STRING_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "INT64_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "NUMERIC_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "FLOAT64_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "STRING_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "INT64_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "NUMERIC_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "FLOAT64_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "STRING_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "INT64_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "FLOAT64_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "NUMERIC_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "STRING_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "INT64_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "FLOAT64_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "STRING_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "NUMERIC_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "INT64_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "FLOAT32_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "FLOAT64_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "STRING_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "NUMERIC_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "INT64_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "FLOAT64_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "STRING_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "NUMERIC_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "INT64_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "INT64_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "INT64_TO_INTEGER_PK_TABLE" ("integer_pk_col" INTEGER PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "STRING_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "FLOAT64_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "INT64_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "INT64_TO_INT_PK_TABLE" ("int_pk_col" INT PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "STRING_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "FLOAT64_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "INT64_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "INT64_TO_SMALLINT_PK_TABLE" ("smallint_pk_col" SMALLINT PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "STRING_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "FLOAT64_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "STRING_TO_CLOB_TABLE" ("id" NUMBER PRIMARY KEY, "clob_col" CLOB); +CREATE TABLE "STRING_TO_NCLOB_TABLE" ("id" NUMBER PRIMARY KEY, "nclob_col" NCLOB); +CREATE TABLE "BOOL_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); +CREATE TABLE "BOOL_TO_BOOLEAN_PK_TABLE" ("boolean_pk_col" NUMBER(1) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "INT64_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); +CREATE TABLE "STRING_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/session.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesIT/session.json @@ -0,0 +1 @@ +{} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-postgresql-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-postgresql-spanner-schema.sql new file mode 100644 index 0000000000..1643af1d4a --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-postgresql-spanner-schema.sql @@ -0,0 +1,76 @@ +CREATE TABLE "VARCHAR_TO_VARCHAR2_TABLE" ("id" BIGINT, "varchar2_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_VARCHAR2_PK_TABLE" ("varchar2_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("varchar2_pk_col")); +CREATE TABLE "VARCHAR_TO_VARCHAR_TABLE" ("id" BIGINT, "varchar_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_VARCHAR_PK_TABLE" ("varchar_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("varchar_pk_col")); +CREATE TABLE "VARCHAR_TO_CHAR_TABLE" ("id" BIGINT, "char_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_CHAR_PK_TABLE" ("char_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("char_pk_col")); +CREATE TABLE "VARCHAR_TO_CHARACTER_TABLE" ("id" BIGINT, "character_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_CHARACTER_PK_TABLE" ("character_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("character_pk_col")); +CREATE TABLE "VARCHAR_TO_NCHAR_TABLE" ("id" BIGINT, "nchar_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NCHAR_PK_TABLE" ("nchar_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("nchar_pk_col")); +CREATE TABLE "VARCHAR_TO_NCHAR_VARYING_TABLE" ("id" BIGINT, "nchar_varying_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NCHAR_VARYING_PK_TABLE" ("nchar_varying_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("nchar_varying_pk_col")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_TABLE" ("id" BIGINT, "national_character_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_PK_TABLE" ("national_character_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("national_character_pk_col")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_TABLE" ("id" BIGINT, "national_char_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_PK_TABLE" ("national_char_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("national_char_pk_col")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_TABLE" ("id" BIGINT, "national_character_varying_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE" ("national_character_varying_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("national_character_varying_pk_col")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_VARYING_TABLE" ("id" BIGINT, "national_char_varying_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_VARYING_PK_TABLE" ("national_char_varying_pk_col" VARCHAR, "dummy_col" VARCHAR(255), PRIMARY KEY ("national_char_varying_pk_col")); +CREATE TABLE "DOUBLE_PRECISION_TO_NUMBER_TABLE" ("id" BIGINT, "number_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_NUMBER_TABLE" ("id" BIGINT, "number_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NUMBER_TABLE" ("id" BIGINT, "number_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_NUMBER_TABLE" ("id" BIGINT, "number_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_NUMERIC_TABLE" ("id" BIGINT, "numeric_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_NUMERIC_TABLE" ("id" BIGINT, "numeric_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NUMERIC_TABLE" ("id" BIGINT, "numeric_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_NUMERIC_TABLE" ("id" BIGINT, "numeric_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_DECIMAL_TABLE" ("id" BIGINT, "decimal_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_DECIMAL_TABLE" ("id" BIGINT, "decimal_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_DECIMAL_TABLE" ("id" BIGINT, "decimal_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_DECIMAL_TABLE" ("id" BIGINT, "decimal_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_DEC_TABLE" ("id" BIGINT, "dec_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_DEC_TABLE" ("id" BIGINT, "dec_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_DEC_TABLE" ("id" BIGINT, "dec_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_DEC_TABLE" ("id" BIGINT, "dec_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_FLOAT_TABLE" ("id" BIGINT, "float_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_FLOAT_TABLE" ("id" BIGINT, "float_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_FLOAT_TABLE" ("id" BIGINT, "float_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_DOUBLE_PRECISION_TABLE" ("id" BIGINT, "double_precision_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_DOUBLE_PRECISION_TABLE" ("id" BIGINT, "double_precision_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_DOUBLE_PRECISION_TABLE" ("id" BIGINT, "double_precision_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_DOUBLE_PRECISION_TABLE" ("id" BIGINT, "double_precision_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_REAL_TABLE" ("id" BIGINT, "real_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_REAL_TABLE" ("id" BIGINT, "real_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_REAL_TABLE" ("id" BIGINT, "real_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_REAL_TABLE" ("id" BIGINT, "real_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "REAL_TO_BINARY_FLOAT_TABLE" ("id" BIGINT, "binary_float_col" REAL, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_BINARY_FLOAT_TABLE" ("id" BIGINT, "binary_float_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_BINARY_FLOAT_TABLE" ("id" BIGINT, "binary_float_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_BINARY_FLOAT_TABLE" ("id" BIGINT, "binary_float_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_BINARY_DOUBLE_TABLE" ("id" BIGINT, "binary_double_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_BINARY_DOUBLE_TABLE" ("id" BIGINT, "binary_double_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "NUMERIC_TO_BINARY_DOUBLE_TABLE" ("id" BIGINT, "binary_double_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_INTEGER_TABLE" ("id" BIGINT, "integer_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_INTEGER_PK_TABLE" ("integer_pk_col" BIGINT, "dummy_col" VARCHAR(255), PRIMARY KEY ("integer_pk_col")); +CREATE TABLE "NUMERIC_TO_INTEGER_TABLE" ("id" BIGINT, "integer_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_INTEGER_TABLE" ("id" BIGINT, "integer_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_INTEGER_TABLE" ("id" BIGINT, "integer_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_INT_TABLE" ("id" BIGINT, "int_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_INT_PK_TABLE" ("int_pk_col" BIGINT, "dummy_col" VARCHAR(255), PRIMARY KEY ("int_pk_col")); +CREATE TABLE "NUMERIC_TO_INT_TABLE" ("id" BIGINT, "int_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_INT_TABLE" ("id" BIGINT, "int_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_INT_TABLE" ("id" BIGINT, "int_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_SMALLINT_TABLE" ("id" BIGINT, "smallint_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "BIGINT_TO_SMALLINT_PK_TABLE" ("smallint_pk_col" BIGINT, "dummy_col" VARCHAR(255), PRIMARY KEY ("smallint_pk_col")); +CREATE TABLE "NUMERIC_TO_SMALLINT_TABLE" ("id" BIGINT, "smallint_col" NUMERIC, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_SMALLINT_TABLE" ("id" BIGINT, "smallint_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "DOUBLE_PRECISION_TO_SMALLINT_TABLE" ("id" BIGINT, "smallint_col" DOUBLE PRECISION, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_CLOB_TABLE" ("id" BIGINT, "clob_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_NCLOB_TABLE" ("id" BIGINT, "nclob_col" VARCHAR, PRIMARY KEY ("id")); +CREATE TABLE "BOOLEAN_TO_BOOLEAN_TABLE" ("id" BIGINT, "boolean_col" BOOLEAN, PRIMARY KEY ("id")); +CREATE TABLE "BOOLEAN_TO_BOOLEAN_PK_TABLE" ("boolean_pk_col" BOOLEAN, "dummy_col" VARCHAR(255), PRIMARY KEY ("boolean_pk_col")); +CREATE TABLE "BIGINT_TO_BOOLEAN_TABLE" ("id" BIGINT, "boolean_col" BIGINT, PRIMARY KEY ("id")); +CREATE TABLE "VARCHAR_TO_BOOLEAN_TABLE" ("id" BIGINT, "boolean_col" VARCHAR, PRIMARY KEY ("id")); +CREATE CHANGE STREAM allstream FOR ALL WITH (value_capture_type = 'NEW_ROW', retention_period = '7d'); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-schema.sql new file mode 100644 index 0000000000..c13fa362ce --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/oracle-schema.sql @@ -0,0 +1,75 @@ +CREATE TABLE "VARCHAR_TO_VARCHAR2_TABLE" ("id" NUMBER PRIMARY KEY, "varchar2_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_VARCHAR2_PK_TABLE" ("varchar2_pk_col" VARCHAR2(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_VARCHAR_TABLE" ("id" NUMBER PRIMARY KEY, "varchar_col" VARCHAR(255)); +CREATE TABLE "VARCHAR_TO_VARCHAR_PK_TABLE" ("varchar_pk_col" VARCHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_CHAR_TABLE" ("id" NUMBER PRIMARY KEY, "char_col" CHAR(255)); +CREATE TABLE "VARCHAR_TO_CHAR_PK_TABLE" ("char_pk_col" CHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_CHARACTER_TABLE" ("id" NUMBER PRIMARY KEY, "character_col" CHARACTER(255)); +CREATE TABLE "VARCHAR_TO_CHARACTER_PK_TABLE" ("character_pk_col" CHARACTER(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NCHAR_TABLE" ("id" NUMBER PRIMARY KEY, "nchar_col" NCHAR(255)); +CREATE TABLE "VARCHAR_TO_NCHAR_PK_TABLE" ("nchar_pk_col" NCHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NCHAR_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "nchar_varying_col" NCHAR VARYING(255)); +CREATE TABLE "VARCHAR_TO_NCHAR_VARYING_PK_TABLE" ("nchar_varying_pk_col" NCHAR VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_TABLE" ("id" NUMBER PRIMARY KEY, "national_character_col" NATIONAL CHARACTER(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_PK_TABLE" ("national_character_pk_col" NATIONAL CHARACTER(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_TABLE" ("id" NUMBER PRIMARY KEY, "national_char_col" NATIONAL CHAR(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_PK_TABLE" ("national_char_pk_col" NATIONAL CHAR(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "national_character_varying_col" NATIONAL CHARACTER VARYING(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHARACTER_VARYING_PK_TABLE" ("national_character_varying_pk_col" NATIONAL CHARACTER VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_VARYING_TABLE" ("id" NUMBER PRIMARY KEY, "national_char_varying_col" NATIONAL CHAR VARYING(255)); +CREATE TABLE "VARCHAR_TO_NATIONAL_CHAR_VARYING_PK_TABLE" ("national_char_varying_pk_col" NATIONAL CHAR VARYING(255) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "DOUBLE_PRECISION_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "NUMERIC_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "VARCHAR_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "BIGINT_TO_NUMBER_TABLE" ("id" NUMBER PRIMARY KEY, "number_col" NUMBER); +CREATE TABLE "NUMERIC_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "DOUBLE_PRECISION_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "VARCHAR_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "BIGINT_TO_NUMERIC_TABLE" ("id" NUMBER PRIMARY KEY, "numeric_col" NUMERIC); +CREATE TABLE "NUMERIC_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "DOUBLE_PRECISION_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "VARCHAR_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "BIGINT_TO_DECIMAL_TABLE" ("id" NUMBER PRIMARY KEY, "decimal_col" DECIMAL); +CREATE TABLE "NUMERIC_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "DOUBLE_PRECISION_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "VARCHAR_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "BIGINT_TO_DEC_TABLE" ("id" NUMBER PRIMARY KEY, "dec_col" DEC); +CREATE TABLE "DOUBLE_PRECISION_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "NUMERIC_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "VARCHAR_TO_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "float_col" FLOAT); +CREATE TABLE "DOUBLE_PRECISION_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "NUMERIC_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "VARCHAR_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "BIGINT_TO_DOUBLE_PRECISION_TABLE" ("id" NUMBER PRIMARY KEY, "double_precision_col" DOUBLE PRECISION); +CREATE TABLE "DOUBLE_PRECISION_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "NUMERIC_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "VARCHAR_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "BIGINT_TO_REAL_TABLE" ("id" NUMBER PRIMARY KEY, "real_col" REAL); +CREATE TABLE "REAL_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "DOUBLE_PRECISION_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "VARCHAR_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "NUMERIC_TO_BINARY_FLOAT_TABLE" ("id" NUMBER PRIMARY KEY, "binary_float_col" BINARY_FLOAT); +CREATE TABLE "DOUBLE_PRECISION_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "VARCHAR_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "NUMERIC_TO_BINARY_DOUBLE_TABLE" ("id" NUMBER PRIMARY KEY, "binary_double_col" BINARY_DOUBLE); +CREATE TABLE "BIGINT_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "BIGINT_TO_INTEGER_PK_TABLE" ("integer_pk_col" INTEGER PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "VARCHAR_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "DOUBLE_PRECISION_TO_INTEGER_TABLE" ("id" NUMBER PRIMARY KEY, "integer_col" INTEGER); +CREATE TABLE "BIGINT_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "BIGINT_TO_INT_PK_TABLE" ("int_pk_col" INT PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "VARCHAR_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "DOUBLE_PRECISION_TO_INT_TABLE" ("id" NUMBER PRIMARY KEY, "int_col" INT); +CREATE TABLE "BIGINT_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "BIGINT_TO_SMALLINT_PK_TABLE" ("smallint_pk_col" SMALLINT PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "NUMERIC_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "VARCHAR_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "DOUBLE_PRECISION_TO_SMALLINT_TABLE" ("id" NUMBER PRIMARY KEY, "smallint_col" SMALLINT); +CREATE TABLE "VARCHAR_TO_CLOB_TABLE" ("id" NUMBER PRIMARY KEY, "clob_col" CLOB); +CREATE TABLE "VARCHAR_TO_NCLOB_TABLE" ("id" NUMBER PRIMARY KEY, "nclob_col" NCLOB); +CREATE TABLE "BOOLEAN_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); +CREATE TABLE "BOOLEAN_TO_BOOLEAN_PK_TABLE" ("boolean_pk_col" NUMBER(1) PRIMARY KEY, "dummy_col" VARCHAR2(255)); +CREATE TABLE "BIGINT_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); +CREATE TABLE "VARCHAR_TO_BOOLEAN_TABLE" ("id" NUMBER PRIMARY KEY, "boolean_col" NUMBER(1)); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/writes.txt b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/writes.txt new file mode 100644 index 0000000000..be7a17aeee --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDataTypesPGDialectIT/writes.txt @@ -0,0 +1,48 @@ + spannerResourceManager.write(Mutation.newInsertBuilder("varchar2_table").set("id").to(10L).set("varchar2_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar2_table").set("id").to(11L).set("varchar2_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar2_table").set("id").to(12L).set("varchar2_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar2_table").set("id").to(13L).set("varchar2_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar_table").set("id").to(10L).set("varchar_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar_table").set("id").to(11L).set("varchar_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar_table").set("id").to(12L).set("varchar_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("varchar_table").set("id").to(13L).set("varchar_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("char_table").set("id").to(10L).set("char_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("char_table").set("id").to(11L).set("char_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("char_table").set("id").to(12L).set("char_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("char_table").set("id").to(13L).set("char_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("character_table").set("id").to(10L).set("character_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("character_table").set("id").to(11L).set("character_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("character_table").set("id").to(12L).set("character_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("character_table").set("id").to(13L).set("character_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nvarchar2_table").set("id").to(10L).set("nvarchar2_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nvarchar2_table").set("id").to(11L).set("nvarchar2_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nvarchar2_table").set("id").to(12L).set("nvarchar2_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nvarchar2_table").set("id").to(13L).set("nvarchar2_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_table").set("id").to(10L).set("nchar_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_table").set("id").to(11L).set("nchar_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_table").set("id").to(12L).set("nchar_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_table").set("id").to(13L).set("nchar_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_varying_table").set("id").to(10L).set("nchar_varying_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_varying_table").set("id").to(11L).set("nchar_varying_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_varying_table").set("id").to(12L).set("nchar_varying_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("nchar_varying_table").set("id").to(13L).set("nchar_varying_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_table").set("id").to(10L).set("national_character_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_table").set("id").to(11L).set("national_character_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_table").set("id").to(12L).set("national_character_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_table").set("id").to(13L).set("national_character_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_table").set("id").to(10L).set("national_char_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_table").set("id").to(11L).set("national_char_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_table").set("id").to(12L).set("national_char_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_table").set("id").to(13L).set("national_char_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_varying_table").set("id").to(10L).set("national_character_varying_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_varying_table").set("id").to(11L).set("national_character_varying_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_varying_table").set("id").to(12L).set("national_character_varying_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_character_varying_table").set("id").to(13L).set("national_character_varying_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_varying_table").set("id").to(10L).set("national_char_varying_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_varying_table").set("id").to(11L).set("national_char_varying_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_varying_table").set("id").to(12L).set("national_char_varying_col").to("DROP TABLE").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("national_char_varying_table").set("id").to(13L).set("national_char_varying_col").to("<32767_A_CHARACTERS>").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("raw_table").set("id").to(10L).set("raw_col").to("").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("raw_table").set("id").to(11L).set("raw_col").to(" ").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("raw_table").set("id").to(12L).set("raw_col").to("A").build()); + spannerResourceManager.write(Mutation.newInsertBuilder("raw_table").set("id").to(13L).set("raw_col").to("DROP TABLE").build()); \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/oracle-schema.sql new file mode 100644 index 0000000000..1876a99f9a --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/oracle-schema.sql @@ -0,0 +1,174 @@ +CREATE TABLE "default_types_table" ( + "id" VARCHAR2(255) PRIMARY KEY, + "varchar2_col" VARCHAR2(255), + "varchar_col" VARCHAR(255), + "char_col" CHAR(255), + "character_col" CHARACTER(255), + "nvarchar2_col" NVARCHAR2(255), + "nchar_col" NCHAR(255), + "nchar_varying_col" NCHAR VARYING(255), + "national_character_col" NATIONAL CHARACTER(255), + "national_char_col" NATIONAL CHAR(255), + "national_character_varying_col" NATIONAL CHARACTER VARYING(255), + "national_char_varying_col" NATIONAL CHAR VARYING(255), + "number_col" NUMBER, + "numeric_col" NUMERIC, + "decimal_col" DECIMAL, + "dec_col" DEC, + "float_col" FLOAT, + "double_precision_col" DOUBLE PRECISION, + "real_col" REAL, + "binary_float_col" BINARY_FLOAT, + "binary_double_col" BINARY_DOUBLE, + "integer_col" INTEGER, + "int_col" INT, + "smallint_col" SMALLINT, + "date_col" DATE, + "timestamp_col" TIMESTAMP, + "timestamp_with_time_zone_col" TIMESTAMP WITH TIME ZONE, + "timestamp_with_local_time_zone_col" TIMESTAMP WITH LOCAL TIME ZONE, + "raw_col" RAW(255), + "blob_col" BLOB, + "clob_col" CLOB, + "nclob_col" NCLOB, + "boolean_col" NUMBER(1), + "json_col" CLOB +); + +CREATE TABLE "alt_types_table" ( + "id" VARCHAR2(255) PRIMARY KEY, + "varchar2_to_string_col" VARCHAR2(255), + "varchar_to_string_col" VARCHAR(255), + "char_to_string_col" CHAR(255), + "character_to_string_col" CHARACTER(255), + "nvarchar2_to_string_col" NVARCHAR2(255), + "nchar_to_string_col" NCHAR(255), + "nchar_varying_to_string_col" NCHAR VARYING(255), + "national_character_to_string_col" NATIONAL CHARACTER(255), + "national_char_to_string_col" NATIONAL CHAR(255), + "national_character_varying_to_string_col" NATIONAL CHARACTER VARYING(255), + "national_char_varying_to_string_col" NATIONAL CHAR VARYING(255), + "number_to_float64_col" NUMBER, + "numeric_to_float64_col" NUMERIC, + "decimal_to_float64_col" DECIMAL, + "dec_to_float64_col" DEC, + "float_to_float64_col" FLOAT, + "double_precision_to_numeric_col" DOUBLE PRECISION, + "real_to_string_col" REAL, + "binary_float_to_float64_col" BINARY_FLOAT, + "binary_double_to_string_col" BINARY_DOUBLE, + "integer_to_numeric_col" INTEGER, + "int_to_numeric_col" INT, + "smallint_to_numeric_col" SMALLINT, + "date_to_date_col" DATE, + "timestamp_to_string_col" TIMESTAMP, + "timestamp_with_time_zone_to_string_col" TIMESTAMP WITH TIME ZONE, + "timestamp_with_local_time_zone_to_string_col" TIMESTAMP WITH LOCAL TIME ZONE, + "raw_to_bytes_col" RAW(255), + "blob_to_string_col" BLOB, + "clob_to_bytes_col" CLOB, + "nclob_to_bytes_col" NCLOB, + "boolean_to_int64_col" NUMBER(1), + "json_to_string_col" CLOB +); + +CREATE TABLE "varchar2_pk_table" ( + "varchar2_pk_col" VARCHAR2(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "varchar_pk_table" ( + "varchar_pk_col" VARCHAR(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "char_pk_table" ( + "char_pk_col" CHAR(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "character_pk_table" ( + "character_pk_col" CHARACTER(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "nvarchar2_pk_table" ( + "nvarchar2_pk_col" NVARCHAR2(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "nchar_pk_table" ( + "nchar_pk_col" NCHAR(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "nchar_varying_pk_table" ( + "nchar_varying_pk_col" NCHAR VARYING(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "national_character_pk_table" ( + "national_character_pk_col" NATIONAL CHARACTER(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "national_char_pk_table" ( + "national_char_pk_col" NATIONAL CHAR(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "national_character_varying_pk_table" ( + "national_character_varying_pk_col" NATIONAL CHARACTER VARYING(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "national_char_varying_pk_table" ( + "national_char_varying_pk_col" NATIONAL CHAR VARYING(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "number_pk_table" ( + "number_pk_col" NUMBER PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "dec_pk_table" ( + "dec_pk_col" DEC PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "integer_pk_table" ( + "integer_pk_col" INTEGER PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "int_pk_table" ( + "int_pk_col" INT PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "smallint_pk_table" ( + "smallint_pk_col" SMALLINT PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "date_pk_table" ( + "date_pk_col" DATE PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "timestamp_pk_table" ( + "timestamp_pk_col" TIMESTAMP PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "raw_pk_table" ( + "raw_pk_col" RAW(255) PRIMARY KEY, + "val" VARCHAR2(255) +); + +CREATE TABLE "boolean_pk_table" ( + "boolean_pk_col" NUMBER(1) PRIMARY KEY, + "val" VARCHAR2(255) +); + diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/spanner-schema.sql new file mode 100644 index 0000000000..28b1430cad --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDatatypesIT/spanner-schema.sql @@ -0,0 +1,182 @@ +CREATE TABLE default_types_table ( + id STRING(MAX) NOT NULL, + varchar2_col STRING(MAX), + varchar_col STRING(MAX), + char_col STRING(MAX), + character_col STRING(MAX), + nvarchar2_col STRING(MAX), + nchar_col STRING(MAX), + nchar_varying_col STRING(MAX), + national_character_col STRING(MAX), + national_char_col STRING(MAX), + national_character_varying_col STRING(MAX), + national_char_varying_col STRING(MAX), + number_col NUMERIC, + numeric_col NUMERIC, + decimal_col NUMERIC, + dec_col NUMERIC, + float_col NUMERIC, + double_precision_col FLOAT64, + real_col FLOAT64, + binary_float_col FLOAT32, + binary_double_col FLOAT64, + integer_col INT64, + int_col INT64, + smallint_col INT64, + date_col TIMESTAMP, + timestamp_col TIMESTAMP, + timestamp_with_time_zone_col TIMESTAMP, + timestamp_with_local_time_zone_col TIMESTAMP, + raw_col BYTES(MAX), + blob_col BYTES(MAX), + clob_col STRING(MAX), + nclob_col STRING(MAX), + boolean_col BOOL, + json_col JSON +) PRIMARY KEY(id); + +CREATE TABLE alt_types_table ( + id STRING(MAX) NOT NULL, + varchar2_to_string_col STRING(MAX), + varchar_to_string_col STRING(MAX), + char_to_string_col STRING(MAX), + character_to_string_col STRING(MAX), + nvarchar2_to_string_col STRING(MAX), + nchar_to_string_col STRING(MAX), + nchar_varying_to_string_col STRING(MAX), + national_character_to_string_col STRING(MAX), + national_char_to_string_col STRING(MAX), + national_character_varying_to_string_col STRING(MAX), + national_char_varying_to_string_col STRING(MAX), + number_to_float64_col FLOAT64, + numeric_to_float64_col FLOAT64, + decimal_to_float64_col FLOAT64, + dec_to_float64_col FLOAT64, + float_to_float64_col FLOAT64, + double_precision_to_numeric_col NUMERIC, + real_to_string_col STRING(MAX), + binary_float_to_float64_col FLOAT64, + binary_double_to_string_col STRING(MAX), + integer_to_numeric_col NUMERIC, + int_to_numeric_col NUMERIC, + smallint_to_numeric_col NUMERIC, + date_to_date_col DATE, + timestamp_to_string_col STRING(MAX), + timestamp_with_time_zone_to_string_col STRING(MAX), + timestamp_with_local_time_zone_to_string_col STRING(MAX), + raw_to_bytes_col BYTES(MAX), + blob_to_string_col STRING(MAX), + clob_to_bytes_col BYTES(MAX), + nclob_to_bytes_col BYTES(MAX), + boolean_to_int64_col INT64, + json_to_string_col STRING(MAX) +) PRIMARY KEY(id); + +CREATE TABLE varchar2_pk_table ( + varchar2_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(varchar2_pk_col); + +CREATE TABLE varchar_pk_table ( + varchar_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(varchar_pk_col); + +CREATE TABLE char_pk_table ( + char_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(char_pk_col); + +CREATE TABLE character_pk_table ( + character_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(character_pk_col); + +CREATE TABLE nvarchar2_pk_table ( + nvarchar2_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(nvarchar2_pk_col); + +CREATE TABLE nchar_pk_table ( + nchar_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(nchar_pk_col); + +CREATE TABLE nchar_varying_pk_table ( + nchar_varying_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(nchar_varying_pk_col); + +CREATE TABLE national_character_pk_table ( + national_character_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(national_character_pk_col); + +CREATE TABLE national_char_pk_table ( + national_char_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(national_char_pk_col); + +CREATE TABLE national_character_varying_pk_table ( + national_character_varying_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(national_character_varying_pk_col); + +CREATE TABLE national_char_varying_pk_table ( + national_char_varying_pk_col STRING(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(national_char_varying_pk_col); + +CREATE TABLE number_pk_table ( + number_pk_col NUMERIC NOT NULL, + val STRING(MAX) +) PRIMARY KEY(number_pk_col); + +CREATE TABLE dec_pk_table ( + dec_pk_col NUMERIC NOT NULL, + val STRING(MAX) +) PRIMARY KEY(dec_pk_col); + +CREATE TABLE integer_pk_table ( + integer_pk_col INT64 NOT NULL, + val STRING(MAX) +) PRIMARY KEY(integer_pk_col); + +CREATE TABLE int_pk_table ( + int_pk_col INT64 NOT NULL, + val STRING(MAX) +) PRIMARY KEY(int_pk_col); + +CREATE TABLE smallint_pk_table ( + smallint_pk_col INT64 NOT NULL, + val STRING(MAX) +) PRIMARY KEY(smallint_pk_col); + +CREATE TABLE date_pk_table ( + date_pk_col TIMESTAMP NOT NULL, + val STRING(MAX) +) PRIMARY KEY(date_pk_col); + +CREATE TABLE timestamp_pk_table ( + timestamp_pk_col TIMESTAMP NOT NULL, + val STRING(MAX) +) PRIMARY KEY(timestamp_pk_col); + + + +CREATE TABLE raw_pk_table ( + raw_pk_col BYTES(MAX) NOT NULL, + val STRING(MAX) +) PRIMARY KEY(raw_pk_col); + +CREATE TABLE boolean_pk_table ( + boolean_pk_col BOOL NOT NULL, + val STRING(MAX) +) PRIMARY KEY(boolean_pk_col); + + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d' +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..5860defcea --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS Users1; +DROP TABLE IF EXISTS AllDatatypeTransformation; + +CREATE TABLE IF NOT EXISTS Users1 ( + id INT64 NOT NULL, + name STRING(25), +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS AllDatatypeTransformation ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + int_column INT64, + bigint_column INT64, + float_column FLOAT64, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + blob_column BYTES(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY (varchar_column); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-schema.sql new file mode 100644 index 0000000000..081784f1b7 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/oracle-schema.sql @@ -0,0 +1,28 @@ +CREATE TABLE "Users1" ( + "id" INT NOT NULL, + "first_name" VARCHAR2(25), + "last_name" VARCHAR2(25), + PRIMARY KEY("id")); + +CREATE TABLE "AllDatatypeTransformation" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "source_only_pk" INT NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "int_column" INT, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" DECIMAL(10,2), + "datetime_column" TIMESTAMP, + "timestamp_column" TIMESTAMP, + "time_column" VARCHAR2(20), + "year_column" VARCHAR2(10), + "blob_column" BLOB, + "enum_column" VARCHAR2(20), + "bool_column" NUMBER(1), + "binary_column" RAW(150), + "bit_column" RAW(10), + PRIMARY KEY ("source_only_pk", "varchar_column") +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/session.json new file mode 100644 index 0000000000..c2fa4bbaa1 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleDbCustomTransformationIT/session.json @@ -0,0 +1,911 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "rr_write", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t113": { + "Name": "AllDatatypeTransformation", + "ColIds": [ + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132" + ], + "ShardIdColumn": "", + "ColDefs": { + "c115": { + "Name": "varchar_column", + "T": { + "Name": "STRING", + "Len": 20, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: varchar_column varchar(20)", + "Id": "c115", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c116": { + "Name": "tinyint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_column tinyint(3)", + "Id": "c116", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c117": { + "Name": "text_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_column text(65535)", + "Id": "c117", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c118": { + "Name": "date_column", + "T": { + "Name": "DATE", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_column date", + "Id": "c118", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c119": { + "Name": "int_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_column int(10)", + "Id": "c119", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c120": { + "Name": "bigint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_column bigint(19)", + "Id": "c120", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c121": { + "Name": "float_column", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_column float(10,2)", + "Id": "c121", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c122": { + "Name": "double_column", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_column double(22)", + "Id": "c122", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c123": { + "Name": "decimal_column", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_column decimal(10,2)", + "Id": "c123", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c124": { + "Name": "datetime_column", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_column datetime", + "Id": "c124", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c125": { + "Name": "timestamp_column", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_column timestamp", + "Id": "c125", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c126": { + "Name": "time_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_column time", + "Id": "c126", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c127": { + "Name": "year_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_column year", + "Id": "c127", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c128": { + "Name": "blob_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_column blob(65535)", + "Id": "c128", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c129": { + "Name": "enum_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_column enum(1)", + "Id": "c129", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c130": { + "Name": "bool_column", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_column tinyint(1)", + "Id": "c130", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c131": { + "Name": "binary_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_column binary(20)", + "Id": "c131", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c132": { + "Name": "bit_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_column bit(7)", + "Id": "c132", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c115", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table AllDatatypeTransformation", + "Id": "t113" + }, + "t114": { + "Name": "Users1", + "ColIds": [ + "c133", + "c134" + ], + "ShardIdColumn": "", + "ColDefs": { + "c133": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c133", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c134": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c134", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c133", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t114" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t113": { + "Name": "AllDatatypeTransformation", + "Schema": "", + "ColIds": [ + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c136" + ], + "ColDefs": { + "c136": { + "Name": "source_only_pk", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c136" + }, + "c115": { + "Name": "varchar_column", + "Type": { + "Name": "varchar", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c115" + }, + "c116": { + "Name": "tinyint_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 3 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c116" + }, + "c117": { + "Name": "text_column", + "Type": { + "Name": "text", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c117" + }, + "c118": { + "Name": "date_column", + "Type": { + "Name": "date", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c118" + }, + "c119": { + "Name": "int_column", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c119" + }, + "c120": { + "Name": "bigint_column", + "Type": { + "Name": "bigint", + "Mods": [ + 19 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c120" + }, + "c121": { + "Name": "float_column", + "Type": { + "Name": "float", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c121" + }, + "c122": { + "Name": "double_column", + "Type": { + "Name": "double", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c122" + }, + "c123": { + "Name": "decimal_column", + "Type": { + "Name": "decimal", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c123" + }, + "c124": { + "Name": "datetime_column", + "Type": { + "Name": "datetime", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c124" + }, + "c125": { + "Name": "timestamp_column", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c125" + }, + "c126": { + "Name": "time_column", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c126" + }, + "c127": { + "Name": "year_column", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c127" + }, + "c128": { + "Name": "blob_column", + "Type": { + "Name": "blob", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c128" + }, + "c129": { + "Name": "enum_column", + "Type": { + "Name": "enum", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c129" + }, + "c130": { + "Name": "bool_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c130" + }, + "c131": { + "Name": "binary_column", + "Type": { + "Name": "binary", + "Mods": [ + 150 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c131" + }, + "c132": { + "Name": "bit_column", + "Type": { + "Name": "bit", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c132" + } + }, + "PrimaryKeys": [ + { + "ColId": "c136", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c115", + "Desc": false, + "Order": 2 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t113" + }, + "t114": { + "Name": "Users1", + "Schema": "", + "ColIds": [ + "c133", + "c134", + "c135" + ], + "ColDefs": { + "c133": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c133" + }, + "c134": { + "Name": "first_name", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c134" + }, + "c135": { + "Name": "last_name", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c135" + } + }, + "PrimaryKeys": [ + { + "ColId": "c133", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t114" + } + }, + "SchemaIssues": { + "t113": { + "ColumnLevelIssues": { + "c116": [ + 14 + ], + "c119": [ + 14 + ], + "c121": [ + 14 + ], + "c124": [ + 13 + ], + "c126": [ + 15 + ], + "c127": [ + 15 + ] + }, + "TableLevelIssues": null + }, + "t114": { + "ColumnLevelIssues": { + "c133": [ + 14 + ] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/file-overrides.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/file-overrides.json new file mode 100644 index 0000000000..ddb2ed4acb --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/file-overrides.json @@ -0,0 +1,13 @@ +{ + "renamedTables": { + "source_table1": "Target_Table_1" + }, + "renamedColumns": { + "source_table1": { + "name_col1": "Target_Name_Col_1" + }, + "source_table2": { + "category_col2": "Target_Category_Col_2" + } + } +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..0ecd26b9b7 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE Target_Table_1 ( + id_col1 INT64 NOT NULL, + Target_Name_Col_1 STRING(255), + data_col1 STRING(MAX) +) PRIMARY KEY (id_col1); + +CREATE TABLE source_table2 ( + key_col2 STRING(50) NOT NULL, + Target_Category_Col_2 STRING(100), + value_col2 STRING(MAX) +) PRIMARY KEY (key_col2); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d' +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-schema.sql new file mode 100644 index 0000000000..51f1ddf7fd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleFileOverridesSchemaMapperIT/oracle-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE "source_table1" ( + "id_col1" INT PRIMARY KEY, + "name_col1" VARCHAR2(255), + "data_col1" CLOB +); + +CREATE TABLE "source_table2" ( + "key_col2" VARCHAR2(50) PRIMARY KEY, + "category_col2" VARCHAR2(100), + "value_col2" CLOB +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..f1d710efec --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,66 @@ +CREATE TABLE IF NOT EXISTS Users ( + id INT64 NOT NULL, + full_name STRING(25), + `from` STRING(25) +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS Users2 ( + id INT64 NOT NULL, + name STRING(25), + ) PRIMARY KEY(id); + +CREATE TABLE TableWithVirtualGeneratedColumn ( + id INT64 NOT NULL, + column1 INT64, + virtual_generated_column INT64 AS (column1 + id), +) PRIMARY KEY(id); + +CREATE TABLE TableWithStoredGeneratedColumn ( + id INT64 NOT NULL, + column1 INT64, + stored_generated_column INT64 AS (column1 + id) STORED, +) PRIMARY KEY(id); + + CREATE TABLE IF NOT EXISTS testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO( + id INT64 NOT NULL, + col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY STRING(25), + ) PRIMARY KEY(id); + +CREATE TABLE TableWithIdentityColumn ( + id INT64 NOT NULL GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE), + column1 STRING(25), +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS `generated_pk_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, +) PRIMARY KEY (`generated_column_col`); + +CREATE TABLE IF NOT EXISTS `generated_non_pk_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, + `id` INT64 not null, +) PRIMARY KEY (`id`); + +CREATE TABLE IF NOT EXISTS `non_generated_to_generated_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, + `generated_column_pk_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, +) PRIMARY KEY (`generated_column_pk_col`); + +CREATE TABLE IF NOT EXISTS `generated_to_non_generated_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) DEFAULT(NULL), + `generated_column_pk_col` STRING(100) DEFAULT(NULL), +) PRIMARY KEY (`generated_column_pk_col`); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-schema.sql new file mode 100644 index 0000000000..31b5268349 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/oracle-schema.sql @@ -0,0 +1,66 @@ +CREATE TABLE "Users" ( + "id" INT NOT NULL, + "name" VARCHAR(25), + "from" VARCHAR(25), + PRIMARY KEY("id")); + +CREATE TABLE "Users2" ( + "id" INT NOT NULL, + "name" VARCHAR(25), + PRIMARY KEY("id")); + + CREATE TABLE "TableWithVirtualGeneratedColumn" ( + "id" INT NOT NULL, + "column1" INT, + "virtual_generated_column" INT GENERATED ALWAYS AS ("column1" + "id") VIRTUAL, + PRIMARY KEY("id") + ); + + CREATE TABLE "TableWithStoredGeneratedColumn" ( + "id" INT NOT NULL, + "column1" INT, + "stored_generated_column" INT GENERATED ALWAYS AS ("column1" + "id"), + PRIMARY KEY("id") + ); + + CREATE TABLE "testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO"( + "id" INT NOT NULL, + "col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY" VARCHAR(25), + PRIMARY KEY("id")); + + CREATE TABLE "TableWithIdentityColumn" ( + "id" INT GENERATED BY DEFAULT ON NULL AS IDENTITY NOT NULL, + "column1" VARCHAR(25), + PRIMARY KEY("id") + ); + +CREATE TABLE "generated_pk_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS ("first_name_col" || ' ') NOT NULL, + PRIMARY KEY ("generated_column_col") +); + +CREATE TABLE "generated_non_pk_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS ("first_name_col" || ' ') NOT NULL, + "id" INT NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE "non_generated_to_generated_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) NOT NULL, + "generated_column_pk_col" VARCHAR2(100) NOT NULL, + PRIMARY KEY ("generated_column_pk_col") +); + +CREATE TABLE "generated_to_non_generated_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS ("first_name_col" || ' ') NOT NULL, + "generated_column_pk_col" VARCHAR2(100) GENERATED ALWAYS AS (CAST("first_name_col" || ' ' AS VARCHAR2(100))) NOT NULL, + PRIMARY KEY ("generated_column_pk_col") +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/session.json new file mode 100644 index 0000000000..0abccb6a4d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleIT/session.json @@ -0,0 +1,2793 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "mysql", + "DatabaseName": "test_limits", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "Users", + "ColIds": [ + "c2", + "c3", + "c4" + ], + "ShardIdColumn": "", + "ColDefs": { + "c2": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c3": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c4": { + "Name": "from", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: from varchar(25)", + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table Users", + "Id": "t1" + }, + "t10": { + "Name": "TableWithStoredGeneratedColumn", + "ColIds": [ + "c11", + "c12", + "c14" + ], + "ShardIdColumn": "", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c12": { + "Name": "column1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: column1 int(11)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c14": { + "Name": "stored_generated_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: stored_generated_column int(11)", + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table TableWithStoredGeneratedColumn", + "Id": "t10" + }, + "t15": { + "Name": "testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO", + "ColIds": [ + "c16", + "c17" + ], + "ShardIdColumn": "", + "ColDefs": { + "c16": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c17": { + "Name": "col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY varchar(25)", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO", + "Id": "t15" + }, + "t18": { + "Name": "TableWithIdentityColumn", + "ColIds": [ + "c19", + "c20" + ], + "ShardIdColumn": "", + "ColDefs": { + "c19": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id bigint(20)", + "Id": "c19", + "AutoGen": { + "Name": "Identity", + "GenerationType": "Identity", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c20": { + "Name": "column1", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: column1 varchar(25)", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table TableWithIdentityColumn", + "Id": "t18" + }, + "t21": { + "Name": "generated_pk_column_table", + "ColIds": [ + "c22", + "c23", + "c25" + ], + "ShardIdColumn": "", + "ColDefs": { + "c22": { + "Name": "first_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: first_name_col varchar(50)", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c23": { + "Name": "last_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: last_name_col varchar(50)", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c25": { + "Name": "generated_column_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_col varchar(100)", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c25", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table generated_pk_column_table", + "Id": "t21" + }, + "t26": { + "Name": "generated_non_pk_column_table", + "ColIds": [ + "c27", + "c28", + "c30", + "c31" + ], + "ShardIdColumn": "", + "ColDefs": { + "c27": { + "Name": "first_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: first_name_col varchar(50)", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c28": { + "Name": "last_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: last_name_col varchar(50)", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c30": { + "Name": "generated_column_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_col varchar(100)", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c31": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c31", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table generated_non_pk_column_table", + "Id": "t26" + }, + "t32": { + "Name": "non_generated_to_generated_column_table", + "ColIds": [ + "c33", + "c34", + "c35", + "c36" + ], + "ShardIdColumn": "", + "ColDefs": { + "c33": { + "Name": "first_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: first_name_col varchar(50)", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c34": { + "Name": "last_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: last_name_col varchar(50)", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c35": { + "Name": "generated_column_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_col varchar(100)", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c36": { + "Name": "generated_column_pk_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_pk_col varchar(100)", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c36", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table non_generated_to_generated_column_table", + "Id": "t32" + }, + "t37": { + "Name": "generated_to_non_generated_column_table", + "ColIds": [ + "c38", + "c39", + "c41", + "c43" + ], + "ShardIdColumn": "", + "ColDefs": { + "c38": { + "Name": "first_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: first_name_col varchar(50)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c39": { + "Name": "last_name_col", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: last_name_col varchar(50)", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c41": { + "Name": "generated_column_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_col varchar(100)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c43": { + "Name": "generated_column_pk_col", + "T": { + "Name": "STRING", + "Len": 100, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: generated_column_pk_col varchar(100)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table generated_to_non_generated_column_table", + "Id": "t37" + }, + "t5": { + "Name": "TableWithVirtualGeneratedColumn", + "ColIds": [ + "c6", + "c7", + "c9" + ], + "ShardIdColumn": "", + "ColDefs": { + "c6": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c7": { + "Name": "column1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: column1 int(11)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c9": { + "Name": "virtual_generated_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: virtual_generated_column int(11)", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table TableWithVirtualGeneratedColumn", + "Id": "t5" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "Users", + "Schema": "", + "ColIds": [ + "c2", + "c3", + "c4" + ], + "ColDefs": { + "c2": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c3": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c4": { + "Name": "from", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t1" + }, + "t10": { + "Name": "TableWithStoredGeneratedColumn", + "Schema": "", + "ColIds": [ + "c11", + "c12", + "c14" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c12": { + "Name": "column1", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c14": { + "Name": "stored_generated_column", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e13", + "Statement": "column1+id" + }, + "Type": "STORED" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t10" + }, + "t15": { + "Name": "testtable_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvYZPAeGeqiO", + "Schema": "", + "ColIds": [ + "c16", + "c17" + ], + "ColDefs": { + "c16": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c17": { + "Name": "col_qcbF69RmXTRe3B_03TpCoVF16ED0KLxM3v808cH3bTGQ0uK_FEXuZHbttvY", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t15" + }, + "t18": { + "Name": "TableWithIdentityColumn", + "Schema": "", + "ColIds": [ + "c19", + "c20" + ], + "ColDefs": { + "c19": { + "Name": "id", + "Type": { + "Name": "bigint", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "Auto Increment", + "GenerationType": "Auto Increment", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c20": { + "Name": "column1", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t18" + }, + "t21": { + "Name": "generated_pk_column_table", + "Schema": "", + "ColIds": [ + "c22", + "c23", + "c25" + ], + "ColDefs": { + "c22": { + "Name": "first_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c23": { + "Name": "last_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c25": { + "Name": "generated_column_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e24", + "Statement": "CONCAT(first_name_col, _UTF8MB4' ')" + }, + "Type": "STORED" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c25", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t21" + }, + "t26": { + "Name": "generated_non_pk_column_table", + "Schema": "", + "ColIds": [ + "c27", + "c28", + "c30", + "c31" + ], + "ColDefs": { + "c27": { + "Name": "first_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c28": { + "Name": "last_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c30": { + "Name": "generated_column_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e29", + "Statement": "CONCAT(first_name_col, _UTF8MB4' ')" + }, + "Type": "STORED" + } + }, + "c31": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c31", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t26" + }, + "t32": { + "Name": "non_generated_to_generated_column_table", + "Schema": "", + "ColIds": [ + "c33", + "c34", + "c35", + "c36" + ], + "ColDefs": { + "c33": { + "Name": "first_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c34": { + "Name": "last_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c35": { + "Name": "generated_column_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c36": { + "Name": "generated_column_pk_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c36", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t32" + }, + "t37": { + "Name": "generated_to_non_generated_column_table", + "Schema": "", + "ColIds": [ + "c38", + "c39", + "c41", + "c43" + ], + "ColDefs": { + "c38": { + "Name": "first_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c39": { + "Name": "last_name_col", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c41": { + "Name": "generated_column_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e40", + "Statement": "CONCAT(first_name_col, _UTF8MB4' ')" + }, + "Type": "STORED" + } + }, + "c43": { + "Name": "generated_column_pk_col", + "Type": { + "Name": "varchar", + "Mods": [ + 100 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e42", + "Statement": "CONCAT(first_name_col, _UTF8MB4' ')" + }, + "Type": "STORED" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t37" + }, + "t5": { + "Name": "TableWithVirtualGeneratedColumn", + "Schema": "", + "ColIds": [ + "c6", + "c7", + "c9" + ], + "ColDefs": { + "c6": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c7": { + "Name": "column1", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c9": { + "Name": "virtual_generated_column", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": true, + "Value": { + "ExpressionId": "e8", + "Statement": "column1+id" + }, + "Type": "VIRTUAL" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t5" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c2": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t10": { + "ColumnLevelIssues": { + "c11": [ + 14 + ], + "c12": [ + 14 + ], + "c14": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t15": { + "ColumnLevelIssues": { + "c16": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t18": { + "ColumnLevelIssues": { + "c19": [ + 53 + ] + }, + "TableLevelIssues": null + }, + "t21": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + }, + "t26": { + "ColumnLevelIssues": { + "c31": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t32": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + }, + "t37": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + }, + "t5": { + "ColumnLevelIssues": { + "c6": [ + 14 + ], + "c7": [ + 14 + ], + "c9": [ + 14 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "Source": "mysql" +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3a1b988b71 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,52 @@ +CREATE TABLE IF NOT EXISTS parent1 ( + id INT64 NOT NULL, + update_ts TIMESTAMP, + in_ts TIMESTAMP, + migration_shard_id STRING(50), +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS parent2 ( + id INT64 NOT NULL, + update_ts TIMESTAMP, + in_ts TIMESTAMP, + migration_shard_id STRING(50), +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS child11 ( + child_id INT64 NOT NULL, + parent_id INT64, + update_ts TIMESTAMP, + in_ts TIMESTAMP, + migration_shard_id STRING(50), +) PRIMARY KEY(child_id); + +CREATE INDEX par_ind ON child11(parent_id); + +CREATE TABLE IF NOT EXISTS child21 ( + child_id INT64 NOT NULL, + id INT64 NOT NULL, + update_ts TIMESTAMP, + in_ts TIMESTAMP, + migration_shard_id STRING(50), +) PRIMARY KEY(id, child_id), + INTERLEAVE IN parent2; + +CREATE INDEX par_ind_5 ON child21(id); + +CREATE TABLE IF NOT EXISTS child31 ( + child_id INT64 NOT NULL, + id INT64 NOT NULL, + update_ts TIMESTAMP, + in_ts TIMESTAMP, + migration_shard_id STRING(50), + ) PRIMARY KEY(id, child_id), + INTERLEAVE IN PARENT parent2 ON DELETE CASCADE; + +CREATE INDEX par_ind_6 ON child31(id); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-schema.sql new file mode 100644 index 0000000000..15b9fe235e --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "parent1" ( "id" INT NOT NULL, "update_ts" TIMESTAMP DEFAULT NULL, "in_ts" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("id") ); +CREATE TABLE "child11" ( "child_id" INT NOT NULL, "parent_id" INT, "update_ts" TIMESTAMP DEFAULT NULL, "in_ts" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("child_id"), FOREIGN KEY ("parent_id") REFERENCES "parent1"("id") ); +CREATE INDEX "par_ind" ON "child11"("parent_id"); +CREATE TABLE "parent2" ( "id" INT NOT NULL, "update_ts" TIMESTAMP DEFAULT NULL, "in_ts" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("id") ); +CREATE TABLE "child21" ( "child_id" INT NOT NULL, "parent_id" INT, "update_ts" TIMESTAMP DEFAULT NULL, "in_ts" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("child_id"), FOREIGN KEY ("parent_id") REFERENCES "parent2"("id") ); +CREATE INDEX "par_ind_5" ON "child21"("parent_id"); +CREATE TABLE "child31" ( "child_id" INT NOT NULL, "parent_id" INT, "update_ts" TIMESTAMP DEFAULT NULL, "in_ts" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("child_id"), FOREIGN KEY ("parent_id") REFERENCES "parent2"("id") ); +CREATE INDEX "par_ind_6" ON "child31"("parent_id"); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/session.json new file mode 100644 index 0000000000..ad19ad0694 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleInterleaveMultiShardIT/session.json @@ -0,0 +1,1286 @@ +{ + "SpSchema": { + "t1": { + "Name": "child21", + "ColIds": [ + "c6", + "c7", + "c8", + "c9", + "c23" + ], + "ShardIdColumn": "c23", + "ColDefs": { + "c23": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c6": { + "Name": "child_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: child_id int(10)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c7": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: parent_id int(10)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c8": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c9": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [], + "Indexes": [ + { + "Name": "par_ind_5", + "TableId": "t1", + "Unique": false, + "Keys": [ + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "Id": "i11", + "StoredColumnIds": null + } + ], + "ParentId": "", + "Comment": "Spanner schema for source table child21", + "Id": "t1" + }, + "t2": { + "Name": "parent1", + "ColIds": [ + "c16", + "c17", + "c18", + "c24" + ], + "ShardIdColumn": "c24", + "ColDefs": { + "c16": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c17": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c18": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c24": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table parent1", + "Id": "t2" + }, + "t3": { + "Name": "child11", + "ColIds": [ + "c12", + "c13", + "c14", + "c15", + "c26" + ], + "ShardIdColumn": "c26", + "ColDefs": { + "c12": { + "Name": "child_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: child_id int(10)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "parent_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: parent_id int(10)", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c14": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c26": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c12", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child11_ibfk_1", + "ColIds": [ + "c26", + "c13" + ], + "ReferTableId": "t2", + "ReferColumnIds": [ + "c24", + "c16" + ], + "Id": "f10" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "TableId": "t3", + "Unique": false, + "Keys": [ + { + "ColId": "c13", + "Desc": false, + "Order": 1 + } + ], + "Id": "i22", + "StoredColumnIds": null + } + ], + "ParentId": "", + "Comment": "Spanner schema for source table child11", + "Id": "t3" + }, + "t4": { + "Name": "parent2", + "ColIds": [ + "c19", + "c20", + "c21", + "c25" + ], + "ShardIdColumn": "c25", + "ColDefs": { + "c19": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c20": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c21": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c25": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table parent2", + "Id": "t4" + }, + "t5": { + "Name": "child31", + "ColIds": [ + "c27", + "c28", + "c29", + "c30", + "c31" + ], + "ShardIdColumn": "c31", + "ColDefs": { + "c31": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c27": { + "Name": "child_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: child_id int(10)", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c28": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: parent_id int(10)", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c29": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c30": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c27", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c28", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [], + "Indexes": [ + { + "Name": "par_ind_6", + "TableId": "t5", + "Unique": false, + "Keys": [ + { + "ColId": "c28", + "Desc": false, + "Order": 1 + } + ], + "Id": "i12", + "StoredColumnIds": null + } + ], + "ParentId": "", + "Comment": "Spanner schema for source table child21", + "Id": "t5" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "child21", + "Schema": "", + "ColIds": [ + "c6", + "c7", + "c8", + "c9" + ], + "ColDefs": { + "c6": { + "Name": "child_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c7": { + "Name": "parent_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c8": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c9": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child21_ibfk_1", + "ColIds": [ + "c7" + ], + "ReferTableId": "t4", + "ReferColumnIds": [ + "c19" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f5" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "Unique": false, + "Keys": [ + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "Id": "i11", + "StoredColumnIds": null + } + ], + "Id": "t1" + }, + "t2": { + "Name": "parent1", + "Schema": "", + "ColIds": [ + "c16", + "c17", + "c18" + ], + "ColDefs": { + "c16": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c17": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c18": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + }, + "t3": { + "Name": "child11", + "Schema": "", + "ColIds": [ + "c12", + "c13", + "c14", + "c15" + ], + "ColDefs": { + "c12": { + "Name": "child_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "parent_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c14": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c12", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child11_ibfk_1", + "ColIds": [ + "c13" + ], + "ReferTableId": "t2", + "ReferColumnIds": [ + "c16" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f10" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "Unique": false, + "Keys": [ + { + "ColId": "c13", + "Desc": false, + "Order": 1 + } + ], + "Id": "i22", + "StoredColumnIds": null + } + ], + "Id": "t3" + }, + "t4": { + "Name": "parent2", + "Schema": "", + "ColIds": [ + "c19", + "c20", + "c21" + ], + "ColDefs": { + "c19": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c20": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c21": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t4" + }, + "t5": { + "Name": "child31", + "Schema": "", + "ColIds": [ + "c27", + "c28", + "c29", + "c30" + ], + "ColDefs": { + "c27": { + "Name": "child_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c28": { + "Name": "parent_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c29": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c30": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c27", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child31_ibfk_1", + "ColIds": [ + "c28" + ], + "ReferTableId": "t4", + "ReferColumnIds": [ + "c19" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f6" + } + ], + "Indexes": [ + { + "Name": "par_ind2", + "Unique": false, + "Keys": [ + { + "ColId": "c28", + "Desc": false, + "Order": 1 + } + ], + "Id": "i12", + "StoredColumnIds": null + } + ], + "Id": "t5" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c23": [ + 29, + 30 + ], + "c6": [ + 14 + ], + "c7": [ + 14, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19 + ], + "c8": [], + "c9": [ + 0 + ] + }, + "TableLevelIssues": null + }, + "t2": { + "ColumnLevelIssues": { + "c16": [ + 14 + ], + "c17": [], + "c18": [ + 0 + ], + "c24": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + }, + "t3": { + "ColumnLevelIssues": { + "c12": [ + 14 + ], + "c13": [ + 14, + 24 + ], + "c14": [], + "c15": [ + 0 + ], + "c26": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + }, + "t4": { + "ColumnLevelIssues": { + "c19": [ + 14 + ], + "c20": [], + "c21": [ + 0 + ], + "c25": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + }, + "t5": { + "ColumnLevelIssues": { + "c31": [ + 29, + 30 + ], + "c27": [ + 14 + ], + "c28": [ + 14, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19 + ], + "c29": [], + "c30": [ + 0 + ] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r27", + "Name": "r27", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": true, + "SpSequences": {}, + "SrcSequences": {} + } \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..286b0f40ba --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE `true` ( + `COLUMN` INT64 NOT NULL, + `TABLE` STRING(MAX), + `WITH` STRING(MAX) +) PRIMARY KEY (`COLUMN`); + +CREATE CHANGE STREAM allstream FOR ALL; diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-schema.sql new file mode 100644 index 0000000000..ce7db9eb56 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/oracle-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE "true" ( + "COLUMN" INTEGER NOT NULL, + "TABLE" VARCHAR2(255), + "WITH" VARCHAR2(255), + PRIMARY KEY ("COLUMN") +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/session.json new file mode 100644 index 0000000000..f8d2bf629d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleReservedKeywordsIT/session.json @@ -0,0 +1,48 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "SP_DATABASE", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "true", + "ColIds": ["c1", "c2", "c3"], + "ShardIdColumn": "", + "ColDefs": { + "c1": {"Name": "COLUMN", "T": {"Name": "INT64"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "TABLE", "T": {"Name": "STRING", "Len": "MAX"}, "Id": "c2"}, + "c3": {"Name": "WITH", "T": {"Name": "STRING", "Len": "MAX"}, "Id": "c3"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "true", + "Schema": "SRC_DATABASE", + "ColIds": ["c1", "c2", "c3"], + "ColDefs": { + "c1": {"Name": "COLUMN", "T": {"Name": "INTEGER"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "TABLE", "T": {"Name": "VARCHAR2", "Len": 255}, "Id": "c2"}, + "c3": {"Name": "WITH", "T": {"Name": "VARCHAR2", "Len": 255}, "Id": "c3"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SchemaIssues": {}, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-10mb-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-10mb-schema.sql new file mode 100644 index 0000000000..a6febf6408 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-10mb-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE large_data ( + id VARCHAR2(36) PRIMARY KEY, + large_blob BLOB NOT NULL +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-col-mb-session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-col-mb-session.json new file mode 100644 index 0000000000..5f5010bfa3 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-col-mb-session.json @@ -0,0 +1,235 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "oracle_10mb", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t157": { + "Name": "large_data", + "ColIds": [ + "c158", + "c159", + "c162" + ], + "ShardIdColumn": "c162", + "ColDefs": { + "c158": { + "Name": "id", + "T": { + "Name": "STRING", + "Len": 36, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id varchar(36)", + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "" + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c159": { + "Name": "large_blob", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: large_data longblob(4294967295)", + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "" + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c162": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c162", + "AutoGen": { + "Name": "", + "GenerationType": "" + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c158", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c162", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table large_data", + "Id": "t157" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t157": { + "Name": "LARGE_DATA", + "Schema": "oracle_10mb", + "ColIds": [ + "c158", + "c159" + ], + "ColDefs": { + "c158": { + "Name": "ID", + "Type": { + "Name": "varchar2", + "Mods": [ + 36 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "" + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c159": { + "Name": "LARGE_BLOB", + "Type": { + "Name": "blob", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "" + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c158", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t157" + } + }, + "SchemaIssues": { + "t157": { + "ColumnLevelIssues": { + "c162": [ + 29 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r163", + "Name": "r163", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "SpProjectId": "daring-fiber-439305-v4", + "SpInstanceId": "rr-demo", + "Source": "oracle" +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-google_standard_sql-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..0aeac6d715 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleSourceDbWideRow10MbIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE large_data ( + id STRING(36) NOT NULL, + large_blob BYTES(10485760) NOT NULL +) PRIMARY KEY (id); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..5f74770110 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,14 @@ +ALTER DATABASE db SET OPTIONS (default_time_zone = 'Australia/Brisbane'); + +CREATE TABLE IF NOT EXISTS Users ( + id INT64 NOT NULL, + time_colm TIMESTAMP +) PRIMARY KEY(id); + + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-schema.sql new file mode 100644 index 0000000000..e16c4fc7e8 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/oracle-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE "Users" ( + "id" NUMBER(38,0) NOT NULL, + "time_colm" TIMESTAMP, + PRIMARY KEY("id")); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/session.json new file mode 100644 index 0000000000..fe2aacf589 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleTimezoneIT/session.json @@ -0,0 +1,135 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "timestamp_it", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t136": { + "Name": "Users", + "ColIds": [ + "c142", + "c143" + ], + "ShardIdColumn": "", + "ColDefs": { + "c142": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int", + "Id": "c142" + }, + "c143": { + "Name": "time_colm", + "T": { + "Name": "TIMESTAMP", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Id": "c143" + } + }, + "PrimaryKeys": [ + { + "ColId": "c142", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Category", + "Id": "t136" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t136": { + "Name": "Users", + "Schema": "", + "ColIds": [ + "c142", + "c143" + ], + "ColDefs": { + "c142": { + "Name": "id", + "Type": { + "Name": "integer", + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c142" + }, + "c143": { + "Name": "time_colm", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c143" + } + }, + "PrimaryKeys": [ + { + "ColId": "c142", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t136" + } + }, + "SchemaIssues": { + "t136": { + "ColumnLevelIssues": { + "c142": [ + 14 + ], + "c143": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+10:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "DatabaseOptions": { + "DefaultTimezone": "Australia/Brisbane" + } +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3774e4cffa --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS `generated_pk_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, +) PRIMARY KEY (`generated_column_col`); + +CREATE TABLE IF NOT EXISTS `generated_non_pk_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, + `id` INT64 not null, +) PRIMARY KEY (`id`); + +CREATE TABLE IF NOT EXISTS `non_generated_to_generated_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, + `generated_column_pk_col` STRING(100) AS (concat(`first_name_col`,' ')) STORED, +) PRIMARY KEY (`generated_column_pk_col`); + +CREATE TABLE IF NOT EXISTS `generated_to_non_generated_column_table` ( + `first_name_col` STRING(50), + `last_name_col` STRING(50) DEFAULT(NULL), + `generated_column_col` STRING(100) DEFAULT(NULL), + `generated_column_pk_col` STRING(100) DEFAULT(NULL), +) PRIMARY KEY (`generated_column_pk_col`); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-schema.sql new file mode 100644 index 0000000000..a66a5acbec --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToOracleWithoutSessionIT/oracle-schema.sql @@ -0,0 +1,30 @@ +CREATE TABLE "generated_pk_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS (CONCAT("first_name_col", ' ')) VIRTUAL NOT NULL, + PRIMARY KEY ("generated_column_col") +); + +CREATE TABLE "generated_non_pk_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS (CONCAT("first_name_col", ' ')) VIRTUAL NOT NULL, + "id" INT not null, + PRIMARY KEY ("id") +); + +CREATE TABLE "non_generated_to_generated_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) NOT NULL, + "generated_column_pk_col" VARCHAR2(100) NOT NULL, + PRIMARY KEY ("generated_column_pk_col") +); + +CREATE TABLE "generated_to_non_generated_column_table" ( + "first_name_col" VARCHAR2(50) DEFAULT NULL, + "last_name_col" VARCHAR2(50) DEFAULT NULL, + "generated_column_col" VARCHAR2(100) GENERATED ALWAYS AS (CONCAT("first_name_col", ' ')) VIRTUAL NOT NULL, + "generated_column_pk_col" VARCHAR2(100) GENERATED ALWAYS AS (CASE WHEN "first_name_col" IS NOT NULL THEN "first_name_col" END || ' ') VIRTUAL NOT NULL, + PRIMARY KEY ("generated_column_pk_col") +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..f8ab26e57f --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,58 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER(10, 2) NOT NULL, + "LegacyRegion" VARCHAR2(50), + CONSTRAINT "CHK_CreditLimit" CHECK ("CreditLimit" > 1000) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER(10, 2), + "LegacyOrderSystem" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "LegacyOrderSystem", "OrderId"), + CONSTRAINT "FK_CustomerOrder" FOREIGN KEY ("CustomerId") REFERENCES "Customers"("CustomerId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER NOT NULL PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER DEFAULT NULL, + "tinyint_unsigned_col" NUMBER DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER DEFAULT NULL, + "smallint_unsigned_col" NUMBER DEFAULT NULL, + "mediumint_col" NUMBER DEFAULT NULL, + "mediumint_unsigned_col" NUMBER DEFAULT NULL, + "bigint_col" NUMBER DEFAULT NULL, + "bigint_unsigned_col" NUMBER DEFAULT NULL, + "float_col" FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" VARCHAR2(4) DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" RAW(255) DEFAULT NULL, + "tinytext_col" VARCHAR2(255) DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" CLOB DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(50) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" NUMBER DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER DEFAULT NULL, + "integer_unsigned_col" NUMBER DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(255) DEFAULT NULL +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-spanner-schema.sql new file mode 100644 index 0000000000..bef40bd874 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/oracle-spanner-schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, -- No constraint + LoyaltyTier STRING(50), -- Renamed from LegacyRegion of Oracle +) PRIMARY KEY (CustomerId); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + OrderSource STRING(50) NOT NULL, -- Added column, NOT part of Spanner PK +) PRIMARY KEY (CustomerId, OrderId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar_col STRING(21000), + tinyint_col INT64, + tinyint_unsigned_col INT64, + text_col STRING(MAX), + date_col DATE, + smallint_col INT64, + smallint_unsigned_col INT64, + mediumint_col INT64, + mediumint_unsigned_col INT64, + bigint_col INT64, + bigint_unsigned_col NUMERIC, + float_col FLOAT64, + double_col FLOAT64, + decimal_col NUMERIC, + datetime_col TIMESTAMP, + time_col STRING(MAX), + year_col STRING(MAX), + char_col STRING(255), + tinyblob_col BYTES(MAX), + tinytext_col STRING(MAX), + blob_col BYTES(MAX), + mediumblob_col BYTES(MAX), + mediumtext_col STRING(MAX), + test_json_col JSON, + longblob_col BYTES(MAX), + longtext_col STRING(MAX), + enum_col STRING(MAX), + bool_col BOOL, + binary_col BYTES(MAX), + varbinary_col BYTES(MAX), + bit_col BYTES(MAX), + bit8_col INT64, + bit1_col BOOL, + boolean_col BOOL, + int_col INT64, + integer_unsigned_col INT64, + timestamp_col TIMESTAMP, + set_col STRING(MAX), +) PRIMARY KEY(id); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/overrides.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..2c964b41cd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..9ff2f9551a --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, -- No constraint + LoyaltyTier STRING(50), -- Renamed from LegacyRegion of MySQL +) PRIMARY KEY (CustomerId); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + OrderSource STRING(50) NOT NULL, -- Added column, NOT part of Spanner PK +) PRIMARY KEY (CustomerId, OrderId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar_col STRING(21000), + tinyint_col INT64, + tinyint_unsigned_col INT64, + text_col STRING(MAX), + date_col DATE, + smallint_col INT64, + smallint_unsigned_col INT64, + mediumint_col INT64, + mediumint_unsigned_col INT64, + bigint_col INT64, + bigint_unsigned_col NUMERIC, + float_col FLOAT64, + double_col FLOAT64, + decimal_col NUMERIC, + datetime_col TIMESTAMP, + time_col STRING(MAX), + year_col STRING(MAX), + char_col STRING(255), + tinyblob_col BYTES(MAX), + tinytext_col STRING(MAX), + blob_col BYTES(MAX), + mediumblob_col BYTES(MAX), + mediumtext_col STRING(MAX), + test_json_col JSON, + longblob_col BYTES(MAX), + longtext_col STRING(MAX), + enum_col STRING(MAX), + bool_col BOOL, + binary_col BYTES(MAX), + varbinary_col BYTES(MAX), + bit_col BYTES(MAX), + bit8_col INT64, + bit1_col BOOL, + boolean_col BOOL, + int_col INT64, + integer_unsigned_col INT64, + timestamp_col TIMESTAMP, + set_col STRING(MAX), +) PRIMARY KEY(id); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..6f77eaeb4c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/oracle-schema.sql @@ -0,0 +1,58 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER(10) NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER(10, 2) NOT NULL, + "LegacyRegion" VARCHAR2(50), + CONSTRAINT "CHK_CreditLimit" CHECK ("CreditLimit" > 1000) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER(10) NOT NULL, + "OrderId" NUMBER(10) NOT NULL, + "OrderValue" NUMBER(10, 2), + "LegacyOrderSystem" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "LegacyOrderSystem", "OrderId"), + CONSTRAINT "FK_CustomerOrder" FOREIGN KEY ("CustomerId") REFERENCES "Customers"("CustomerId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER(10) PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER(3) DEFAULT NULL, + "tinyint_unsigned_col" NUMBER(3) DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER(5) DEFAULT NULL, + "smallint_unsigned_col" NUMBER(5) DEFAULT NULL, + "mediumint_col" NUMBER(7) DEFAULT NULL, + "mediumint_unsigned_col" NUMBER(7) DEFAULT NULL, + "bigint_col" NUMBER(19) DEFAULT NULL, + "bigint_unsigned_col" NUMBER(20) DEFAULT NULL, + "float_col" BINARY_FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER(38, 10) DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" VARCHAR2(4) DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" BLOB DEFAULT NULL, + "tinytext_col" CLOB DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" CLOB DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(10) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" RAW(1) DEFAULT NULL, + "bit1_col" RAW(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER(10) DEFAULT NULL, + "integer_unsigned_col" NUMBER(10) DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(50) DEFAULT NULL +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/overrides.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/overrides.json new file mode 100644 index 0000000000..2c964b41cd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBOracleRetryDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..4b9d38d2d7 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,28 @@ +CREATE TABLE `Customers` ( + `CustomerId` INT64 NOT NULL, + `CustomerName` STRING(255), + `CreditLimit` NUMERIC, + `LoyaltyTier` STRING(50), +) PRIMARY KEY (`CustomerId`); + +CREATE TABLE `Orders` ( + `CustomerId` INT64 NOT NULL, + `OrderId` INT64 NOT NULL, + `OrderValue` NUMERIC, + `OrderSource` STRING(50) NOT NULL, +) PRIMARY KEY (`CustomerId`, `OrderId`); + +CREATE TABLE `AllDataTypes` ( + `id` INT64 NOT NULL, + `varchar_col` STRING(1000), + `bit8_col` BYTES(MAX), + `bit1_col` BOOL, + `boolean_col` BOOL, +) PRIMARY KEY (`id`); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..5bf4c861a2 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,24 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER NOT NULL, + "LegacyRegion" VARCHAR2(50), + CONSTRAINT "CHK_CreditLimit" CHECK ("CreditLimit" > 1000) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER, + "LegacyOrderSystem" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "LegacyOrderSystem", "OrderId"), + CONSTRAINT "FK_CustomerOrder" FOREIGN KEY ("CustomerId") REFERENCES "Customers"("CustomerId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "bit8_col" RAW(8) DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/overrides.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..8000ac7d0d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..19857fac16 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,65 @@ +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar_col STRING(1000), + tinyint_col INT64, + tinyint_unsigned_col INT64, + text_col STRING(MAX), + date_col DATE, + smallint_col INT64, + smallint_unsigned_col INT64, + mediumint_col INT64, + mediumint_unsigned_col INT64, + bigint_col INT64, + bigint_unsigned_col INT64, + float_col FLOAT32, + double_col FLOAT64, + decimal_col NUMERIC, + datetime_col TIMESTAMP, + time_col STRING(MAX), + year_col STRING(MAX), + char_col STRING(255), + tinyblob_col BYTES(255), + tinytext_col STRING(MAX), + blob_col BYTES(65535), + mediumblob_col BYTES(10485760), + mediumtext_col STRING(MAX), + test_json_col JSON, + longblob_col BYTES(10485760), + longtext_col STRING(MAX), + enum_col STRING(MAX), + bool_col BOOL, + binary_col BYTES(255), + varbinary_col BYTES(1000), + bit_col BYTES(MAX), + bit8_col INT64, + bit1_col BOOL, + boolean_col BOOL, + int_col INT64, + integer_unsigned_col INT64, + timestamp_col TIMESTAMP, + set_col STRING(MAX), + migration_shard_id STRING(50), +) PRIMARY KEY (migration_shard_id, id); + +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC NOT NULL, + LoyaltyTier STRING(50), + migration_shard_id STRING(50), +) PRIMARY KEY (migration_shard_id, CustomerId); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + migration_shard_id STRING(50), + OrderSource STRING(50) NOT NULL, +) PRIMARY KEY (migration_shard_id, CustomerId, OrderId); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d', + allow_txn_exclusion = true +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..342ecbd1b2 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/oracle-schema.sql @@ -0,0 +1,58 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER(10, 2) NOT NULL, + "LegacyRegion" VARCHAR2(50), + CONSTRAINT CHK_CreditLimit CHECK ("CreditLimit" > 1000) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER(10, 2), + "LegacyOrderSystem" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "LegacyOrderSystem", "OrderId"), + CONSTRAINT FK_CustomerOrder FOREIGN KEY ("CustomerId") REFERENCES "Customers"("CustomerId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000), + "tinyint_col" NUMBER, + "tinyint_unsigned_col" NUMBER, + "text_col" CLOB, + "date_col" DATE, + "smallint_col" NUMBER, + "smallint_unsigned_col" NUMBER, + "mediumint_col" NUMBER, + "mediumint_unsigned_col" NUMBER, + "bigint_col" NUMBER, + "bigint_unsigned_col" NUMBER, + "float_col" FLOAT, + "double_col" FLOAT, + "decimal_col" NUMBER, + "datetime_col" TIMESTAMP, + "time_col" VARCHAR2(50), + "year_col" NUMBER, + "char_col" CHAR(255), + "tinyblob_col" BLOB, + "tinytext_col" CLOB, + "blob_col" BLOB, + "mediumblob_col" BLOB, + "mediumtext_col" CLOB, + "test_json_col" CLOB, + "longblob_col" BLOB, + "longtext_col" CLOB, + "enum_col" VARCHAR2(255), + "bool_col" NUMBER(1), + "binary_col" RAW(255), + "varbinary_col" RAW(1000), + "bit_col" RAW(64), + "bit8_col" NUMBER, + "bit1_col" NUMBER(1), + "boolean_col" NUMBER(1), + "int_col" NUMBER, + "integer_unsigned_col" NUMBER, + "timestamp_col" TIMESTAMP, + "set_col" VARCHAR2(255) +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/session.json new file mode 100644 index 0000000000..fddcbbf308 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDBShardedOracleRetryDLQIT/session.json @@ -0,0 +1,4329 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "mysql", + "DatabaseName": "dlq-it-test", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "Customers", + "ColIds": [ + "c6", + "c7", + "c8", + "c9", + "c55" + ], + "ShardIdColumn": "c55", + "ColDefs": { + "c55": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c6": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c7": { + "Name": "CustomerName", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: CustomerName varchar(255)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c8": { + "Name": "CreditLimit", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CreditLimit decimal(10,2)", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c9": { + "Name": "LoyaltyTier", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: LegacyRegion varchar(50)", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c55", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": [], + "Comment": "Spanner schema for source table Customers", + "Id": "t1" + }, + "t2": { + "Name": "Orders", + "ColIds": [ + "c11", + "c12", + "c13", + "c56", + "c58" + ], + "ShardIdColumn": "c56", + "ColDefs": { + "c11": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c12": { + "Name": "OrderId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: OrderId int(10)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c13": { + "Name": "OrderValue", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: OrderValue decimal(10,2)", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c56": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c56", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c58": { + "Name": "OrderSource", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": true, + "Comment": "", + "Id": "c58", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c12", + "Desc": false, + "Order": 3 + }, + { + "ColId": "c56", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table Orders", + "Id": "t2" + }, + "t3": { + "Name": "AllDataTypes", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53", + "c54" + ], + "ShardIdColumn": "c54", + "ColDefs": { + "c15": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c16": { + "Name": "varchar_col", + "T": { + "Name": "STRING", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varchar_col varchar(1000)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c17": { + "Name": "tinyint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_col tinyint(3)", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c18": { + "Name": "tinyint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_unsigned_col tinyint", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c19": { + "Name": "text_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_col text(65535)", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c20": { + "Name": "date_col", + "T": { + "Name": "DATE", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_col date", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c21": { + "Name": "smallint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_col smallint(5)", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c22": { + "Name": "smallint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_unsigned_col smallint(5)", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c23": { + "Name": "mediumint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_col mediumint(7)", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c24": { + "Name": "mediumint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_unsigned_col mediumint(7)", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c25": { + "Name": "bigint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_col bigint(19)", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c26": { + "Name": "bigint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_unsigned_col bigint unsigned(20)", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c27": { + "Name": "float_col", + "T": { + "Name": "FLOAT32", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_col float(12)", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c28": { + "Name": "double_col", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_col double(22)", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c29": { + "Name": "decimal_col", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_col decimal(65,30)", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c30": { + "Name": "datetime_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_col datetime", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c31": { + "Name": "time_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_col time", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c32": { + "Name": "year_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_col year", + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c33": { + "Name": "char_col", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: char_col char(255)", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c34": { + "Name": "tinyblob_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyblob_col tinyblob(255)", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c35": { + "Name": "tinytext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinytext_col tinytext(255)", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c36": { + "Name": "blob_col", + "T": { + "Name": "BYTES", + "Len": 65535, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_col blob(65535)", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c37": { + "Name": "mediumblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumblob_col mediumblob(16777215)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c38": { + "Name": "mediumtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumtext_col mediumtext(16777215)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c39": { + "Name": "test_json_col", + "T": { + "Name": "JSON", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: test_json_col json", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c40": { + "Name": "longblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longblob_col longblob(4294967295)", + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c41": { + "Name": "longtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longtext_col longtext(4294967295)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c42": { + "Name": "enum_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_col enum(1)", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c43": { + "Name": "bool_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_col tinyint(1)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c44": { + "Name": "binary_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_col binary(255)", + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c45": { + "Name": "varbinary_col", + "T": { + "Name": "BYTES", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varbinary_col varbinary(1000)", + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c46": { + "Name": "bit_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_col bit(64)", + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c47": { + "Name": "bit8_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit8_col bit(8)", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c48": { + "Name": "bit1_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit1_col bit(1)", + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c49": { + "Name": "boolean_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: boolean_col tinyint(1)", + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c50": { + "Name": "int_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_col int(10)", + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c51": { + "Name": "integer_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: integer_unsigned_col int(10)", + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c52": { + "Name": "timestamp_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_col timestamp", + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c53": { + "Name": "set_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: set_col set[]", + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c54": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c54", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c54", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table AllDataTypes", + "Id": "t3" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "Customers", + "Schema": "dlq-it-test", + "ColIds": [ + "c6", + "c7", + "c8", + "c9" + ], + "ColDefs": { + "c6": { + "Name": "CustomerId", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c7": { + "Name": "CustomerName", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c8": { + "Name": "CreditLimit", + "Type": { + "Name": "decimal", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c9": { + "Name": "LegacyRegion", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": [ + { + "Name": "CHK_CreditLimit", + "Expr": "(`CreditLimit` > 1000)", + "ExprId": "e4", + "Id": "cc5" + } + ], + "Indexes": null, + "Id": "t1" + }, + "t2": { + "Name": "Orders", + "Schema": "dlq-it-test", + "ColIds": [ + "c11", + "c12", + "c13", + "c14" + ], + "ColDefs": { + "c11": { + "Name": "CustomerId", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c12": { + "Name": "OrderId", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c13": { + "Name": "OrderValue", + "Type": { + "Name": "decimal", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c14": { + "Name": "LegacyOrderSystem", + "Type": { + "Name": "varchar", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c14", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c12", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": [ + { + "Name": "FK_CustomerOrder", + "ColIds": [ + "c11" + ], + "ReferTableId": "t1", + "ReferColumnIds": [ + "c6" + ], + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION", + "Id": "f10" + } + ], + "CheckConstraints": null, + "Indexes": null, + "Id": "t2" + }, + "t3": { + "Name": "AllDataTypes", + "Schema": "dlq-it-test", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53" + ], + "ColDefs": { + "c15": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c16": { + "Name": "varchar_col", + "Type": { + "Name": "varchar", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c17": { + "Name": "tinyint_col", + "Type": { + "Name": "tinyint", + "Mods": [ + 3 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c18": { + "Name": "tinyint_unsigned_col", + "Type": { + "Name": "tinyint", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c19": { + "Name": "text_col", + "Type": { + "Name": "text", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c20": { + "Name": "date_col", + "Type": { + "Name": "date", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c21": { + "Name": "smallint_col", + "Type": { + "Name": "smallint", + "Mods": [ + 5 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c22": { + "Name": "smallint_unsigned_col", + "Type": { + "Name": "smallint", + "Mods": [ + 5 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c23": { + "Name": "mediumint_col", + "Type": { + "Name": "mediumint", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c24": { + "Name": "mediumint_unsigned_col", + "Type": { + "Name": "mediumint", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c25": { + "Name": "bigint_col", + "Type": { + "Name": "bigint", + "Mods": [ + 19 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c26": { + "Name": "bigint_unsigned_col", + "Type": { + "Name": "bigint unsigned", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c27": { + "Name": "float_col", + "Type": { + "Name": "float", + "Mods": [ + 12 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c28": { + "Name": "double_col", + "Type": { + "Name": "double", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c29": { + "Name": "decimal_col", + "Type": { + "Name": "decimal", + "Mods": [ + 65, + 30 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c30": { + "Name": "datetime_col", + "Type": { + "Name": "datetime", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c31": { + "Name": "time_col", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c32": { + "Name": "year_col", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c33": { + "Name": "char_col", + "Type": { + "Name": "char", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c34": { + "Name": "tinyblob_col", + "Type": { + "Name": "tinyblob", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c35": { + "Name": "tinytext_col", + "Type": { + "Name": "tinytext", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c36": { + "Name": "blob_col", + "Type": { + "Name": "blob", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c37": { + "Name": "mediumblob_col", + "Type": { + "Name": "mediumblob", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c38": { + "Name": "mediumtext_col", + "Type": { + "Name": "mediumtext", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c39": { + "Name": "test_json_col", + "Type": { + "Name": "json", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c40": { + "Name": "longblob_col", + "Type": { + "Name": "longblob", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c41": { + "Name": "longtext_col", + "Type": { + "Name": "longtext", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c42": { + "Name": "enum_col", + "Type": { + "Name": "enum", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c43": { + "Name": "bool_col", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c44": { + "Name": "binary_col", + "Type": { + "Name": "binary", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c45": { + "Name": "varbinary_col", + "Type": { + "Name": "varbinary", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c46": { + "Name": "bit_col", + "Type": { + "Name": "bit", + "Mods": [ + 64 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c47": { + "Name": "bit8_col", + "Type": { + "Name": "bit", + "Mods": [ + 8 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c48": { + "Name": "bit1_col", + "Type": { + "Name": "bit", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c49": { + "Name": "boolean_col", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c50": { + "Name": "int_col", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c51": { + "Name": "integer_unsigned_col", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c52": { + "Name": "timestamp_col", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c53": { + "Name": "set_col", + "Type": { + "Name": "set", + "Mods": null, + "ArrayBounds": [ + -1 + ] + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t3" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c55": [ + 5 + ], + "c6": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t2": { + "ColumnLevelIssues": { + "c11": [ + 14 + ], + "c12": [ + 14 + ], + "c56": [ + 5 + ] + }, + "TableLevelIssues": null + }, + "t3": { + "ColumnLevelIssues": { + "c15": [ + 14 + ], + "c17": [ + 14 + ], + "c18": [ + 14 + ], + "c21": [ + 14 + ], + "c22": [ + 14 + ], + "c23": [ + 14 + ], + "c24": [ + 14 + ], + "c26": [ + 52 + ], + "c30": [ + 13 + ], + "c31": [ + 15 + ], + "c32": [ + 15 + ], + "c50": [ + 14 + ], + "c51": [ + 14 + ], + "c53": [ + 31 + ], + "c54": [ + 29 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "ToSpanner": { + "AllDataTypes": { + "Name": "AllDataTypes", + "Cols": { + "bigint_col": "bigint_col", + "bigint_unsigned_col": "bigint_unsigned_col", + "binary_col": "binary_col", + "bit1_col": "bit1_col", + "bit8_col": "bit8_col", + "bit_col": "bit_col", + "blob_col": "blob_col", + "bool_col": "bool_col", + "boolean_col": "boolean_col", + "char_col": "char_col", + "date_col": "date_col", + "datetime_col": "datetime_col", + "decimal_col": "decimal_col", + "double_col": "double_col", + "enum_col": "enum_col", + "float_col": "float_col", + "id": "id", + "int_col": "int_col", + "integer_unsigned_col": "integer_unsigned_col", + "longblob_col": "longblob_col", + "longtext_col": "longtext_col", + "mediumblob_col": "mediumblob_col", + "mediumint_col": "mediumint_col", + "mediumint_unsigned_col": "mediumint_unsigned_col", + "mediumtext_col": "mediumtext_col", + "set_col": "set_col", + "smallint_col": "smallint_col", + "smallint_unsigned_col": "smallint_unsigned_col", + "test_json_col": "test_json_col", + "text_col": "text_col", + "time_col": "time_col", + "timestamp_col": "timestamp_col", + "tinyblob_col": "tinyblob_col", + "tinyint_col": "tinyint_col", + "tinyint_unsigned_col": "tinyint_unsigned_col", + "tinytext_col": "tinytext_col", + "varbinary_col": "varbinary_col", + "varchar_col": "varchar_col", + "year_col": "year_col" + } + }, + "Customers": { + "Name": "Customers", + "Cols": { + "CreditLimit": "CreditLimit", + "CustomerId": "CustomerId", + "CustomerName": "CustomerName", + "LegacyRegion": "LoyaltyTier" + } + }, + "Orders": { + "Name": "Orders", + "Cols": { + "CustomerId": "CustomerId", + "LegacyOrderSystem": "LegacyOrderSystem", + "OrderId": "OrderId", + "OrderValue": "OrderValue" + } + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r57", + "Name": "r57", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": null + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "SpProjectId": "span-cloud-ck-testing-external", + "SpInstanceId": "ea-functional-tests", + "Source": "mysql", + "DatabaseOptions": { + "DbName": "", + "DefaultTimezone": "" + }, + "DefaultIdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/oracle-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/oracle-schema.sql new file mode 100644 index 0000000000..51f1ddf7fd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/oracle-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE "source_table1" ( + "id_col1" INT PRIMARY KEY, + "name_col1" VARCHAR2(255), + "data_col1" CLOB +); + +CREATE TABLE "source_table2" ( + "key_col2" VARCHAR2(50) PRIMARY KEY, + "category_col2" VARCHAR2(100), + "value_col2" CLOB +); diff --git a/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/spanner-schema.sql new file mode 100644 index 0000000000..0ecd26b9b7 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/oracle/SpannerToSourceDbOracleStringOverridesSchemaMapperIT/spanner-schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE Target_Table_1 ( + id_col1 INT64 NOT NULL, + Target_Name_Col_1 STRING(255), + data_col1 STRING(MAX) +) PRIMARY KEY (id_col1); + +CREATE TABLE source_table2 ( + key_col2 STRING(50) NOT NULL, + Target_Category_Col_2 STRING(100), + value_col2 STRING(MAX) +) PRIMARY KEY (key_col2); + +CREATE CHANGE STREAM allstream + FOR ALL OPTIONS ( + value_capture_type = 'NEW_ROW', + retention_period = '7d' +);