diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index 0bb8633866..8b035e4b4d 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -173,7 +173,10 @@ jobs: --it-project="span-cloud-teleport-testing" \ --it-artifact-bucket="span-cloud-teleport-testing-it-gitactions" \ --it-private-connectivity="datastream-connect-2" \ - --it-cloud-proxy-host="10.128.0.16" + --it-cloud-proxy-host="10.128.0.16" \ + --it-oracle-host="10.128.0.108" \ + --it-oracle-sys-password="TestPassword123" \ + --it-oracle-password="TestPassword123" - name: Upload Smoke Tests Report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() # always run even if the previous step fails @@ -217,6 +220,9 @@ jobs: --it-artifact-bucket="span-cloud-teleport-testing-it-gitactions" \ --it-private-connectivity="datastream-connect-2" \ --it-cloud-proxy-host="10.128.0.16" \ + --it-oracle-host="10.128.0.108" \ + --it-oracle-sys-password="TestPassword123" \ + --it-oracle-password="TestPassword123" \ --test="$SPECIFIC_TEST" - name: Upload Integration Tests Report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/cicd/cmd/run-it-smoke-tests/main.go b/cicd/cmd/run-it-smoke-tests/main.go index 7290477c44..65c6810759 100644 --- a/cicd/cmd/run-it-smoke-tests/main.go +++ b/cicd/cmd/run-it-smoke-tests/main.go @@ -69,6 +69,7 @@ func main() { flags.RetryFailures(), flags.StaticOracleHost(), flags.StaticOracleSysPassword(), + flags.StaticOraclePassword(), flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index e4fe0669e4..f654b3a9b6 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -70,6 +70,7 @@ func main() { flags.RetryFailures(), flags.StaticOracleHost(), flags.StaticOracleSysPassword(), + flags.StaticOraclePassword(), flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7fd488819a..acbda5bec5 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -39,6 +39,7 @@ var ( dCloudProxyPassword string dOracleHost string dCloudOracleSysPassword string + dCloudOraclePassword string dUnifiedWorkerHarnessContainerImage string dIntegrationTestParallelism string dThreadCount string @@ -61,6 +62,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") + flag.StringVar(&dCloudOraclePassword, "it-oracle-password", "oracle", "app password of static Oracle DB") flag.StringVar(&dUnifiedWorkerHarnessContainerImage, "it-unified-worker-harness-container-image", "", "Runner harness image to run tests against") flag.StringVar(&dIntegrationTestParallelism, "it-integration-test-parallelism", "3", "The level of parallelism for integration tests") flag.StringVar(&dThreadCount, "it-thread-count", "4", "The IT thread count to use for maven, which is the number of threads per core") @@ -149,6 +151,10 @@ func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } +func StaticOraclePassword() string { + return "-DcloudOraclePassword=" + dCloudOraclePassword +} + func UnifiedWorkerHarnessContainerImage() string { if dUnifiedWorkerHarnessContainerImage != "" { return "-DunifiedWorkerHarnessContainerImage=" + dUnifiedWorkerHarnessContainerImage diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java index 94ac8b7013..828a3f6d60 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java @@ -24,6 +24,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager; import com.google.common.io.Resources; import com.google.pubsub.v1.SubscriptionName; import com.google.pubsub.v1.TopicName; @@ -225,7 +226,8 @@ private void simpleJdbcToSpannerTest( cloudSqlResourceManager = jdbcType.equals(JDBCType.MYSQL) ? CloudMySQLResourceManager.builder(testName).build() - : CloudOracleResourceManager.builder(testName).build(); + : new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) CloudOracleResourceManager.builder(testName)); // Create Spanner Resource Manager SpannerResourceManager.Builder spannerResourceManagerBuilder = @@ -507,7 +509,8 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager) + .runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); } @@ -578,7 +581,8 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager) + .runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index dbb355be98..9fb272eaaa 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -90,6 +90,45 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { .build(); } + public com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager + setUpOracleResourceManager() { + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.maybeUseStaticInstance(); + if (System.getProperty("cloudOracleHost") != null) { + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager sysdba = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + String isoUser = + "U_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toUpperCase(); + String isoPassword = + "P_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toUpperCase(); + + sysdba.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", isoUser, isoPassword)); + sysdba.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", isoUser)); + sysdba.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", isoUser)); + sysdba.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", isoUser)); + + builder.setPassword(isoPassword); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(isoUser); + builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); + builder.setDatabaseName("XEPDB1"); + } + return new com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager(builder); + } + public String generateSessionFile( int numOfTables, String srcDb, String spannerDb, List tableNames, String sessionFile) throws IOException { @@ -614,4 +653,37 @@ protected ConditionCheck buildBaseConditionCheck( } return combinedCondition; } + + public void flushOracleRedoLogs( + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager) { + boolean success = false; + if (oracleResourceManager != null) { + try { + oracleResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .info("Successfully flushed Oracle redo logs natively."); + success = true; + } catch (Exception e) { + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .warn( + "Failed to switch Oracle log via ResourceManager, attempting raw JDBC fallback...", + e); + } + } + + if (!success && System.getProperty("cloudOracleHost") != null) { + String url = "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost") + ":1521/XE"; + String user = System.getProperty("cloudOracleUsername", "system"); + String pass = System.getProperty("cloudOraclePassword", "TestPassword123"); + try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); + java.sql.Statement stmt = conn.createStatement()) { + stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .info("Successfully flushed Oracle redo logs via raw JDBC fallback."); + } catch (Exception ex) { + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .error("Raw JDBC fallback log flush also failed.", ex); + } + } + } } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java new file mode 100644 index 0000000000..93a8cccedd --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -0,0 +1,675 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck.CheckResult; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for datatype conversions for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleDDLIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(DataStreamToSpannerOracleDDLIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static final String TABLE1 = "AllDatatypeColumns"; + private static final String TABLE2 = "AllDatatypeColumns2"; + private static final String TABLE3 = "DatatypeColumnsWithSizes"; + private static final String TABLE4 = "DatatypeColumnsReducedSizes"; + private static final String TABLE5 = "Users"; + private static final String TABLE6 = "Books"; + private static final String TABLE7 = "Authors"; + private static final String TABLE8 = "Singers"; + + private static final String TRANSFORMATION_TABLE = "AllDatatypeTransformation"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager + datastreamResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleDDLIT.class) { + testInstances.add(this); + if (jobInfo == null) { + oracleResourceManager = setUpOracleResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + createAndUploadJarToGcs("DatatypeIT", gcsResourceManager); + CustomTransformation customTransformation = + CustomTransformation.builder( + "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") + .build(); + + String[] tables = { + "AllDatatypeColumns", + "AllDatatypeColumns2", + "DatatypeColumnsWithSizes", + "DatatypeColumnsReducedSizes", + "Users", + "Authors", + "AllDatatypeTransformation", + "Singers", + "Books" + }; + for (String table : tables) { + try { + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + } catch (Exception e) { + } + executeSqlScript( + oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql"); + flushOracleRedoLogs(oracleResourceManager); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DatatypeIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + customTransformation, // customTransformation + null, // shardingContextFileResourceName + gcsResourceManager, + datastreamResourceManager, + null, // sessionResourceContent + org.apache.beam.it.gcp.datastream.OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager + .getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleResourceManager.getUsername().toUpperCase(), + java.util.Arrays.asList( + TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + .build()); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleDDLIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager); + } + + @Test + public void migrationTestWithAllDatatypeConversionMapping() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + @Override + protected String getDescription() { + return "Run SQL inserts"; + } + + @Override + protected CheckResult check() { + try { + executeSqlScript( + oracleResourceManager, + "oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql"); + flushOracleRedoLogs(oracleResourceManager); + return new CheckResult(true, "Success"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(15)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertAllDatatypeColumnsTableBackfillContents(); + } + + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + // Only asserting primary key + simple int column to verify end-to-end sync without complex + // base64 logic + row.put("int_column", "50000"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, int_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("date_column", "2024-02-08T00:00:00Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("decimal_column", "456.12"); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + // text, char, tinytext, mediumtext, longtext are BYTE columns + row.put("text_column", "/u/9n58P"); + row.put("char_column", "v58P"); + row.put("tinytext_column", "7+/+7/2fnw8="); + row.put("mediumtext_column", "/+3v79/v2vrx"); + row.put("longtext_column", "/+/v3+/a+vE="); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f3100000000000000000..."); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column," + + " tinyblob_column, blob_column, mediumblob_column, longblob_column," + + " enum_column, bool_column, other_bool_column, varbinary_column, bit_column," + + " decimal_column, text_column, binary_column, char_column, tinytext_column," + + " mediumtext_column, longtext_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("decimal_column", 123.45); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "15:30:45"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 21); + row.put("text_column", "Some text 2 append"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 201); + row.put("bigint_column", 987655); + row.put("float_column", 24.45); + row.put("double_column", 235.567); + row.put("decimal_column", 23457.78); + row.put("datetime_column", "2022-12-31T23:59:58Z"); + row.put("timestamp_column", "2022-12-31T23:59:58Z"); + row.put("time_column", "00:59:59"); + row.put("year_column", "2023"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 31); + row.put("text_column", "Some text 3 append"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 301); + row.put("bigint_column", 112234); + row.put("float_column", 35.56); + row.put("double_column", 346.678); + row.put("decimal_column", 34568.89); + row.put("datetime_column", "2023-12-31T23:59:59Z"); + row.put("timestamp_column", "2023-12-31T23:59:59Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example4"); + row.put("tinyint_column", 41); + row.put("text_column", "Some text 4 append"); + row.put("date_column", "2021-11-12"); + row.put("int_column", 401); + row.put("bigint_column", 223345); + row.put("float_column", 46.67); + row.put("double_column", 457.789); + row.put("decimal_column", 45679.90); + row.put("datetime_column", "2021-11-11T11:11:10Z"); + row.put("timestamp_column", "2021-11-11T11:11:10Z"); + row.put("time_column", "12:11:11"); + row.put("year_column", "2022"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column," + + " bigint_column, float_column, double_column, decimal_column," + + " datetime_column, timestamp_column, time_column, year_column, blob_column," + + " enum_column, bool_column, binary_column, bit_column FROM" + + " AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 25); + row.put("text_column", "Updated text 2"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 250); + row.put("bigint_column", 56789); + row.put("float_column", 25.45); + row.put("double_column", 345.678); + row.put("decimal_column", 23456.79); + row.put("datetime_column", "2023-01-01T12:00:00Z"); + row.put("timestamp_column", "2023-01-01T12:00:00Z"); + row.put("time_column", "12:00:00"); + row.put("year_column", "2023"); + row.put("blob_column", "EjRWeJCrze8="); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "EjRWeJCrze8SNFZ4kKvN7xI0Vng="); + row.put("bit_column", "fw=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 35); + row.put("text_column", "Updated text 3"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 350); + row.put("bigint_column", 88000); + row.put("float_column", 35.67); + row.put("double_column", 456.789); + row.put("decimal_column", 34567.90); + row.put("datetime_column", "2024-01-02T00:00:00Z"); + row.put("timestamp_column", "2024-01-02T00:00:00Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "q83vEjRWeJA="); + row.put("enum_column", "1"); + row.put("bool_column", false); + row.put("binary_column", "q83vEjRWeJCrze8SNFZ4kKvN7xI="); + row.put("bit_column", "AA=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("SELECT * FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDataTypeSizeConversionBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("float_column", 45.67); + row.put("decimal_column", 456.12); + row.put("char_column", "char_1"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsWithSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOException { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsReducedSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertUsersBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("user_id", 1); + row.put("first_name", "Lorem"); + row.put("last_name", "Epsum"); + row.put("full_name", "Lorem Epsum"); + row.put("age", 20); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 2); + row.put("first_name", "Jane"); + row.put("last_name", "Doe"); + row.put("full_name", "Jane Doe"); + row.put("age", 21); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 9); + row.put("first_name", "James"); + row.put("last_name", "Dove"); + row.put("full_name", "James Dove"); + row.put("age", 22); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select user_id, first_name, last_name, full_name, age from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAuthorsBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "J.R.R. Tolkien"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Jane Austen"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Douglas Adams"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, name from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhiker's Guide to the Galaxy"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..a379303394 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -0,0 +1,377 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleRetryAllDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-retryalldlq-test"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // Setup Oracle Source + CloudOracleResourceManager sysUser = setUpOracleResourceManager(); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + CloudOracleResourceManager.builder(testName); + sysBuilder.setHost(sysUser.getHost()); + sysBuilder.setPort(sysUser.getPort()); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + sysBuilder.setDatabaseName(sysUser.getDatabaseName()); + CloudOracleResourceManager trueSysUser = + (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); + + String oracleUser = sysUser.getUsername(); + String oraclePassword = sysUser.getPassword(); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + CloudOracleResourceManager.builder(testName); + builder.setHost(sysUser.getHost()); + builder.setPort(sysUser.getPort()); + builder.setUsername(oracleUser); + builder.setPassword(oraclePassword); + builder.setDatabaseName("XEPDB1"); + + jdbcResourceManager = + (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + // Create Oracle Schema using helper + executeSqlScript(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE); + + // Add Supplemental Log Data required by Datastream CDC + List.of("Customers", "Orders", "AllDataTypes").forEach(tableName -> {}); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + // Upload overrides file + gcsResourceManager.uploadArtifact( + "input/overrides.json", Resources.getResource(OVERRIDES_FILE_RESOURCE).getPath()); + + // Upload custom transformation jar + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + // Prepare job parameters + Map jobParameters = new HashMap<>(); + jobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "20"); + jobParameters.put("dlqRetryMinutes", "1"); + + OracleSource oracleSource = + OracleSource.builder( + jdbcResourceManager.getHost(), + jdbcResourceManager.getUsername(), + jdbcResourceManager.getPassword(), + jdbcResourceManager.getPort(), + jdbcResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + jdbcResourceManager.getUsername().toUpperCase(), + List.of("Customers", "Orders", "AllDataTypes"))) + .build(); + + // Launch regular pipeline using ITBase method + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + null, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, jdbcResourceManager, gcsResourceManager, datastreamResourceManager); + } + + @Test + public void testDataStreamToSpannerRetryAllDLQ() throws Exception { + LOG.info("Starting testDataStreamToSpannerRetryAllDLQ"); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Inserting parent rows directly into Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .build())); + + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + LOG.info("Waiting for DLQ events to appear in retry and severe buckets"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder( + gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) // id = 102 + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) // id= 2 + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) // id = 1 + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); + assertTrue( + "id=999 should NOT exist yet in AllDataTypes", + !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); + assertTrue( + "id=1 should NOT exist yet in Customers", + !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); + assertTrue( + "id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue( + "id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(15))); + + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Parent Customer") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .build())); + + Map retryJobParameters = new HashMap<>(); + retryJobParameters.put("runMode", "retryAllDLQ"); + retryJobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryJobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryJobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryJobParameters.put("transformationCustomParameters", "mode=good"); + retryJobParameters.put("dlqMaxRetryCount", "20"); + retryJobParameters.put("dlqRetryMinutes", "1"); + retryJobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + null, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryJobParameters, + null, + null, + gcsResourceManager, + null, + null, + null); + + assertThatPipeline(retryJobInfo).isRunning(); + + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(1) + .setMaxEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .setMaxEvents(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(30)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + + assertTrue( + "id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + + List rows = + spannerResourceManager.runQuery("SELECT * FROM AllDataTypes WHERE id = 999"); + Map expectedRow = createExpectedRowFor999(); + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(List.of(expectedRow)); + + assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); + assertTrue( + "id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + } + + private boolean rowExistsInSpanner(String tableName, String idColumnName, long id) { + List rows = + spannerResourceManager.readTableRecords(tableName, List.of(idColumnName)); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id) { + return true; + } + } + return false; + } + + private void insertDataInOracle() { + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar2_col\") VALUES (1, 'test1')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar2_col\") VALUES (999, 'test999')"); + } + + private Map createExpectedRowFor999() { + Map row = new HashMap<>(); + row.put("id", 999L); + row.put("varchar2_col", "test999"); + return row; + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java new file mode 100644 index 0000000000..77804088f9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -0,0 +1,466 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleRetryDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerOracleRetryDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/session.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-sharded-retrydlq"; + + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static PubsubResourceManager pubsubResourceManager; + + private static String streamNameA; + + private CloudOracleResourceManager setUpOracleResourceManagerForShard(String shardName) { + if (System.getProperty("cloudOracleHost") != null) { + CloudOracleResourceManager.Builder shardBuilder = + CloudOracleResourceManager.builder(testName + shardName); + shardBuilder.setUsername(System.getProperty("cloudOracleUsername", "system")); + shardBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + shardBuilder.setHost(System.getProperty("cloudOracleHost")); + shardBuilder.setPort(1521); + shardBuilder.setDatabaseName("XEPDB1"); + return new SpannerOracleResourceManager(shardBuilder); + } else { + return new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName + shardName)); + } + } + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleRetryDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + jdbcResourceManagerShardA = setUpOracleResourceManagerForShard("A"); + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + pubsubResourceManager = setUpPubSubResourceManager(); + + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + String gcsPrefix = + getGcsPath(GCS_PATH_PREFIX + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + + OracleSource oracleSourceA = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + "XEPDB1") + .setAllowedTables( + Map.of( + jdbcResourceManagerShardA.getUsername().toUpperCase(), + List.of( + "Customers", + "Orders", + "AllDataTypes"))) // Assuming schemas are username + .build(); + + SourceConfig sourceConfigA = + datastreamResourceManager.buildJDBCSourceConfig("oracle-profile-shardA", oracleSourceA); + DestinationConfig destinationConfigA = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile-shardA", + gcsResourceManager.getBucket(), + gcsPrefix, + DestinationOutputFormat.AVRO_FILE_FORMAT); + Stream streamA = + datastreamResourceManager.createStream( + "stream-shardA", sourceConfigA, destinationConfigA); + datastreamResourceManager.startStream(streamA); + streamNameA = streamA.getName().substring(streamA.getName().lastIndexOf('/') + 1); + + String shardConfig = + generateSourceConfig( + streamNameA, + jdbcResourceManagerShardA.getUsername().toUpperCase(), + "shard1", + streamNameA, + "DUMMY_DB_B", + "shard2"); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", shardConfig.getBytes(StandardCharsets.UTF_8)); + + Map jobParameters = new HashMap<>(); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "1000"); + jobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParameters.put( + "inputFilePattern", getGcsPath(GCS_PATH_PREFIX + "/cdc/", gcsResourceManager)); + jobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + jobParameters.put("datastreamSourceType", "oracle"); + jobParameters.put("dlqRetryMinutes", "1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + null); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleRetryDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + jdbcResourceManagerShardA, + jdbcResourceManagerShardA, + gcsResourceManager, + datastreamResourceManager, + pubsubResourceManager); + } + + @Test + public void testDataStreamToSpannerShardedRetryDLQ() throws Exception { + LOG.info("Starting testDataStreamToSpannerShardedRetryDLQ"); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Inserting parent rows directly into Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .set("migration_shard_id") + .to("shard1") + .build())); + + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + LOG.info("Waiting for DLQ events to appear in severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build() + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + assertTrue( + "id=1 should exist in AllDataTypes on shard1", + rowExistsInSpanner("AllDataTypes", "id", 1, "shard1")); + assertTrue( + "id=999 should NOT exist yet in AllDataTypes on shard1", + !rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); + + assertTrue( + "id=2 should exist in Customers on shard1", + rowExistsInSpanner("Customers", "CustomerId", 2, "shard1")); + assertTrue( + "id=1 should NOT exist yet in Customers on shard1", + !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); + + assertTrue( + "id=102 should exist in Orders on shard1", + rowExistsInSpanner("Orders", "OrderId", 102, "shard1")); + assertTrue( + "id=101 should NOT exist yet in Orders on shard1", + !rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); + assertTrue( + "id=103 should NOT exist yet in Orders on shard1", + !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); + + LOG.info("Launching retryDLQ job with session file to process DLQ"); + Map retryJobParameters = new HashMap<>(); + retryJobParameters.put("runMode", "retryDLQ"); + retryJobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryJobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryJobParameters.put("transformationCustomParameters", "mode=good"); + retryJobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryJobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + retryJobParameters.put("datastreamSourceType", "oracle"); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + SESSION_FILE_RESOURCE, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryJobParameters, + null, + null, + gcsResourceManager, + null, + null, + null); + LOG.info("RetryDLQ job launched: {}", retryJobInfo.jobId()); + + assertThatPipeline(retryJobInfo).isRunning(); + + LOG.info("Applying partial fixes in Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Parent Customer") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .set("migration_shard_id") + .to("shard1") + .build())); + + LOG.info("Waiting for the retryDLQ job to complete automatically"); + PipelineOperator.Result retryJobResult = + pipelineOperator().waitUntilDone(createConfig(retryJobInfo, Duration.ofMinutes(15))); + assertThatResult(retryJobResult).isLaunchFinished(); + + assertTrue( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .build() + .get()); + + LOG.info("Waiting for fixed rows to appear in Spanner"); + PipelineOperator.Result finalWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(2) + .build() + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build())); + assertThatResult(finalWaitResult).meetsConditions(); + + assertTrue("id=999 should exist", rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); + assertTrue("id=3 should exist", rowExistsInSpanner("Customers", "CustomerId", 3, "shard1")); + assertTrue( + "id=1 should NOT exist", !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); + assertTrue("id=101 should exist", rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); + assertTrue("id=103 should NOT exist", !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); + + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineLauncher.cancelJob(PROJECT, REGION, jobInfo.jobId()); + } + + private boolean rowExistsInSpanner( + String tableName, String idColumnName, long id, String shardId) { + List rows = + spannerResourceManager.readTableRecords( + tableName, List.of(idColumnName, "migration_shard_id")); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id && row.getString("migration_shard_id").equals(shardId)) { + return true; + } + } + return false; + } + + private void insertDataInOracle() { + LOG.info("Inserting data in Oracle Shard A"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); + + LOG.info("Inserting data in Oracle Shard B"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (1, 'test1')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (999, 'test999')"); + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private void executeSqlScript(CloudOracleResourceManager resourceManager, String resourceName) + throws IOException { + String sql = Resources.toString(Resources.getResource(resourceName), StandardCharsets.UTF_8); + for (String statement : sql.split(";")) { + if (!statement.trim().isEmpty()) { + resourceManager.runSQLUpdate(statement); + } + } + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA, String streamB, String dbB, String shardB) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " },\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardB + + "\",\n" + + " \"dbName\": \"" + + dbB + + "\",\n" + + " \"streamId\": \"" + + streamB + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java new file mode 100644 index 0000000000..3d7de24578 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleStringOverridesIT extends DataStreamToSpannerITBase { + + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + + private static HashSet testInstances = + new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static String gcsPrefix; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleStringOverridesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + + try { + } catch (Exception e) { + } + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + try { + } catch (Exception e) { + // Might exist or fail if we are not SYS, ignore wrapper + } + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("tableOverrides", "[{person1, human1}]"); + overridesMap.put("columnOverrides", "[{person1.first_name1, person1.name1}]"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "oracleStringOverridesIT", + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleStringOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + oracleResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new org.apache.beam.it.conditions.ConditionCheck() { + @Override + protected String getDescription() { + return "Insert records into Oracle"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('Alice', 'Johnson')"); + return new CheckResult(true, "Inserted successfully"); + } catch (Exception e) { + return new CheckResult(false, "Failed to insert"); + } + } + }, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java new file mode 100644 index 0000000000..104f480a15 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -0,0 +1,366 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT + extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + private static final Integer NUM_COLUMNS = 1000; + + private static CloudOracleResourceManager cloudOracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + + private static final List COLUMNS = new ArrayList<>(); + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + // According to instructions on quoting/identifiers: Oracle often folds to uppercase but + // Datastream captures whatever it is. Oh wait, Oracle tables are usually uppercase or need + // quotes. + // Let's use uppercase to avoid case folding issues if unquoted + TABLE_NAMES.add( + "DATASTREAMTOSPANNER_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + } + for (int i = 1; i <= NUM_COLUMNS; i++) { + COLUMNS.add("COL_" + i); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(System.getProperty("cloudOracleUsername", "system")); + builder.setDatabaseName("XEPDB1"); + } + cloudOracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); + + String sessionContent = generateBaseSchema(); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", cloudOracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()) + .replaceAll("\"mysql\"", "\"oracle\""); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + cloudOracleResourceManager.getHost(), + cloudOracleResourceManager.getUsername(), + cloudOracleResourceManager.getPassword(), + cloudOracleResourceManager.getPort(), + cloudOracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudOracleResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudOracleResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + cloudOracleResourceManager.runSQLUpdate(getJDBCSchema(tableName)); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerForMaxColumnsPerTables() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + PipelineOperator.Config.builder() + .setJobId(jobInfo.jobId()) + .setProject(PROJECT) + .setRegion(REGION) + .setTimeoutAfter(Duration.ofMinutes(20)) + .setCheckAfter(Duration.ofSeconds(5)) + .build(), + conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + return gson.toJson(sessionTemplate); + } + + public static Map createSessionTemplate() { + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + return createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "VARCHAR2"); + colType.put("Len", 0); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + " VARCHAR2(5)"); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)).append(" VARCHAR2(5) NOT NULL"); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(", PRIMARY KEY (").append(COLUMNS.get(0)).append("))"); + return sb.toString(); + } + + private void createSpannerTables() { + for (String tableName : TABLE_NAMES) { + List columns = new ArrayList<>(); + columns.add(COLUMNS.get(0) + " STRING(10) NOT NULL"); + + for (int i = 2; i <= NUM_COLUMNS; i++) { + columns.add(COLUMNS.get(i - 1) + " STRING(10)"); + } + + String ddlStatement = + String.format( + "CREATE TABLE %s (%s) PRIMARY KEY (%s)", + tableName, String.join(", ", columns), COLUMNS.get(0)); + + spannerResourceManager.executeDdlStatement(ddlStatement); + } + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : TABLE_NAMES) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + values.put(COLUMNS.get(ci - 1), String.valueOf(ci)); + } + rows.add(values); + } + cdcEvents.put(tableName, rows); + success &= cloudOracleResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..7571d073b2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -0,0 +1,440 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for live replication from Oracle to Spanner using the retryAllDLQ mode for a + * sharded setup. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerShardedOracleRetryAllDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerShardedOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-sharded-retryall"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + private static String streamNameA; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerShardedOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + jdbcResourceManagerShardA = setUpOracleResourceManager(); + String oracleUser = jdbcResourceManagerShardA.getUsername(); + String oraclePassword = jdbcResourceManagerShardA.getPassword(); + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder( + testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = + org.apache.beam.it.gcp.datastream.OracleSource.builder( + System.getProperty("cloudOracleHost"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleUser, java.util.List.of("Customers", "Orders", "AllDataTypes"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + GCS_PATH_PREFIX + "/cdc/", + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat + .AVRO_FILE_FORMAT); + + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5) + .toLowerCase(), + sourceConfig, + destinationConfig); + + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + String shardConfig = + generateSourceConfig( + streamNameA, oracleUser, "shard1", streamNameA, oracleUser, "shard2"); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", + shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + gcsResourceManager.uploadArtifact( + "input/overrides.json", + com.google.common.io.Resources.getResource( + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json") + .getPath()); + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + java.util.Map jobParameters = new java.util.HashMap<>(); + jobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "20"); + jobParameters.put("dlqRetryMinutes", "60"); + jobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + jobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + // Explicitly set inputFilePattern for manual stream + jobParameters.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/" + GCS_PATH_PREFIX + "/cdc/"); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParameters.put("jdbcDriverJars", driverPath); + } + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + null, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + null); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerShardedOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + jdbcResourceManagerShardA, + gcsResourceManager, + datastreamResourceManager); + } + + @Test + public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + // 1. Insert parent rows directly into Spanner. + com.google.cloud.spanner.Mutation customer2 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .build(); + spannerResourceManager.write(customer2); + + // 2. Insert all test data into Oracle. + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + // 3. Wait for DLQ events to appear in corresponding buckets. + LOG.info("Waiting for DLQ events to appear in shared severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(25)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder( + gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + LOG.info("DLQ events successfully generated in corresponding buckets"); + + // Verify Spanner state + LOG.info("Verifying Spanner state before retry job runs"); + assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); + assertTrue( + "id=999 should NOT exist yet in AllDataTypes", + !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); + assertTrue( + "id=1 should NOT exist yet in Customers", + !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); + assertTrue( + "id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue( + "id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + + // 4. Stop the regular pipeline + LOG.info("Stopping regular pipeline"); + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(5))); + LOG.info("Regular pipeline stopped successfully"); + + // 5. Apply partial fixes + LOG.info("Applying partial fixes in Spanner"); + com.google.cloud.spanner.Mutation customer3 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Customer 3") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .build(); + spannerResourceManager.write(customer3); + + // 6. Launch retryAllDLQ job + LOG.info("Launching retryAllDLQ job"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryAllDLQ"); + retryParams.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryParams.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryParams.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryParams.put("transformationCustomParameters", "mode=good"); + retryParams.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryParams.put("dlqMaxRetryCount", "20"); + retryParams.put("dlqRetryMinutes", "60"); + retryParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + + // Proprietary Drivers + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + retryParams.put("jdbcDriverJars", driverPath); + } + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + null, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryParams, + null, + null, + gcsResourceManager, + null, + null, + null); + LOG.info("RetryAllDLQ job launched: {}", retryJobInfo.jobId()); + assertThatPipeline(retryJobInfo).isRunning(); + + // 7. Wait for the retry job + LOG.info("Waiting for DLQ events to appear in retry and severe buckets after retry"); + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(15)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + LOG.info("Retry job completed processing successfully"); + + // 8. Verify Spanner state + assertTrue( + "id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); + assertTrue( + "id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + } + + private boolean rowExistsInSpanner(String tableName, String idColumnName, long id) { + List rows = + spannerResourceManager.readTableRecords(tableName, List.of(idColumnName)); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id) { + return true; + } + } + return false; + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private void insertDataInOracle() { + LOG.info("Inserting data in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (1, 'test1')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (999, 'test999')"); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA, String streamB, String dbB, String shardB) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " },\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardB + + "\",\n" + + " \"dbName\": \"" + + dbB + + "\",\n" + + " \"streamId\": \"" + + streamB + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java new file mode 100644 index 0000000000..67c3a4c35d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static java.util.Map.entry; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DatastreamToSpannerReservedKeywordsOracleIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DatastreamToSpannerReservedKeywordsOracleIT.class); + + private static final String ORACLE_DDL_RESOURCE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json"; + + private CloudOracleResourceManager oracleResourceManager; + private SpannerResourceManager spannerResourceManager; + private GcsResourceManager gcsResourceManager; + private PubsubResourceManager pubsubResourceManager; + private DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException { + LOG.info("Setting up Oracle resource manager..."); + oracleResourceManager = setUpOracleResourceManager(); + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + LOG.info( + "Spanner resource manager created with instance ID: {}", + spannerResourceManager.getInstanceId()); + + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + LOG.info("GCS resource manager created with bucket: {}", gcsResourceManager.getBucket()); + + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + LOG.info("Pub/Sub resource manager created."); + + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") /* from original */ + .build(); + LOG.info("Datastream resource manager created"); + } + + @After + public void cleanUp() { + LOG.info("Cleaning up resources..."); + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void testOracleReservedKeywords() throws Exception { + LOG.info("Executing Oracle DDL script..."); + try { + } catch (Exception e) { + LOG.info("Table true does not exist or could not be dropped: " + e.getMessage()); + } + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + flushOracleRedoLogs(oracleResourceManager); + + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + LOG.info("Generating session file content..."); + String sessionFileContent = + generateSessionFile( + 1, + oracleResourceManager.getDatabaseName(), + spannerResourceManager.getDatabaseId(), + List.of("true"), // Table name + SESSION_FILE); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("true"))) + .build(); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + + LOG.info("Launching Dataflow job..."); + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + "oracle-reserved-keywords", + null, + null, + "datastream-to-spanner-reserved-keywords", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + oracleSource); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(20)), + SpannerRowsCheck.builder(spannerResourceManager, "`true`").setMinRows(1).build()); + assertThatResult(result).meetsConditions(); + + List rows = null; + try { + rows = spannerResourceManager.readTableRecords("true", List.of("id", "ALL", "AND", "AS")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from `true`", e); + throw e; + } + + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns( + List.of( + Map.ofEntries( + entry("id", 1), entry("ALL", "all"), entry("AND", "and"), entry("AS", "as")))); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java new file mode 100644 index 0000000000..9b052cd89f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java @@ -0,0 +1,388 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerEventsIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerEventsIT.class); + + private static final String TABLE1 = "Users"; + private static final String TABLE2 = "Movie"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerEventsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleEventsIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerEventsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager); + } + + @Test + public void migrationTestWithUpdatesAndDeletes() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE1, + "backfill_users.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build(), + uploadDataStreamFile( + jobInfo, + TABLE1, + "cdc_users.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertUsersTableContents(); + } + + @Test + public void migrationTestWithInsertsOnly() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE2, + "backfill_movie.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertMovieTableContents(); + } + + @Test + public void interleavedAndFKAndIndexTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + "Articles", + "mysql_articles.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro", + gcsResourceManager), + uploadDataStreamFile( + jobInfo, + "Authors", + "mysql_authors.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro", + gcsResourceManager), + uploadDataStreamFile( + jobInfo, + "Books", + "mysql_books.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, "Articles") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(18)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTable(); + assertBooksTable(); + assertArticlesTable(); + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("id", 1); + row1.put("name", "Tester Kumar"); + row1.put("age", 30); + row1.put("subscribed", false); + row1.put("plan", "A"); + row1.put("startDate", Timestamp.parseTimestamp("2023-01-01T00:00:00Z")); + + Map row2 = new HashMap<>(); + row2.put("id", 3); + row2.put("name", "Tester Gupta"); + row2.put("age", 50); + row2.put("subscribed", false); + row2.put("plan", "Z"); + row2.put("startDate", Timestamp.parseTimestamp("2023-06-07T00:00:00Z")); + + Map row3 = new HashMap<>(); + row3.put("id", 4); + row3.put("name", "Tester"); + row3.put("age", 38); + row3.put("subscribed", true); + row3.put("plan", "D"); + row3.put("startDate", Timestamp.parseTimestamp("2023-09-10T00:00:00Z")); + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertMovieTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("id", 1); + row1.put("name", "movie1"); + row1.put("startTime", Timestamp.parseTimestamp("2023-01-01T12:12:12.000Z")); + + Map row2 = new HashMap<>(); + row2.put("id", 2); + row2.put("name", "movie2"); + row2.put("startTime", Timestamp.parseTimestamp("2023-11-25T17:10:12.000Z")); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, name, startTime from Movie where id in (1, 2)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + + ImmutableList numericVals = + spannerResourceManager.runQuery("select actor from Movie order by id"); + Assert.assertEquals(12345.09876, numericVals.get(0).getBigDecimal(0).doubleValue(), 0.00000001); + Assert.assertEquals(931.5123, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.00000001); + } + + private void assertAuthorsTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("author_id", 1); + row.put("name", "a1"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 2); + row.put("name", "a2"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 3); + row.put("name", "a3"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 4); + row.put("name", "a4"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "Book005"); + row.put("author_id", 3); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Book002"); + row.put("author_id", 3); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "Book004"); + row.put("author_id", 4); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("title", "Book005"); + row.put("author_id", 2); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertArticlesTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "Article001"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Article002"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Article004"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 4); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("name", "Article005"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 3); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java new file mode 100644 index 0000000000..ed6b1478f0 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -0,0 +1,272 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.com.google.common.io.Resources; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerFileOverridesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerFileOverridesIT.class); + + private static final String ORACLE_SCHEMA = + "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = + "oracle/OracleDataStreamToSpannerFileOverridesIT/override.json"; + private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = new HashSet<>(); + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerFileOverridesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); + + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + gcsResourceManager.uploadArtifact( + GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerFileOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() throws Exception { + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put( + "schemaOverridesFilePath", + getGcsPath(GCS_PATH_PREFIX + "/override.json", gcsResourceManager)); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck sendDataCondition = + new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1," + + " 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2," + + " 'Alice', 'Johnson')"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + sendDataCondition, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java new file mode 100644 index 0000000000..39b4a7a5ca --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -0,0 +1,491 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.function.Function; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerTemplateITBase; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(Parameterized.class) +public class OracleDataStreamToSpannerIT extends SpannerTemplateITBase { + + private static final Integer NUM_EVENTS = 10; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + + private String gcsPrefix; + private String dlqGcsPrefix; + + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private static final List COLUMNS = List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED); + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + dlqGcsPrefix = + getGcsPath(testName + "/dlq/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + @Test + public void testDataStreamOracleToSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToPostgresSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.POSTGRESQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToSpannerJson() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.JSON_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + private void simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat fileFormat, + Dialect spannerDialect, + Function paramsAdder) + throws IOException { + + cloudOracleSysUser = + (CloudOracleResourceManager) + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XE") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521)); + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); + // setUpOracleUser(oracleUser, oraclePassword); + // setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521)); + + SpannerResourceManager.Builder spannerResourceManagerBuilder = + SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) + .maybeUseStaticInstance() + .useCustomHost(spannerHost) + .setCredentials(credentials); + spannerResourceManager = spannerResourceManagerBuilder.build(); + + List tableNames = + List.of( + ("DatastreamToSpanner_1_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase(), + ("DatastreamToSpanner_2_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase()); + + tableNames.forEach( + tableName -> { + cloudSqlResourceManager.createTable(tableName, createJdbcSchema()); + }); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of(tableNames.get(0), tableNames.get(1)))) + .build(); + + createSpannerTables(tableNames, spannerDialect); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, fileFormat); + + Stream stream = + datastreamResourceManager.createStream( + "stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + datastreamResourceManager.startStream(stream); + + createPubSubNotifications(); + String jobName = PipelineUtils.createJobName(testName); + PipelineLauncher.LaunchConfig.Builder options = + paramsAdder + .apply( + PipelineLauncher.LaunchConfig.builder(jobName, specPath) + .addParameter("gcsPubSubSubscription", subscription.toString()) + .addParameter("dlqGcsPubSubSubscription", dlqSubscription.toString()) + .addParameter("streamName", stream.getName()) + .addParameter("instanceId", spannerResourceManager.getInstanceId()) + .addParameter("databaseId", spannerResourceManager.getDatabaseId()) + .addParameter("projectId", PROJECT) + .addParameter( + "deadLetterQueueDirectory", + getGcsPath(testName, gcsResourceManager) + "/dlq/") + .addParameter("spannerHost", spannerResourceManager.getSpannerHost()) + .addParameter( + "inputFileFormat", + fileFormat.equals( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT) + ? "avro" + : "json")) + .addParameter("workerMachineType", "n2-standard-4"); + + PipelineLauncher.LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(tableNames, cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(1)) + .setMinRows(NUM_EVENTS) + .build(), + changeJdbcData(tableNames, cdcEvents), + checkDestinationRows(tableNames, cdcEvents))) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel(createConfig(info, Duration.ofMinutes(20)), conditionCheck); + + checkSpannerTables(tableNames, cdcEvents); + assertThatResult(result).meetsConditions(); + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "INTEGER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "INTEGER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createPubSubNotifications() throws IOException { + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + + TopicName topic = pubsubResourceManager.createTopic("it"); + TopicName dlqTopic = pubsubResourceManager.createTopic("dlq"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "dlq-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + gcsResourceManager.createNotification(dlqTopic.toString(), dlqGcsPrefix.substring(1)); + } + + private void createSpannerTables(List tableNames, Dialect spannerDialect) { + boolean usingPg = Dialect.POSTGRESQL.equals(spannerDialect); + tableNames.forEach( + tableName -> { + String q = usingPg ? "\"" : "`"; + spannerResourceManager.executeDdlStatement( + "CREATE TABLE " + + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") + + " (" + + q + + ROW_ID + + q + + (usingPg ? " bigint " : " INT64 ") + + "NOT NULL, " + + q + + NAME + + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + + AGE + + q + + (usingPg ? " bigint, " : " INT64, ") + + q + + MEMBER + + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + + ENTRY_ADDED + + q + + (usingPg ? " character varying(200)" : " STRING(MAX)") + + (usingPg ? ", " : ") ") + + "PRIMARY KEY (" + + q + + ROW_ID + + q + + ")" + + (usingPg ? ")" : "")); + }); + } + + private ConditionCheck checkDestinationRows( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : tableNames) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + try { + checkSpannerTables(tableNames, cdcEvents); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + private void checkSpannerTables( + List tableNames, Map>> cdcEvents) { + tableNames.forEach( + tableName -> + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(tableName, COLUMNS)) + .hasRecordsUnorderedCaseInsensitiveColumns(cdcEvents.get(tableName))); + } + + private ConditionCheck writeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + values.put(ROW_ID, i); + values.put(NAME, RandomStringUtils.randomAlphabetic(10)); + values.put(AGE, new Random().nextInt(100)); + values.put(MEMBER, new Random().nextInt() % 2 == 0 ? "Y" : "N"); + values.put(ENTRY_ADDED, Instant.now().toString()); + rows.add(values); + } + + List> cdcRows = new ArrayList<>(); + for (Map row : rows) { + Map cdcRow = new HashMap<>(); + cdcRow.put(ROW_ID, row.get(ROW_ID)); + cdcRow.put(NAME, row.get(NAME)); + cdcRow.put(AGE, row.get(AGE)); + cdcRow.put(MEMBER, row.get(MEMBER)); + cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); + cdcRows.add(cdcRow); + } + cdcEvents.put(tableName, cdcRows); + + success &= cloudSqlResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } + + private ConditionCheck changeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send JDBC changes."; + } + + @Override + protected CheckResult check() { + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> newCdcEvents = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + if (i % 2 == 0) { + Map values = cdcEvents.get(tableName).get(i); + + String newName = values.get(NAME).toString().toUpperCase(); + int newAge = new Random().nextInt(100); + String newMember = Objects.equals(values.get(MEMBER).toString(), "Y") ? "N" : "Y"; + + values.put(NAME, newName); + values.put(AGE, newAge); + values.put(MEMBER, newMember); + + String updateSql = + "UPDATE " + + tableName + + " SET " + + NAME + + " = '" + + newName + + "', " + + AGE + + " = " + + newAge + + ", " + + MEMBER + + " = '" + + newMember + + "' WHERE " + + ROW_ID + + " = " + + i; + cloudSqlResourceManager.runSQLUpdate(updateSql); + newCdcEvents.add(values); + } else { + cloudSqlResourceManager.runSQLUpdate( + "DELETE FROM " + tableName + " WHERE " + ROW_ID + "=" + i); + } + } + cdcEvents.put(tableName, newCdcEvents); + messages.add(String.format("%d changes to %s", newCdcEvents.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig new file mode 100644 index 0000000000..8f97be3766 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig @@ -0,0 +1,455 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.function.Function; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerTemplateITBase; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(Parameterized.class) +public class OracleDataStreamToSpannerIT extends SpannerTemplateITBase { + + private static final Integer NUM_EVENTS = 10; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + + private String gcsPrefix; + private String dlqGcsPrefix; + + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private static final List COLUMNS = List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED); + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + dlqGcsPrefix = + getGcsPath(testName + "/dlq/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + @Test + public void testDataStreamOracleToSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToPostgresSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.POSTGRESQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToSpannerJson() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.JSON_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s", user)); + } + + private void simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat fileFormat, + Dialect spannerDialect, + Function paramsAdder) + throws IOException { + + cloudOracleSysUser = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword")) + .setDatabaseName("XEPDB1") + .build(); + String oracleUser = "U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .build(); + + SpannerResourceManager.Builder spannerResourceManagerBuilder = + SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) + .maybeUseStaticInstance() + .useCustomHost(spannerHost) + .setCredentials(credentials); + spannerResourceManager = spannerResourceManagerBuilder.build(); + + List tableNames = + List.of( + ("DatastreamToSpanner_1_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase(), + ("DatastreamToSpanner_2_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase()); + + tableNames.forEach( + tableName -> + cloudSqlResourceManager.createTable( + tableName, createJdbcSchema())); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of(tableNames.get(0), tableNames.get(1)))) + .build(); + + createSpannerTables(tableNames, spannerDialect); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, fileFormat); + + Stream stream = + datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + + createPubSubNotifications(); + String jobName = PipelineUtils.createJobName(testName); + PipelineLauncher.LaunchConfig.Builder options = + paramsAdder + .apply( + PipelineLauncher.LaunchConfig.builder(jobName, specPath) + .addParameter("gcsPubSubSubscription", subscription.toString()) + .addParameter("dlqGcsPubSubSubscription", dlqSubscription.toString()) + .addParameter("streamName", stream.getName()) + .addParameter("instanceId", spannerResourceManager.getInstanceId()) + .addParameter("databaseId", spannerResourceManager.getDatabaseId()) + .addParameter("projectId", PROJECT) + .addParameter( + "deadLetterQueueDirectory", + getGcsPath(testName, gcsResourceManager) + "/dlq/") + .addParameter("spannerHost", spannerResourceManager.getSpannerHost()) + .addParameter( + "inputFileFormat", + fileFormat.equals( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT) + ? "avro" + : "json")) + .addParameter("workerMachineType", "n2-standard-4"); + + PipelineLauncher.LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(tableNames, cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(1)) + .setMinRows(NUM_EVENTS) + .build(), + changeJdbcData(tableNames, cdcEvents), + checkDestinationRows(tableNames, cdcEvents))) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel(createConfig(info, Duration.ofMinutes(20)), conditionCheck); + + checkSpannerTables(tableNames, cdcEvents); + assertThatResult(result).meetsConditions(); + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "INTEGER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "INTEGER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createPubSubNotifications() throws IOException { + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + + TopicName topic = pubsubResourceManager.createTopic("it"); + TopicName dlqTopic = pubsubResourceManager.createTopic("dlq"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "dlq-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + gcsResourceManager.createNotification(dlqTopic.toString(), dlqGcsPrefix.substring(1)); + } + + private void createSpannerTables(List tableNames, Dialect spannerDialect) { + boolean usingPg = Dialect.POSTGRESQL.equals(spannerDialect); + tableNames.forEach( + tableName -> { + String q = usingPg ? "\"" : "`"; + spannerResourceManager.executeDdlStatement( + "CREATE TABLE " + + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") + + " (" + + q + ROW_ID + q + + (usingPg ? " bigint " : " INT64 ") + + "NOT NULL, " + + q + NAME + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + AGE + q + + (usingPg ? " bigint, " : " INT64, ") + + q + MEMBER + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + ENTRY_ADDED + q + + (usingPg ? " character varying(200)" : " STRING(MAX)") + + (usingPg ? ", " : ") ") + + "PRIMARY KEY (" + + q + ROW_ID + q + + ")" + + (usingPg ? ")" : "")); + }); + } + + private ConditionCheck checkDestinationRows( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : tableNames) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + try { + checkSpannerTables(tableNames, cdcEvents); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + private void checkSpannerTables( + List tableNames, Map>> cdcEvents) { + tableNames.forEach( + tableName -> + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(tableName, COLUMNS)) + .hasRecordsUnorderedCaseInsensitiveColumns(cdcEvents.get(tableName))); + } + + private ConditionCheck writeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + values.put(ROW_ID, i); + values.put(NAME, RandomStringUtils.randomAlphabetic(10)); + values.put(AGE, new Random().nextInt(100)); + values.put(MEMBER, new Random().nextInt() % 2 == 0 ? "Y" : "N"); + values.put(ENTRY_ADDED, Instant.now().toString()); + rows.add(values); + } + + List> cdcRows = new ArrayList<>(); + for (Map row : rows) { + Map cdcRow = new HashMap<>(); + cdcRow.put(ROW_ID, row.get(ROW_ID)); + cdcRow.put(NAME, row.get(NAME)); + cdcRow.put(AGE, row.get(AGE)); + cdcRow.put(MEMBER, row.get(MEMBER)); + cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); + cdcRows.add(cdcRow); + } + cdcEvents.put(tableName, cdcRows); + + success &= cloudSqlResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } + + private ConditionCheck changeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send JDBC changes."; + } + + @Override + protected CheckResult check() { + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> newCdcEvents = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + if (i % 2 == 0) { + Map values = cdcEvents.get(tableName).get(i); + + String newName = values.get(NAME).toString().toUpperCase(); + int newAge = new Random().nextInt(100); + String newMember = Objects.equals(values.get(MEMBER).toString(), "Y") ? "N" : "Y"; + + values.put(NAME, newName); + values.put(AGE, newAge); + values.put(MEMBER, newMember); + + String updateSql = + "UPDATE " + + tableName + + " SET " + NAME + " = '" + + newName + + "', " + AGE + " = " + + newAge + + ", " + MEMBER + " = '" + + newMember + + "' WHERE " + ROW_ID + " = " + + i; + cloudSqlResourceManager.runSQLUpdate(updateSql); + newCdcEvents.add(values); + } else { + cloudSqlResourceManager.runSQLUpdate( + "DELETE FROM " + tableName + " WHERE " + ROW_ID + "=" + i); + } + } + cdcEvents.put(tableName, newCdcEvents); + messages.add(String.format("%d changes to %s", newCdcEvents.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java new file mode 100644 index 0000000000..0f7426d0b0 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerMixedIT extends DataStreamToSpannerITBase { + + private String gcsPrefix; + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @Test + public void mixedMigrationTest() throws IOException { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL( + spannerResourceManager, + "oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + executeSqlScript( + cloudSqlResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Authors", "Books", "Genre"))) + .build(); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + gcsPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + + Stream stream = + datastreamResourceManager.createStream( + "stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + datastreamResourceManager.startStream(stream); + + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + TopicName topic = pubsubResourceManager.createTopic("it-topic"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + + TopicName dlqTopic = pubsubResourceManager.createTopic("it-dlq"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "it-dlq-sub"); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("gcsPubSubSubscription", subscription.toString()); + jobParams.put("dlqGcsPubSubSubscription", dlqSubscription.toString()); + jobParams.put("streamName", stream.getName()); + jobParams.put("instanceId", spannerResourceManager.getInstanceId()); + jobParams.put("databaseId", spannerResourceManager.getDatabaseId()); + jobParams.put("projectId", PROJECT); + jobParams.put("deadLetterQueueDirectory", getGcsPath(testName, gcsResourceManager) + "/dlq/"); + jobParams.put("workerMachineType", "n2-standard-4"); + jobParams.put("datastreamSourceType", "oracle"); + + PipelineLauncher.LaunchConfig.Builder options = + PipelineLauncher.LaunchConfig.builder(PipelineUtils.createJobName(testName), specPath) + .setParameters(jobParams) + .addEnvironment("ipConfiguration", "WORKER_IP_PRIVATE"); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + testName, + "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json", + null, + testName, + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager); + + assertThatPipeline(jobInfo).isRunning(); + + // Condition checks + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES + 10)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTableContents(); + assertBooksTableContents(); + } + + private ConditionCheck writeInitialData() { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Write JDBC data"; + } + + @Override + protected CheckResult check() { + List> authorRows = new ArrayList<>(); + authorRows.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + + List> bookRows = new ArrayList<>(); + bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); + bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); + bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); + + List> genreRows = new ArrayList<>(); + genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); + + for (Map r : authorRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", + r.get("author_id"), r.get("full_name"))); + } + for (Map r : bookRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", + r.get("id"), r.get("title"), r.get("author_id"))); + } + for (Map r : genreRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", + r.get("genre_id"), r.get("name"))); + } + + cloudOracleSysUser.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + + return new CheckResult(true, "Sent to Oracle."); + } + }; + } + + private void assertAuthorsTableContents() { + List> authorEvents = new ArrayList<>(); + authorEvents.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorEvents.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorEvents.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorEvents.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(authorEvents); + } + + private void assertBooksTableContents() { + List> bookEvents = new ArrayList<>(); + bookEvents.add(Map.of("id", 1, "title", "Pride and Prejudice")); + bookEvents.add(Map.of("id", 2, "title", "Oliver Twist")); + bookEvents.add(Map.of("id", 3, "title", "War and Peace")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(bookEvents); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java new file mode 100644 index 0000000000..4e8d6271e8 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -0,0 +1,330 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.JDBCSource; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * An integration test for {@link DataStreamToSpanner} Flex template which tests use-cases where a + * session file is required. + */ +@Category({TemplateIntegrationTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerSessionIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Category"; + private static final String TABLE2 = "Books"; + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json"; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerSessionIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + // Pre-insert books + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the" + + " Rings', 1)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and" + + " Prejudice', 2)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers" + + " Guide to the Galaxy', 3)"); + + // Pre-insert categories + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz'," + + " CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc'," + + " CURRENT_TIMESTAMP)"); + + flushOracleRedoLogs(oracleResourceManager); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + JDBCSource jdbcSource = + OracleSource.builder( + System.getProperty("cloudOracleHost"), + System.getProperty("cloudOracleUsername", "system"), + System.getProperty("cloudOraclePassword", "TestPassword123"), + 1521, + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + new HashMap<>() { + { + put( + oracleResourceManager.getUsername().toUpperCase(), + Arrays.asList("Category", "Books")); + } + }) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSessionIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerSessionIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + datastreamResourceManager, + oracleResourceManager); + } + + @Test + public void migrationTestWithRenameAndDrops() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertCategoryTableBackfillContents(); + + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def'," + + " CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi'," + + " CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); + oracleResourceManager.runSQLUpdate("DELETE FROM \"Category\" WHERE \"category_id\"=1"); + + flushOracleRedoLogs(oracleResourceManager); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertCategoryTableCdcContents(); + } + + @Test + public void migrationTestWithSyntheticPKAndExtraColumn() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertBooksBackfillContents(); + } + + private void assertCategoryTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 1); + row1.put("full_name", "xyz"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 2); + row2.put("full_name", "abc"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableCdcContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 2); + row1.put("full_name", "abc1"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 3); + row2.put("full_name", "def"); + + Map row3 = new HashMap<>(); + row3.put("category_id", 4); + row3.put("full_name", "ghi"); + + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + row.put("author_id", 2); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhikers Guide to the Galaxy"); + row.put("author_id", 3); + events.add(row); + + ImmutableList synthIds = spannerResourceManager.runQuery("select synth_id from Books"); + + Assert.assertEquals(3, synthIds.size()); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, title, author_id from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java new file mode 100644 index 0000000000..c507370f36 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -0,0 +1,234 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerTimezoneIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerTimezoneIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql"; + + private static final String TABLE1 = "DateData"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerTimezoneIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + oracleResourceManager = setUpOracleResourceManager(); + CloudOracleResourceManager.Builder sysBuilder = + CloudOracleResourceManager.builder(testName); + sysBuilder.setPassword("TestPassword123"); + sysBuilder.setHost("" + System.getProperty("cloudOracleHost") + ""); + sysBuilder.setPort(1521); + sysBuilder.setUsername("system"); + sysBuilder.setDatabaseName("XEPDB1"); + cloudOracleSysUser = + (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + try { + } catch (Exception e) { + // ignore + } + + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + // Explicit Log flush + flushOracleRedoLogs(cloudOracleSysUser); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("DateData"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "TimezoneIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerTimezoneIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + oracleResourceManager, + cloudOracleSysUser, + datastreamResourceManager); + } + + @Test + public void testTimezoneHandling() { + ConditionCheck insertData = + new ConditionCheck() { + @Override + protected String getDescription() { + return "Insert data into Oracle"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\")" + + " VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02" + + " 20:00:00.000000')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\")" + + " VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03" + + " 06:00:00.000000')"); + + flushOracleRedoLogs(cloudOracleSysUser); + return new CheckResult(true, "Data inserted successfully"); + } catch (Exception e) { + LOG.error("Failed to insert data into Oracle", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertData, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertUsersBackfillContents(); + } + + private void assertUsersBackfillContents() { + List> expectedRows = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("timestamp_column", "2024-02-02T00:00:00Z"); + row.put("datetime_column", "2024-02-02T10:00:00Z"); + expectedRows.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("timestamp_column", "2024-02-02T10:00:00Z"); + row.put("datetime_column", "2024-02-02T20:00:00Z"); + expectedRows.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, timestamp_column, datetime_column from DateData")) + .hasRecordsUnorderedCaseInsensitiveColumns(expectedRows); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java new file mode 100644 index 0000000000..470529c6de --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java @@ -0,0 +1,389 @@ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudSqlResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT + extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + + private static final int NUM_COLUMNS = 16; + private static final List COLUMNS = new ArrayList<>(); + private static CloudSqlResourceManager cloudSqlResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static HashSet + testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + private static DatastreamResourceManager datastreamResourceManager; + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add("DataStreamToSpanner_" + i + "_" + RandomStringUtils.randomAlphanumeric(5)); + } + for (int i = 1; i <= NUM_COLUMNS; i++) { + COLUMNS.add("COL_" + i); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + cloudSqlResourceManager = setUpOracleResourceManager(); + String sessionContent = generateBaseSchema(); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", cloudSqlResourceManager.getDatabaseName()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName))); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerForMax16KeyTablesPerDatabase() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + PipelineOperator.Config.builder() + .setJobId(jobInfo.jobId()) + .setProject(PROJECT) + .setRegion(REGION) + .setTimeoutAfter(Duration.ofMinutes(45)) + .setCheckAfter(Duration.ofSeconds(5)) + .build(), + conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(sessionTemplate); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + final int stringLength = 20; + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "STRING"); + colType.put("Len", stringLength); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + colType.get("Type")); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + public static Map createSessionTemplate() { + + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + Map sessionTemplate = + createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + sessionTemplate.put("DatabaseType", "oracle"); + return sessionTemplate; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + + for (int i = 0; i < NUM_COLUMNS; i++) { + sb.append("\"").append(COLUMNS.get(i)).append("\"").append(" VARCHAR2(20) NOT NULL"); + + if (i != NUM_COLUMNS - 1) { + sb.append(", "); + } + } + + sb.append(", PRIMARY KEY (\"").append(String.join("\", \"", COLUMNS)).append("\"))"); + + return sb.toString(); + } + + /** Creates Spanner tables dynamically with 16 columns as a composite primary key. */ + private void createSpannerTables() { + TABLE_NAMES.forEach( + tableName -> { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)).append(" STRING(20) NOT NULL "); + if (i != NUM_COLUMNS) { + sb.append(","); + } + } + sb.append(")"); + sb.append("PRIMARY KEY ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(")"); + + spannerResourceManager.executeDdlStatement(sb.toString()); + }); + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected @NotNull CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + + for (String tableName : TABLE_NAMES) { + List> rows = new ArrayList<>(); + + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + for (int j = 1; j <= NUM_COLUMNS; j++) { + values.put(COLUMNS.get(j - 1), RandomStringUtils.randomAlphabetic(10)); + } + rows.add(values); + } + + cdcEvents.put(tableName, rows); + success &= cloudSqlResourceManager.write(tableName, rows); + + try { + String dynamicHost = cloudSqlResourceManager.getHost(); + int dynamicPort = cloudSqlResourceManager.getPort(); + + // Set system property because the builder internally relies on it! + System.setProperty("cloudOracleHost", dynamicHost); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder("oracle_sys"); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setPassword("TestPassword123"); + sysBuilder.setHost(dynamicHost); + sysBuilder.setPort(dynamicPort); + sysBuilder.setDatabaseName("XE"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudOracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); + flushOracleRedoLogs(cloudOracleSysUser); + cloudOracleSysUser.cleanupAll(); + } catch (Throwable e) { + org.slf4j.LoggerFactory.getLogger( + OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class) + .error("FAILED TO EXECUTE SYSDBA SWITCH LOGFILE", e); + e.printStackTrace(); + } + + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java new file mode 100644 index 0000000000..a8eef0d641 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java @@ -0,0 +1,377 @@ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT + extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + private static final Integer NUM_COLUMNS = 2; + + private static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudSqlResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static String oracleSchema; + + private static final List COLUMNS = new ArrayList<>(); + private static HashSet + testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add("DataStreamToSpanner_" + i + "_" + RandomStringUtils.randomAlphanumeric(5)); + } + for (int i = 1; i < NUM_COLUMNS; i++) { + COLUMNS.add("col_" + i); + } + COLUMNS.add( + NUM_COLUMNS - 1, "col_" + (NUM_COLUMNS - 1) + RandomStringUtils.randomAlphanumeric(25)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + cloudSqlResourceManager = setUpOracleResourceManager(); + oracleSchema = + ("TEST_U" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5)) + .toUpperCase(); + cloudSqlResourceManager.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s1A", oracleSchema, oracleSchema)); + cloudSqlResourceManager.runSQLUpdate( + String.format("GRANT DBA TO %s CONTAINER=ALL", oracleSchema)); + cloudSqlResourceManager.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", oracleSchema)); + String sessionContent = generateBaseSchema(); + sessionContent = sessionContent.replaceAll("SRC_DATABASE", oracleSchema); + sessionContent = + sessionContent.replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + org.apache.beam.it.gcp.datastream.OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleSchema, TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT instance : + testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName)); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamMySqlToSpannerForMaxTableNameWithMaxColumnNames() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(10)), conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + sessionTemplate.put("DatabaseType", "oracle"); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + return gson.toJson(sessionTemplate); + } + + public static Map createSessionTemplate() { + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + return createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "NUMERIC"); + colType.put("Len", 0); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + " MEDIUMTEXT"); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE \"") + .append(oracleSchema) + .append("\".\"") + .append(tableName) + .append("\" ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append("\"").append(COLUMNS.get(i - 1)).append("\" NUMERIC NOT NULL"); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(", PRIMARY KEY (\"").append(COLUMNS.get(0)).append("\"))"); + return sb.toString(); + } + + private void createSpannerTables() { + for (String tableName : TABLE_NAMES) { + List columns = new ArrayList<>(); + columns.add(COLUMNS.get(0) + " NUMERIC NOT NULL"); + + for (int i = 2; i <= NUM_COLUMNS; i++) { + columns.add(COLUMNS.get(i - 1) + " NUMERIC"); + } + + String ddlStatement = + String.format( + "CREATE TABLE %s (%s) PRIMARY KEY (%s)", + tableName, String.join(", ", columns), COLUMNS.get(0)); + + spannerResourceManager.executeDdlStatement(ddlStatement); + } + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : TABLE_NAMES) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + StringBuilder cols = new StringBuilder(); + StringBuilder vals = new StringBuilder(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + values.put(COLUMNS.get(ci - 1), ci); + cols.append("\"").append(COLUMNS.get(ci - 1)).append("\""); + vals.append(ci); + if (ci != NUM_COLUMNS) { + cols.append(", "); + vals.append(", "); + } + } + rows.add(values); + String sql = + "INSERT INTO \"" + + oracleSchema + + "\".\"" + + tableName + + "\" (" + + cols.toString() + + ") VALUES (" + + vals.toString() + + ")"; + try { + cloudSqlResourceManager.runSQLUpdate(sql); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + } catch (Exception e) { + success = false; + } + } + cdcEvents.put(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + flushOracleRedoLogs(cloudSqlResourceManager); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java new file mode 100644 index 0000000000..394166a219 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -0,0 +1,576 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerDataTypesIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String TABLE_PREFIX = "T_SPRG_"; + private static final List UNSUPPORTED_TYPE_TABLES = + List.of( + "T_SPRG_R27_INTERVAL_Y", + "T_SPRG_R28_INTERVAL_D", + "T_SPRG_R30_LONG_RAW", + "T_SPRG_R34_BFILE", + "T_SPRG_R35_LONG", + "T_SPRG_R36_ROWID", + "T_SPRG_R37_UROWID", + "T_SPRG_R38_BOOLEAN", + "T_SPRG_R39_JSON", + "T_SPRG_R40_XMLTYPE", + "T_SPRG_R41_SDO_GEOMET", + "T_SPRG_R42_SDO_TOPO_G", + "T_SPRG_R43_SDO_GEORAS", + "T_SPRG_R44_VECTOR", + "T_SPRG_R45_ANYDATA", + "T_SPRG_R46_ANYTYPE", + "T_SPRG_R47_ANYDATASET", + "T_SPRG_R48_URITYPE", + "T_SPRG_R49_DBURITYPE", + "T_SPRG_R50_XDBURITYPE", + "T_SPRG_R51_HTTPURITYP", + "T_SPRG_R52_EXPRESSION", + "T_SPRG_R53_ORDAUDIO", + "T_SPRG_R54_ORDIMAGE", + "T_SPRG_R55_ORDVIDEO", + "T_SPRG_R56_ORDDOC", + "T_SPRG_R57_SI_STILLIM", + "T_SPRG_R58_SI_COLOR", + "T_SPRG_R59_SI_AVERAGE", + "T_SPRG_R60_SI_COLORHI", + "T_SPRG_R61_SI_POSITIO", + "T_SPRG_R62_SI_TEXTURE", + "T_SPRG_R63_SI_FEATURE", + "T_SPRG_R64_MLSLABEL", + "T_SPRG_R65_REF", + "T_SPRG_R66_REF_CURSOR", + "T_SPRG_R67_VARRAY", + "T_SPRG_R68_NESTED_TAB", + "T_SPRG_R69_ASSOCIATIV", + "T_SPRG_R70_OBJECT_TYP"); + + private static boolean initialized = false; + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + + private static HashSet testInstances = new HashSet<>(); + private static final String[] ORACLE_DDL_COMMANDS = + new String[] { + "CREATE TABLE T_SPRG_R0_VARCHAR2 (ID NUMBER PRIMARY KEY, COL_0 VARCHAR2(255))", + "ALTER TABLE T_SPRG_R0_VARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R0_VARCHAR2 (ID, COL_0) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R1_VARCHAR (ID NUMBER PRIMARY KEY, COL_1 VARCHAR(255))", + "ALTER TABLE T_SPRG_R1_VARCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R1_VARCHAR (ID, COL_1) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R2_CHAR (ID NUMBER PRIMARY KEY, COL_2 CHAR(255))", + "ALTER TABLE T_SPRG_R2_CHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R2_CHAR (ID, COL_2) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R3_CHARACTER (ID NUMBER PRIMARY KEY, COL_3 CHARACTER(255))", + "ALTER TABLE T_SPRG_R3_CHARACTER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R3_CHARACTER (ID, COL_3) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID NUMBER PRIMARY KEY, COL_4 NVARCHAR2(255))", + "ALTER TABLE T_SPRG_R4_NVARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R4_NVARCHAR2 (ID, COL_4) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R5_NCHAR (ID NUMBER PRIMARY KEY, COL_5 NCHAR(255))", + "ALTER TABLE T_SPRG_R5_NCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R5_NCHAR (ID, COL_5) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID NUMBER PRIMARY KEY, COL_6 NCHAR VARYING(255))", + "ALTER TABLE T_SPRG_R6_NCHAR_VARY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R6_NCHAR_VARY (ID, COL_6) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R7_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_7 NATIONAL CHARACTER(255))", + "ALTER TABLE T_SPRG_R7_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R7_NATIONAL_C (ID, COL_7) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R8_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_8 NATIONAL CHAR(255))", + "ALTER TABLE T_SPRG_R8_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R8_NATIONAL_C (ID, COL_8) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER" + + " VARYING(255))", + "ALTER TABLE T_SPRG_R9_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R9_NATIONAL_C (ID, COL_9) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR" + + " VARYING(255))", + "ALTER TABLE T_SPRG_R10_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R10_NATIONAL_C (ID, COL_10) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R11_NUMBER (ID NUMBER PRIMARY KEY, COL_11 NUMBER)", + "ALTER TABLE T_SPRG_R11_NUMBER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R11_NUMBER (ID, COL_11) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R12_NUMERIC (ID NUMBER PRIMARY KEY, COL_12 NUMERIC)", + "ALTER TABLE T_SPRG_R12_NUMERIC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R12_NUMERIC (ID, COL_12) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R13_DECIMAL (ID NUMBER PRIMARY KEY, COL_13 DECIMAL)", + "ALTER TABLE T_SPRG_R13_DECIMAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R13_DECIMAL (ID, COL_13) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R14_DEC (ID NUMBER PRIMARY KEY, COL_14 DEC)", + "ALTER TABLE T_SPRG_R14_DEC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R14_DEC (ID, COL_14) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R15_FLOAT (ID NUMBER PRIMARY KEY, COL_15 FLOAT)", + "ALTER TABLE T_SPRG_R15_FLOAT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R15_FLOAT (ID, COL_15) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID NUMBER PRIMARY KEY, COL_16 DOUBLE PRECISION)", + "ALTER TABLE T_SPRG_R16_DOUBLE_PRE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R16_DOUBLE_PRE (ID, COL_16) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R17_REAL (ID NUMBER PRIMARY KEY, COL_17 REAL)", + "ALTER TABLE T_SPRG_R17_REAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R17_REAL (ID, COL_17) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R18_BINARY_FLO (ID NUMBER PRIMARY KEY, COL_18 BINARY_FLOAT)", + "ALTER TABLE T_SPRG_R18_BINARY_FLO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R18_BINARY_FLO (ID, COL_18) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R19_BINARY_DOU (ID NUMBER PRIMARY KEY, COL_19 BINARY_DOUBLE)", + "ALTER TABLE T_SPRG_R19_BINARY_DOU ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R19_BINARY_DOU (ID, COL_19) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R20_INTEGER (ID NUMBER PRIMARY KEY, COL_20 INTEGER)", + "ALTER TABLE T_SPRG_R20_INTEGER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R20_INTEGER (ID, COL_20) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R21_INT (ID NUMBER PRIMARY KEY, COL_21 INT)", + "ALTER TABLE T_SPRG_R21_INT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R21_INT (ID, COL_21) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R22_SMALLINT (ID NUMBER PRIMARY KEY, COL_22 SMALLINT)", + "ALTER TABLE T_SPRG_R22_SMALLINT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R22_SMALLINT (ID, COL_22) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R23_DATE (ID NUMBER PRIMARY KEY, COL_23 DATE)", + "ALTER TABLE T_SPRG_R23_DATE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R23_DATE (ID, COL_23) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R24_TIMESTAMP (ID NUMBER PRIMARY KEY, COL_24 TIMESTAMP)", + "ALTER TABLE T_SPRG_R24_TIMESTAMP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R24_TIMESTAMP (ID, COL_24) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME" + + " ZONE)", + "ALTER TABLE T_SPRG_R25_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R25_TIMESTAMP_ (ID, COL_25) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL" + + " TIME ZONE)", + "ALTER TABLE T_SPRG_R26_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R26_TIMESTAMP_ (ID, COL_26) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R27_INTERVAL_Y ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R27_INTERVAL_Y (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R28_INTERVAL_D (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R28_INTERVAL_D ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R28_INTERVAL_D (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R29_RAW (ID NUMBER PRIMARY KEY, COL_29 RAW(255))", + "ALTER TABLE T_SPRG_R29_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R29_RAW (ID, COL_29) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R30_LONG_RAW (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R30_LONG_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R30_LONG_RAW (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R31_BLOB (ID NUMBER PRIMARY KEY, COL_31 BLOB)", + "ALTER TABLE T_SPRG_R31_BLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R31_BLOB (ID, COL_31) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R32_CLOB (ID NUMBER PRIMARY KEY, COL_32 CLOB)", + "ALTER TABLE T_SPRG_R32_CLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R32_CLOB (ID, COL_32) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R33_NCLOB (ID NUMBER PRIMARY KEY, COL_33 NCLOB)", + "ALTER TABLE T_SPRG_R33_NCLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R33_NCLOB (ID, COL_33) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R34_BFILE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R34_BFILE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R34_BFILE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R35_LONG (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R35_LONG ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R35_LONG (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R36_ROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R36_ROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R36_ROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R37_UROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R37_UROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R37_UROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R38_BOOLEAN (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R38_BOOLEAN ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R38_BOOLEAN (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R39_JSON (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R39_JSON ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R39_JSON (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R40_XMLTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R40_XMLTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R40_XMLTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R41_SDO_GEOMET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R41_SDO_GEOMET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R42_SDO_TOPO_G ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R42_SDO_TOPO_G (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R43_SDO_GEORAS ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R43_SDO_GEORAS (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R44_VECTOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R44_VECTOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R44_VECTOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R45_ANYDATA (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R45_ANYDATA ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R45_ANYDATA (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R46_ANYTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R46_ANYTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R46_ANYTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R47_ANYDATASET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R47_ANYDATASET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R47_ANYDATASET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R48_URITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R48_URITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R48_URITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R49_DBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R49_DBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R49_DBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R50_XDBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R50_XDBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R50_XDBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R51_HTTPURITYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R51_HTTPURITYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R51_HTTPURITYP (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R52_EXPRESSION (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R52_EXPRESSION ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R52_EXPRESSION (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R53_ORDAUDIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R53_ORDAUDIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R53_ORDAUDIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R54_ORDIMAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R54_ORDIMAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R54_ORDIMAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R55_ORDVIDEO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R55_ORDVIDEO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R55_ORDVIDEO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R56_ORDDOC (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R56_ORDDOC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R56_ORDDOC (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R57_SI_STILLIM (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R57_SI_STILLIM ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R57_SI_STILLIM (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R58_SI_COLOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R58_SI_COLOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R58_SI_COLOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R59_SI_AVERAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R59_SI_AVERAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R60_SI_COLORHI (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R60_SI_COLORHI ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R60_SI_COLORHI (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R61_SI_POSITIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R61_SI_POSITIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R61_SI_POSITIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R62_SI_TEXTURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R62_SI_TEXTURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R63_SI_FEATURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R63_SI_FEATURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R63_SI_FEATURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R64_MLSLABEL (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R64_MLSLABEL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R64_MLSLABEL (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R65_REF (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R65_REF ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R65_REF (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R66_REF_CURSOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R66_REF_CURSOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R66_REF_CURSOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R67_VARRAY (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R67_VARRAY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R67_VARRAY (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R68_NESTED_TAB (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R68_NESTED_TAB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R68_NESTED_TAB (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R69_ASSOCIATIV ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R69_ASSOCIATIV (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R70_OBJECT_TYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R70_OBJECT_TYP (ID, DUMMY) VALUES (1, 'test')" + }; + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerDataTypesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + // Build isolated RM + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + LOG.info("Executing Oracle DDL commands efficiently using a single JDBC connection..."); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + java.sql.Statement stmt = conn.createStatement()) { + for (String cmd : ORACLE_DDL_COMMANDS) { + stmt.execute(cmd); + } + } catch (Exception e) { + throw new RuntimeException( + "Failed to efficiently upload Oracle DDL via single connection", e); + } + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + LOG.info("Cleaning up resources..."); + for (OracleDatastreamToSpannerDataTypesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void testOracleDataTypes() throws Exception { + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + Map>> expectedData = getExpectedData(); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + oracleResourceManager.getUsername().toUpperCase(), + getAllowedTables(expectedData))) + .build(); + + LOG.info("Launching Dataflow job..."); + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + "oracle-data-types", + null, + null, + "oracle-datastream-to-spanner-data-types", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>(), + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck condition = buildConditionCheck(spannerResourceManager, expectedData); + LOG.info("Waiting for pipeline to process data..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + condition); + assertThatResult(result).meetsConditions(); + + validateResult(spannerResourceManager, expectedData); + } + + private void validateResult( + SpannerResourceManager resourceManager, Map>> expectedData) { + for (Map.Entry>> entry : expectedData.entrySet()) { + String type = entry.getKey(); + String tableName = TABLE_PREFIX + type; + LOG.info("Asserting type: {}", type); + + List rows = + resourceManager.readTableRecords(tableName, entry.getValue().get(0).keySet()); + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(entry.getValue()); + } + } + + private List> createRows(String colName, Object... values) { + List vals = Arrays.asList(values); + List> rows = new ArrayList<>(); + for (int i = 0; i < vals.size(); i++) { + Map row = new HashMap<>(); + row.put("ID", i + 1L); + row.put(colName, vals.get(i)); + rows.add(row); + } + return rows; + } + + private List getAllowedTables(Map>> expectedData) { + List tableNames = new ArrayList<>(expectedData.size() + UNSUPPORTED_TYPE_TABLES.size()); + for (String tablePrefix : expectedData.keySet()) { + tableNames.add(TABLE_PREFIX + tablePrefix); + } + tableNames.addAll(UNSUPPORTED_TYPE_TABLES); + return tableNames; + } + + private ConditionCheck buildConditionCheck( + SpannerResourceManager resourceManager, Map>> expectedData) { + ConditionCheck combinedCondition = null; + for (Map.Entry>> entry : expectedData.entrySet()) { + String tableName = TABLE_PREFIX + entry.getKey(); + int numRows = entry.getValue().size(); + ConditionCheck c = + SpannerRowsCheck.builder(resourceManager, tableName).setMinRows(numRows).build(); + if (combinedCondition == null) { + combinedCondition = c; + } else { + combinedCondition = combinedCondition.and(c); + } + } + return combinedCondition; + } + + private Map>> getExpectedData() { + Map>> expectedData = new HashMap<>(); + expectedData.put("R0_VARCHAR2", createRows("COL_0", "NULL")); + expectedData.put("R1_VARCHAR", createRows("COL_1", "NULL")); + expectedData.put("R2_CHAR", createRows("COL_2", "NULL")); + expectedData.put("R3_CHARACTER", createRows("COL_3", "NULL")); + expectedData.put("R4_NVARCHAR2", createRows("COL_4", "NULL")); + expectedData.put("R5_NCHAR", createRows("COL_5", "NULL")); + expectedData.put("R6_NCHAR_VARY", createRows("COL_6", "NULL")); + expectedData.put("R7_NATIONAL_C", createRows("COL_7", "NULL")); + expectedData.put("R8_NATIONAL_C", createRows("COL_8", "NULL")); + expectedData.put("R9_NATIONAL_C", createRows("COL_9", "NULL")); + expectedData.put("R10_NATIONAL_C", createRows("COL_10", "NULL")); + expectedData.put("R11_NUMBER", createRows("COL_11", "NULL")); + expectedData.put("R12_NUMERIC", createRows("COL_12", "NULL")); + expectedData.put("R13_DECIMAL", createRows("COL_13", "NULL")); + expectedData.put("R14_DEC", createRows("COL_14", "NULL")); + expectedData.put("R15_FLOAT", createRows("COL_15", "NULL")); + expectedData.put("R16_DOUBLE_PRE", createRows("COL_16", "NULL")); + expectedData.put("R17_REAL", createRows("COL_17", "NULL")); + expectedData.put("R18_BINARY_FLO", createRows("COL_18", "NULL")); + expectedData.put("R19_BINARY_DOU", createRows("COL_19", "NULL")); + expectedData.put("R20_INTEGER", createRows("COL_20", "NULL")); + expectedData.put("R21_INT", createRows("COL_21", "NULL")); + expectedData.put("R22_SMALLINT", createRows("COL_22", "NULL")); + expectedData.put("R23_DATE", createRows("COL_23", "NULL")); + expectedData.put("R24_TIMESTAMP", createRows("COL_24", "NULL")); + expectedData.put("R25_TIMESTAMP_", createRows("COL_25", "NULL")); + expectedData.put("R26_TIMESTAMP_", createRows("COL_26", "NULL")); + expectedData.put("R29_RAW", createRows("COL_29", "NULL")); + expectedData.put("R31_BLOB", createRows("COL_31", "NULL")); + expectedData.put("R32_CLOB", createRows("COL_32", "NULL")); + expectedData.put("R33_NCLOB", createRows("COL_33", "NULL")); + + return expectedData; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java new file mode 100644 index 0000000000..71ff01c668 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -0,0 +1,404 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for sharded data migration using a single Dataflow job for Oracle Source + * targeting Spanner. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerSingleDFShardedMigrationIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerSingleDFShardedMigrationIT.class); + + private static final String TABLE = "Users"; + + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql"; + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static DatastreamResourceManager datastreamResourceManager; + private static String streamNameA; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerSingleDFShardedMigrationIT.class) { + testInstances.add(this); + if (spannerResourceManager == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + } + if (pubsubResourceManager == null) { + pubsubResourceManager = setUpPubSubResourceManager(); + } + if (gcsResourceManager == null) { + gcsResourceManager = setUpSpannerITGcsResourceManager(); + } + + if (jobInfo == null) { + + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); + + jdbcResourceManagerShardA = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521)); + try { + } catch (Exception e) { + } + + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + + datastreamResourceManager = + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder( + testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = + org.apache.beam.it.gcp.datastream.OracleSource.builder( + System.getProperty("cloudOracleHost"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") + .setAllowedTables(java.util.Map.of(oracleUser, java.util.List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + "oracle-shard-cdc/cdc/", + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat + .AVRO_FILE_FORMAT); + + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5) + .toLowerCase(), + sourceConfig, + destinationConfig); + + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + String shardConfig = generateSourceConfig(streamNameA, oracleUser, "L1"); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", + shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + String sessionFileContent = + com.google.common.io.Resources.toString( + com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDatastreamToSpannerSingleDFShardedMigrationIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager); + } + + @Test + public void multiShardMigration() throws Exception { + + // Check if pipeline is running + assertThatPipeline(jobInfo).isRunning(); + + insertDataInOracle(); + + // Setup condition + ConditionCheck rowsConditionCheck = + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + // Sleep for cutover time to wait till all CDCs propagate. + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + + // Assert specific rows + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + + LOG.info("Wait, executing Hard-boot raw JDBC switch logfile..."); + // Force log file archive - needed so Datastream can see changes which are read from archived + // log files. + // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + + // System.getProperty("cloudOracleHost") + ":1521 (User: system, Pass: TestPassword123) calling + // ALTER + // SYSTEM SWITCH LOGFILE. + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + LOG.warn("Error while executing ALTER SYSTEM SWITCH LOGFILE. Using framework fallback...", e); + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "Tester1"); + row.put("age", 20); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Tester3"); + row.put("age", 103); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 13); + row.put("name", "Tester13"); + row.put("age", 113); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("name", "Tester4"); + row.put("age", 104); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 5); + row.put("name", "Tester5"); + row.put("age", 105); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 6); + row.put("name", "Tester6"); + row.put("age", 106); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 7); + row.put("name", "Tester7"); + row.put("age", 107); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 8); + row.put("name", "Tester8"); + row.put("age", 108); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 9); + row.put("name", "Tester9"); + row.put("age", 109); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 10); + row.put("name", "Tester10"); + row.put("age", 110); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 11); + row.put("name", "Tester11"); + row.put("age", 111); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 12); + row.put("name", "Tester12"); + row.put("age", 112); + row.put("migration_shard_id", "L1"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig(String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java new file mode 100644 index 0000000000..c9c375b245 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static java.util.Map.entry; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.Statement; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerTableAndIndexLimitsIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerTableAndIndexLimitsIT.class); + + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json"; + + private static final String LARGE_KEY_TABLE = "LargeKey"; + private static final String LARGE_CELL_TABLE = "LargeCell"; + private static final List TABLES = List.of(LARGE_KEY_TABLE, LARGE_CELL_TABLE); + + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerTableAndIndexLimitsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + LOG.info("Setting up Oracle resource manager..."); + oracleResourceManager = setUpOracleResourceManager(); + LOG.info("Oracle resource manager created with URI: {}", oracleResourceManager.getUri()); + + try { + } catch (Exception e) { + } + try { + } catch (Exception e) { + } + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + LOG.info( + "Spanner resource manager created with instance ID: {}", + spannerResourceManager.getInstanceId()); + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + LOG.info("GCS resource manager created with bucket: {}", gcsResourceManager.getBucket()); + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + LOG.info("Pub/Sub resource manager created."); + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + LOG.info("Datastream resource manager created"); + + LOG.info("Executing Oracle DDL script..."); + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // Pre-insert testing data for LargeCell + try (Connection conn = + DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + PreparedStatement pstmt = + conn.prepareStatement( + "INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\"," + + " \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { + + byte[] bytes = new byte[4000]; + Arrays.fill(bytes, (byte) 'b'); + pstmt.setBytes(1, bytes); + pstmt.setString(2, new String(bytes, StandardCharsets.UTF_8)); + pstmt.executeUpdate(); + } catch (Exception e) { + LOG.error("Failed to insert LargeCell locally: ", e); + throw new RuntimeException("Failed to insert LargeCell", e); + } + + // Force a log switch and insert to LargeCell. + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@" + + oracleResourceManager.getHost() + + ":" + + oracleResourceManager.getPort() + + "XEPDB1", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement(); ) { + flushOracleRedoLogs(null); + } catch (Exception e) { + LOG.warn("Failed to switch log file natively: ", e); + } + + LOG.info("Generating session file content..."); + String sessionFileContent = + generateSessionFile( + 1, + oracleResourceManager.getDatabaseName(), + spannerResourceManager.getDatabaseId(), + TABLES, + SESSION_FILE); + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), TABLES)) + .build(); + + LOG.info("Launching Dataflow job..."); + jobInfo = + launchDataflowJob( + "oracle-table-and-index-limits", + null, + null, + "datastream-to-spanner-table-and-index-limits", + spannerResourceManager, + pubsubResourceManager, + Map.of("dlqMaxRetryCount", "1", "inputFileFormat", "avro"), + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + LOG.info("Cleaning up resources..."); + for (OracleDatastreamToSpannerTableAndIndexLimitsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void testKeySize() { + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data for key size test..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, LARGE_KEY_TABLE) + .setMinRows(1) + .build()); + assertThatResult(result).meetsConditions(); + List rows = null; + try { + rows = + spannerResourceManager.readTableRecords(LARGE_KEY_TABLE, List.of("pk_col1", "pk_col2")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from {}", LARGE_KEY_TABLE, e); + throw e; + } + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns( + List.of( + Map.ofEntries( + // `com.google.cloud.spanner.Value.StringImpl::valueToString`, which truncates + // strings to 33 characters + entry("pk_col1", "A".repeat(33) + "..."), + entry("pk_col2", "B".repeat(33) + "...")))); + } + + @Test + public void testCellSize() { + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data for cell size test..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, LARGE_CELL_TABLE) + .setMinRows(1) + .build()); + assertThatResult(result).meetsConditions(); + List rows = null; + try { + rows = spannerResourceManager.readTableRecords(LARGE_CELL_TABLE, List.of("id")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from {}", LARGE_CELL_TABLE, e); + throw e; + } + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(List.of(Map.ofEntries(entry("id", 1)))); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java new file mode 100644 index 0000000000..981793cdd8 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT + extends DataStreamToSpannerITBase { + private static final int STRING_LENGTH = 200; + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + private static final String LARGE_BLOB_ADDED = "LARGE_BLOB"; + + private static final List COLUMNS = + List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB_ADDED); + + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static HashSet + testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add( + "DATASTREAM_WIDETB_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + String sessionContent = generateBaseSchema(); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", oracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleDataStreamToSpannerWideRowFor9MibTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>(), + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + oracleResourceManager.createTable(tableName, createJdbcSchema()); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerFor9MBTablesPerDatabase() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + Map>> cdcEvents = new LinkedHashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(20)), conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(sessionTemplate); + } + + private Map createSessionTemplate() { + List colIds = Arrays.asList("c1", "c2", "c3", "c4", "c5", "c6"); + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + if (j == 6) { + colType.put("Type", "BLOB"); + colType.put("Len", 0); + } else if (j % 2 == 0) { + colType.put("Type", "VARCHAR2"); + colType.put("Len", STRING_LENGTH); + } else { + colType.put("Type", "NUMBER"); + colType.put("Len", 0); + } + colType.put("IsArray", false); + colType.put("Name", "column_" + j); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: column_" + j + " " + colType.get("Type")); + colTypeConfigs.add(colType); + } + List> primaryKeys = + List.of(Map.of("ColId", colIds.get(0), "Desc", false, "Order", 1)); + Map sessionTemplate = + createSessionTemplate(NUM_TABLES, colTypeConfigs, primaryKeys); + sessionTemplate.put("DatabaseType", "oracle"); + return sessionTemplate; + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "NUMBER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "NUMBER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + columns.put(LARGE_BLOB_ADDED, "BLOB"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createSpannerTables() { + TABLE_NAMES.forEach( + tableName -> + spannerResourceManager.executeDdlStatement( + "CREATE TABLE IF NOT EXISTS " + + tableName + + " (" + + ROW_ID + + " INT64 NOT NULL, " + + NAME + + " STRING(1024), " + + AGE + + " INT64, " + + MEMBER + + " STRING(1024), " + + ENTRY_ADDED + + " STRING(1024), " + + LARGE_BLOB_ADDED + + " BYTES(9437184)" // 9 MiB + + ") PRIMARY KEY (" + + ROW_ID + + ")")); + } + + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + return new CheckResult(true, "Spanner tables contain expected rows."); + } + }; + } + + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + Random random = new Random(); + byte[] largeData = new byte[9 * 1024 * 1024]; + random.nextBytes(largeData); + + for (String tableName : TABLE_NAMES) { + List> rows = new ArrayList<>(); + + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new LinkedHashMap<>(); + values.put(COLUMNS.get(0), i); + values.put(COLUMNS.get(1), RandomStringUtils.randomAlphabetic(10)); + values.put(COLUMNS.get(2), random.nextInt(100)); + values.put(COLUMNS.get(3), random.nextBoolean() ? "Y" : "N"); + values.put(COLUMNS.get(4), Instant.now().toString()); + byte[] rowData = largeData.clone(); + values.put(COLUMNS.get(5), rowData); + rows.add(values); + } + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + java.sql.PreparedStatement pstmt = + conn.prepareStatement( + "INSERT INTO " + + tableName + + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?," + + " ?, ?, ?)")) { + for (Map values : rows) { + pstmt.setLong(1, ((Number) values.get(COLUMNS.get(0))).longValue()); + pstmt.setString(2, (String) values.get(COLUMNS.get(1))); + pstmt.setLong(3, ((Number) values.get(COLUMNS.get(2))).longValue()); + pstmt.setString(4, (String) values.get(COLUMNS.get(3))); + pstmt.setString(5, (String) values.get(COLUMNS.get(4))); + pstmt.setBytes(6, (byte[]) values.get(COLUMNS.get(5))); + pstmt.addBatch(); + } + pstmt.executeBatch(); + } catch (Exception e) { + success = false; + messages.add("Failed to write to " + tableName + ": " + e.getMessage()); + } + + rows.forEach( + values -> + values.put( + COLUMNS.get(5), + Base64.getEncoder().encodeToString((byte[]) values.get(COLUMNS.get(5))))); + + cdcEvents.put(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java new file mode 100644 index 0000000000..5859ce3c95 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -0,0 +1,693 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck.CheckResult; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for datatype conversions for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseDDLIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseDDLIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static final String TABLE1 = "AllDatatypeColumns"; + private static final String TABLE2 = "AllDatatypeColumns2"; + private static final String TABLE3 = "DatatypeColumnsWithSizes"; + private static final String TABLE4 = "DatatypeColumnsReducedSizes"; + private static final String TABLE5 = "Users"; + private static final String TABLE6 = "Books"; + private static final String TABLE7 = "Authors"; + private static final String TABLE8 = "Singers"; + + private static final String TRANSFORMATION_TABLE = "AllDatatypeTransformation"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager + datastreamResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseDDLIT.class) { + testInstances.add(this); + if (jobInfo == null) { + oracleResourceManager = setUpOracleResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + createAndUploadJarToGcs("OracleSeparateShadowTableDatabaseDDLIT", gcsResourceManager); + CustomTransformation customTransformation = + CustomTransformation.builder( + "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") + .build(); + + String[] tables = { + "AllDatatypeColumns", + "AllDatatypeColumns2", + "DatatypeColumnsWithSizes", + "DatatypeColumnsReducedSizes", + "Users", + "Authors", + "AllDatatypeTransformation", + "Singers", + "Books" + }; + for (String table : tables) { + try { + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + } catch (Exception e) { + } + executeSqlScript( + oracleResourceManager, + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql"); + flushOracleRedoLogs(oracleResourceManager); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseDDLIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); + } + }, + customTransformation, // customTransformation + null, // shardingContextFileResourceName + gcsResourceManager, + datastreamResourceManager, + null, // sessionResourceContent + org.apache.beam.it.gcp.datastream.OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager + .getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleResourceManager.getUsername().toUpperCase(), + java.util.Arrays.asList( + TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + .build()); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseDDLIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager); + } + + @Test + public void migrationTestWithAllDatatypeConversionMapping() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Run SQL inserts"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + executeSqlScript( + oracleResourceManager, + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql"); + executed = true; // Repositioned ABOVE flushOracleLogs + flushOracleRedoLogs(oracleResourceManager); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Success"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(15)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertAllDatatypeColumnsTableBackfillContents(); + } + + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + // Only asserting primary key + simple int column to verify end-to-end sync without complex + // base64 logic + row.put("int_column", "50000"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, int_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("date_column", "2024-02-08T00:00:00Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("decimal_column", "456.12"); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + // text, char, tinytext, mediumtext, longtext are BYTE columns + row.put("text_column", "/u/9n58P"); + row.put("char_column", "v58P"); + row.put("tinytext_column", "7+/+7/2fnw8="); + row.put("mediumtext_column", "/+3v79/v2vrx"); + row.put("longtext_column", "/+/v3+/a+vE="); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f3100000000000000000..."); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column," + + " tinyblob_column, blob_column, mediumblob_column, longblob_column," + + " enum_column, bool_column, other_bool_column, varbinary_column, bit_column," + + " decimal_column, text_column, binary_column, char_column, tinytext_column," + + " mediumtext_column, longtext_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("decimal_column", 123.45); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "15:30:45"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 21); + row.put("text_column", "Some text 2 append"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 201); + row.put("bigint_column", 987655); + row.put("float_column", 24.45); + row.put("double_column", 235.567); + row.put("decimal_column", 23457.78); + row.put("datetime_column", "2022-12-31T23:59:58Z"); + row.put("timestamp_column", "2022-12-31T23:59:58Z"); + row.put("time_column", "00:59:59"); + row.put("year_column", "2023"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 31); + row.put("text_column", "Some text 3 append"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 301); + row.put("bigint_column", 112234); + row.put("float_column", 35.56); + row.put("double_column", 346.678); + row.put("decimal_column", 34568.89); + row.put("datetime_column", "2023-12-31T23:59:59Z"); + row.put("timestamp_column", "2023-12-31T23:59:59Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example4"); + row.put("tinyint_column", 41); + row.put("text_column", "Some text 4 append"); + row.put("date_column", "2021-11-12"); + row.put("int_column", 401); + row.put("bigint_column", 223345); + row.put("float_column", 46.67); + row.put("double_column", 457.789); + row.put("decimal_column", 45679.90); + row.put("datetime_column", "2021-11-11T11:11:10Z"); + row.put("timestamp_column", "2021-11-11T11:11:10Z"); + row.put("time_column", "12:11:11"); + row.put("year_column", "2022"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column," + + " bigint_column, float_column, double_column, decimal_column," + + " datetime_column, timestamp_column, time_column, year_column, blob_column," + + " enum_column, bool_column, binary_column, bit_column FROM" + + " AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 25); + row.put("text_column", "Updated text 2"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 250); + row.put("bigint_column", 56789); + row.put("float_column", 25.45); + row.put("double_column", 345.678); + row.put("decimal_column", 23456.79); + row.put("datetime_column", "2023-01-01T12:00:00Z"); + row.put("timestamp_column", "2023-01-01T12:00:00Z"); + row.put("time_column", "12:00:00"); + row.put("year_column", "2023"); + row.put("blob_column", "EjRWeJCrze8="); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "EjRWeJCrze8SNFZ4kKvN7xI0Vng="); + row.put("bit_column", "fw=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 35); + row.put("text_column", "Updated text 3"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 350); + row.put("bigint_column", 88000); + row.put("float_column", 35.67); + row.put("double_column", 456.789); + row.put("decimal_column", 34567.90); + row.put("datetime_column", "2024-01-02T00:00:00Z"); + row.put("timestamp_column", "2024-01-02T00:00:00Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "q83vEjRWeJA="); + row.put("enum_column", "1"); + row.put("bool_column", false); + row.put("binary_column", "q83vEjRWeJCrze8SNFZ4kKvN7xI="); + row.put("bit_column", "AA=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("SELECT * FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDataTypeSizeConversionBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("float_column", 45.67); + row.put("decimal_column", 456.12); + row.put("char_column", "char_1"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsWithSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOException { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsReducedSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertUsersBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("user_id", 1); + row.put("first_name", "Lorem"); + row.put("last_name", "Epsum"); + row.put("full_name", "Lorem Epsum"); + row.put("age", 20); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 2); + row.put("first_name", "Jane"); + row.put("last_name", "Doe"); + row.put("full_name", "Jane Doe"); + row.put("age", 21); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 9); + row.put("first_name", "James"); + row.put("last_name", "Dove"); + row.put("full_name", "James Dove"); + row.put("age", 22); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select user_id, first_name, last_name, full_name, age from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAuthorsBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "J.R.R. Tolkien"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Jane Austen"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Douglas Adams"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, name from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhiker's Guide to the Galaxy"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java new file mode 100644 index 0000000000..99a36d1d5a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -0,0 +1,547 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Date; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseEventsIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Users"; + private static final String TABLE2 = "Movie"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseEventsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // SYSTEM AUTHORIZATIONS + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Movie", "Users", "Authors", "Articles", "Books"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseEventsIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + + // Supplement logging requirement + + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseEventsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); + } + + @Test + public void migrationTestWithUpdatesAndDeletes() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeUsersInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build(), + writeUsersNextData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertUsersTableContents(); + } + + @Test + public void migrationTestWithInsertsOnly() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeMovieInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + assertMovieTableContents(); + } + + @Test + public void interleavedAndFKAndIndexTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeArticlesInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Articles") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTable(); + assertBooksTable(); + assertArticlesTable(); + } + + private ConditionCheck writeUsersInitialData() { + return new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Write initial Users data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent initial Users data"); + } + }; + } + + private ConditionCheck writeUsersNextData() { + return new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Write next Users data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent next Users data"); + } + }; + } + + private ConditionCheck writeMovieInitialData() { + return new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Write Movie data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1'," + + " TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2'," + + " TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent Movie data"); + } + }; + } + + private ConditionCheck writeArticlesInitialData() { + return new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Write Articles data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (1, 'a1')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (2, 'a2')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (3, 'a3')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (4, 'a4')"); + + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); + + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (1, 'Book005', 3)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (2, 'Book002', 3)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (3, 'Book004', 4)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (4, 'Book005', 2)"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent Articles data"); + } + }; + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add( + Map.of( + "id", + 1, + "name", + "Tester Kumar", + "age", + 30, + "subscribed", + false, + "plan", + "A", + "startDate", + Date.parseDate("2023-01-01"))); + events.add( + Map.of( + "id", + 3, + "name", + "Tester Gupta", + "age", + 50, + "subscribed", + false, + "plan", + "Z", + "startDate", + Date.parseDate("2023-06-07"))); + events.add( + Map.of( + "id", + 4, + "name", + "Tester", + "age", + 38, + "subscribed", + true, + "plan", + "D", + "startDate", + Date.parseDate("2023-09-10"))); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertMovieTableContents() { + List> events = new ArrayList<>(); + events.add( + Map.of( + "id", + 1, + "name", + "movie1", + "startTime", + Timestamp.parseTimestamp("2023-01-01T12:12:12Z"))); + events.add( + Map.of( + "id", + 2, + "name", + "movie2", + "startTime", + Timestamp.parseTimestamp("2023-11-25T17:10:12Z"))); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, name, startTime from Movie where id in (1, 2)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + + ImmutableList numericVals = + spannerResourceManager.runQuery("select actor from Movie order by id"); + Assert.assertEquals(12345.09876, numericVals.get(0).getBigDecimal(0).doubleValue(), 0.00000001); + Assert.assertEquals(931.5123, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.00000001); + } + + private void assertAuthorsTable() { + List> events = new ArrayList<>(); + events.add(Map.of("author_id", 1, "name", "a1")); + events.add(Map.of("author_id", 2, "name", "a2")); + events.add(Map.of("author_id", 3, "name", "a3")); + events.add(Map.of("author_id", 4, "name", "a4")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksTable() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "title", "Book005", "author_id", 3)); + events.add(Map.of("id", 2, "title", "Book002", "author_id", 3)); + events.add(Map.of("id", 3, "title", "Book004", "author_id", 4)); + events.add(Map.of("id", 4, "title", "Book005", "author_id", 2)); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertArticlesTable() { + List> events = new ArrayList<>(); + events.add( + Map.of( + "id", + 1, + "name", + "Article001", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 1)); + events.add( + Map.of( + "id", + 2, + "name", + "Article002", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 1)); + events.add( + Map.of( + "id", + 3, + "name", + "Article004", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 4)); + events.add( + Map.of( + "id", + 4, + "name", + "Article005", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 3)); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java new file mode 100644 index 0000000000..be4b6b0b85 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -0,0 +1,276 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.com.google.common.io.Resources; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseFileOverridesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseFileOverridesIT.class); + + private static final String ORACLE_SCHEMA = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json"; + private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager shadowSpannerResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = + new HashSet<>(); + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseFileOverridesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); + + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + gcsResourceManager.uploadArtifact( + GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseFileOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() throws Exception { + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + overridesMap.put( + "schemaOverridesFilePath", + getGcsPath(GCS_PATH_PREFIX + "/override.json", gcsResourceManager)); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck sendDataCondition = + new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1," + + " 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2," + + " 'Alice', 'Johnson')"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XE", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + sendDataCondition, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java new file mode 100644 index 0000000000..890acf04bf --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -0,0 +1,289 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseMixedIT extends DataStreamToSpannerITBase { + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseMixedIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Authors", "Books", "Genre"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSeparateShadowTableDatabaseMixedIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseMixedIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); + } + + @Test + public void mixedMigrationTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(3) + .setMaxRows(3) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Genre") + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTableContents(); + assertBooksTableContents(); + } + + private ConditionCheck writeInitialData() { + return new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Write JDBC data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + + List> authorRows = new ArrayList<>(); + authorRows.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + + List> bookRows = new ArrayList<>(); + bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); + bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); + bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); + + List> genreRows = new ArrayList<>(); + genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); + + for (Map r : authorRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", + r.get("author_id"), r.get("full_name"))); + } + for (Map r : bookRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", + r.get("id"), r.get("title"), r.get("author_id"))); + } + for (Map r : genreRows) { + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", + r.get("genre_id"), r.get("name"))); + } + + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + executed = true; + } + return new CheckResult(true, "Sent to Oracle."); + } + }; + } + + private void assertAuthorsTableContents() { + List> authorEvents = new ArrayList<>(); + authorEvents.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorEvents.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorEvents.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorEvents.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(authorEvents); + } + + private void assertBooksTableContents() { + List> bookEvents = new ArrayList<>(); + bookEvents.add(Map.of("id", 1, "title", "Pride and Prejudice")); + bookEvents.add(Map.of("id", 2, "title", "Oliver Twist")); + bookEvents.add(Map.of("id", 3, "title", "War and Peace")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(bookEvents); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java new file mode 100644 index 0000000000..ff99160cb1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabasePKFocusedIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabasePKFocusedIT.class); + + private static final String ORACLE_SCHEMA = + "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String GCS_PATH_PREFIX = "OracleSeparateShadowTableDatabasePKFocusedIT"; + + // Oracle defaults to uppercase unquoted identifiers + private static final String ORACLE_TABLE_MY_TABLE = "MY_TABLE"; + private static final String ORACLE_TABLE_ALLTYPES = "ALLTYPES"; + private static final String SPANNER_TABLE_MY_TABLE = "MY_TABLE"; + private static final String SPANNER_TABLE_ALLTYPES = "ALLTYPES"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager shadowSpannerResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabasePKFocusedIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("cloudOracleHost") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); + + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + oracleResourceManager.getUsername().toUpperCase(), + List.of(ORACLE_TABLE_MY_TABLE, ORACLE_TABLE_ALLTYPES))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put( + "shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put( + "shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabasePKFocusedIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestSimpleTable() throws Exception { + ConditionCheck insertRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Insert 3 records into MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (1, 10)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (2, 20)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (3, 30)"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update and Delete in MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET VAL = 10 WHERE ID = 2"); + oracleResourceManager.runSQLUpdate("DELETE FROM MY_TABLE WHERE ID = 1"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "PK Update in MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET ID = 10 WHERE ID = 3"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data PK updated"); + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_MY_TABLE) + .setMinRows(3) + .setMaxRows(3) + .build(), + updateDeleteRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_MY_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build(), + pkUpdateRecords, + new ConditionCheck() { + @Override + public String getDescription() { + return "Check if MY_TABLE PK updated to 10"; + } + + @Override + protected CheckResult check() { + try { + com.google.common.collect.ImmutableList + rows = + spannerResourceManager.runQuery( + "SELECT ID FROM MY_TABLE WHERE ID = 10"); + if (!rows.isEmpty()) { + return new CheckResult(true, "PK updated to 10"); + } + } catch (Exception e) { + } + return new CheckResult(false, "Not yet"); + } + })) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (Exception e) { + } + + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("ID", 2L); + row1.put("VAL", 10L); + Map row2 = new HashMap<>(); + row2.put("ID", 10L); + row2.put("VAL", 30L); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select ID, VAL from MY_TABLE")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + @Test + public void migrationTestAllDataType() throws Exception { + ConditionCheck insertRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Insert records into ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + // 1 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 1, 3.14, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 10)"); + // 2 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 20)"); + // 3 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 30)"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update and Delete in ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("DELETE FROM ALLTYPES WHERE INT64_FIELD = 1"); + oracleResourceManager.runSQLUpdate( + "UPDATE ALLTYPES SET VAL = 10 WHERE INT64_FIELD = 2"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update PK in ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate( + "UPDATE ALLTYPES SET INT64_FIELD = 10 WHERE INT64_FIELD = 3"); + flushOracleRedoLogs(oracleResourceManager); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data PK updated"); + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_ALLTYPES) + .setMinRows(3) + .setMaxRows(3) + .build(), + updateDeleteRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_ALLTYPES) + .setMinRows(2) + .setMaxRows(2) + .build(), + pkUpdateRecords, + new ConditionCheck() { + @Override + public String getDescription() { + return "Check if ALLTYPES PK updated to 10"; + } + + @Override + protected CheckResult check() { + try { + com.google.common.collect.ImmutableList + rows = + spannerResourceManager.runQuery( + "SELECT INT64_FIELD FROM ALLTYPES WHERE INT64_FIELD = 10"); + if (!rows.isEmpty()) { + return new CheckResult(true, "PK updated to 10"); + } + } catch (Exception e) { + } + return new CheckResult(false, "Not yet"); + } + })) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (Exception e) { + } + + List> events = new ArrayList<>(); + Map r1 = new HashMap<>(); + r1.put("INT64_FIELD", 2L); + r1.put("VAL", 10L); + Map r2 = new HashMap<>(); + r2.put("INT64_FIELD", 10L); + r2.put("VAL", 30L); + events.add(r1); + events.add(r2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select INT64_FIELD, VAL from ALLTYPES")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java new file mode 100644 index 0000000000..ec5152212b --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -0,0 +1,425 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * An integration test using separate shadow table database for {@link DataStreamToSpanner} Flex + * template which tests use-cases where a session file is required. + */ +@Category({TemplateIntegrationTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseSessionIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Category"; + private static final String TABLE2 = "Books"; + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = + new HashSet<>(); + + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudOracleResourceManager; + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static String oracleUser; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json"; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseSessionIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + cloudOracleResourceManager = setUpOracleResourceManager(); + try { + cloudOracleResourceManager.runSQLUpdate( + String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE1)); + } catch (Exception e) { + } + try { + cloudOracleResourceManager.runSQLUpdate( + String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE2)); + } catch (Exception e) { + } + executeSqlScript(cloudOracleResourceManager, ORACLE_DDL_RESOURCE); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource jdbcSource = + OracleSource.builder( + cloudOracleResourceManager.getHost(), + cloudOracleResourceManager.getUsername(), + cloudOracleResourceManager.getPassword(), + cloudOracleResourceManager.getPort(), + cloudOracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudOracleResourceManager.getUsername().toUpperCase(), + List.of(TABLE1, TABLE2))) + .build(); + + Map jobParams = new HashMap<>(); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSeparateShadowTableDatabaseSessionIT", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + public static void setUpOracleUser( + CloudOracleResourceManager sysUser, String user, String password) { + sysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + sysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + + // Add c##datastream grants based on requirements + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseSessionIT instance : testInstances) { + try { + instance.tearDownBase(); + } catch (Exception e) { + // Ignore UnsupportedOperationException from Thread.stop() on Java 21+ + } + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + cloudOracleSysUser, + cloudOracleResourceManager, + datastreamResourceManager, + gcsResourceManager); + } + + @Test + public void migrationTestWithRenameAndDrops() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialCategoryData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + assertCategoryTableBackfillContents(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeCdcCategoryData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertCategoryTableCdcContents(); + } + + @Test + public void migrationTestWithSyntheticPKAndExtraColumn() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeBooksData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + assertBooksBackfillContents(); + } + + private ConditionCheck writeInitialCategoryData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { + return "Write initial Category rows"; + } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent category initial"); + } + try { + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (1, 'xyz')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (2, 'abc')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category initial"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private ConditionCheck writeCdcCategoryData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { + return "Write cdc Category rows"; + } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent category cdc"); + } + try { + cloudOracleResourceManager.runSQLUpdate( + String.format("DELETE FROM \"%s\" WHERE \"category_id\" = 1", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "UPDATE \"%s\" SET \"full_name\" = 'abc1' WHERE \"category_id\" = 2", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (3, 'def')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (4, 'ghi')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category cdc"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private ConditionCheck writeBooksData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { + return "Write Books rows"; + } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent books"); + } + try { + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of" + + " the Rings', 1)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and" + + " Prejudice', 2)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The" + + " Hitchhikers Guide to the Galaxy', 3)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent books"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private void assertCategoryTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 1); + row1.put("full_name", "xyz"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 2); + row2.put("full_name", "abc"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableCdcContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 2); + row1.put("full_name", "abc1"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 3); + row2.put("full_name", "def"); + + Map row3 = new HashMap<>(); + row3.put("category_id", 4); + row3.put("full_name", "ghi"); + + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + row.put("author_id", 2); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhikers Guide to the Galaxy"); + row.put("author_id", 3); + events.add(row); + + ImmutableList synthIds = spannerResourceManager.runQuery("select synth_id from Books"); + + Assert.assertEquals(3, synthIds.size()); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, title, author_id from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java new file mode 100644 index 0000000000..af34bc2db1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -0,0 +1,356 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger( + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class); + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet< + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT> + testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized ( + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = + "oracle-shard-cdc_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + uniqueCdcPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put( + "inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); + if (!fileLocal.exists()) { + fileLocal = + new java.io.File( + "v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); + } + String sessionFileContent = + new String( + java.nio.file.Files.readAllBytes(fileLocal.toPath()), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT instance : + testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20, "migration_shard_id", "L1")); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103, "migration_shard_id", "L1")); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113, "migration_shard_id", "L1")); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104, "migration_shard_id", "L1")); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105, "migration_shard_id", "L1")); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106, "migration_shard_id", "L1")); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107, "migration_shard_id", "L1")); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108, "migration_shard_id", "L1")); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109, "migration_shard_id", "L1")); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110, "migration_shard_id", "L1")); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111, "migration_shard_id", "L1")); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112, "migration_shard_id", "L1")); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig(String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java new file mode 100644 index 0000000000..6db397af65 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -0,0 +1,355 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger( + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class); + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet< + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT> + testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized ( + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = + "oracle-shard-cdc_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + uniqueCdcPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put( + "inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); + if (!fileLocal.exists()) { + fileLocal = + new java.io.File( + "v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); + } + String sessionFileContent = + new String( + java.nio.file.Files.readAllBytes(fileLocal.toPath()), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT instance : + testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + org.apache.beam.it.conditions.ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20)); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103)); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113)); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104)); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105)); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106)); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107)); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108)); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109)); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110)); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111)); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112)); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig(String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java new file mode 100644 index 0000000000..a780f59b45 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -0,0 +1,345 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class); + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet + testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setDatabaseName("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudOracleUsername", "system")) + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + try { + } catch (Exception e) { + } + + try { + } catch (Exception e) { + } + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + "oracle-shard-cdc/cdc/", + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test-stream-" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, "system", "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + String sessionFileContent = + com.google.common.io.Resources.toString( + com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", "system").replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT_shard1", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + org.apache.beam.it.conditions.ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20, "migration_shard_id", "L1")); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103, "migration_shard_id", "L1")); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113, "migration_shard_id", "L1")); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104, "migration_shard_id", "L1")); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105, "migration_shard_id", "L1")); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106, "migration_shard_id", "L1")); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107, "migration_shard_id", "L1")); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108, "migration_shard_id", "L1")); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109, "migration_shard_id", "L1")); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110, "migration_shard_id", "L1")); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111, "migration_shard_id", "L1")); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112, "migration_shard_id", "L1")); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig(String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java new file mode 100644 index 0000000000..b308d14d51 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -0,0 +1,245 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseStringOverridesIT extends DataStreamToSpannerITBase { + + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = + new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseStringOverridesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + builder.setHost(System.getProperty("cloudOracleHost")); + builder.setPort(1521); + builder.setDatabaseName("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudOracleUsername", "system")) + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XE") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + try { + } catch (Exception e) { + } + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("datastreamSourceType", "oracle"); + overridesMap.put("tableOverrides", "[{person1, human1}]"); + overridesMap.put("columnOverrides", "[{person1.first_name1, person1.name1}]"); + overridesMap.put( + "shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put( + "shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseStringOverridesIT", + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseStringOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new org.apache.beam.it.conditions.ConditionCheck() { + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert records into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('John', 'Doe')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('Alice', 'Johnson')"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java new file mode 100644 index 0000000000..7abb5030e7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * Custom class for Oracle implementations that require Service Name format (//host:port/Service) + * instead of the legacy SID format. Specifically built for Spanner integration tests hitting + * Pluggable Databases (PDBs) through GCP Datastream. + */ +public class SpannerOracleResourceManager extends CloudOracleResourceManager { + + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(SpannerOracleResourceManager.class); + + public SpannerOracleResourceManager(CloudOracleResourceManager.Builder builder) { + super(builder); + } + + @Override + public synchronized @NonNull String getUri() { + return String.format( + "jdbc:%s:thin:@//%s:%d/%s", + getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); + } + + @Override + public void cleanupAll() { + super.cleanupAll(); + + String userToDrop = this.getUsername(); + if (userToDrop != null + && !userToDrop.equalsIgnoreCase("system") + && !userToDrop.contains("sysdba")) { + LOG.info("Attempting to dynamically drop isolated Oracle user: {}", userToDrop); + try { + CloudOracleResourceManager sysdba = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(this.getDatabaseName() + "_cleanup") + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost", this.getHost())) + .setPort(1521) + .build(); + + // Oracle Spanner implementation requires specific URI + sysdba = + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(this.getDatabaseName() + "_cleanup") + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost", this.getHost())) + .setPort(1521)); + + sysdba.runSQLUpdate("DROP USER " + userToDrop + " CASCADE"); + LOG.info("Successfully dropped Oracle schema: {}", userToDrop); + } catch (Exception e) { + LOG.warn("Failed to drop schema: " + userToDrop, e); + } + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..6b6168912a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,131 @@ +CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsWithSizes ( + varchar_column STRING(30) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(50), + bool_column BOOL, + binary_column BYTES(30), + varbinary_column BYTES(30), + bit_column BYTES(10), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsReducedSizes ( + varchar_column STRING(10) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(20), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS Users ( + user_id INT64 NOT NULL, + first_name STRING(50), + last_name STRING(50), + age INT64, + full_name STRING(100) AS (ARRAY_TO_STRING([first_name, last_name], " ")) STORED, +) PRIMARY KEY (user_id); + +CREATE TABLE IF NOT EXISTS Authors ( + id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (id); + +CREATE TABLE IF NOT EXISTS AllDatatypeTransformation ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + blob_column BYTES(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY (varchar_column); + +CREATE SEQUENCE sequence_id OPTIONS (sequence_kind='bit_reversed_positive', skip_range_min = 0, skip_range_max = 3); + +CREATE TABLE IF NOT EXISTS Singers ( + singer_id INT64 NOT NULL DEFAULT (GET_NEXT_SEQUENCE_VALUE(SEQUENCE sequence_id)) , + first_name STRING(1024), + last_name STRING(1024), +) PRIMARY KEY (singer_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE), + title STRING(200), +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql new file mode 100644 index 0000000000..fc1f4a088c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql @@ -0,0 +1,4 @@ +INSERT INTO "AllDatatypeColumns" +("varchar_column", "tinyint_column", "text_column", "date_column", "smallint_column", "mediumint_column", "int_column", "bigint_column", "float_column", "double_column", "decimal_column", "datetime_column", "timestamp_column", "time_column", "year_column", "char_column", "tinyblob_column", "tinytext_column", "blob_column", "mediumblob_column", "mediumtext_column", "longblob_column", "longtext_column", "enum_column", "bool_column", "other_bool_column", "binary_column", "varbinary_column", "bit_column") +VALUES +('value1', 10, 'text_content', TO_DATE('2024-02-08', 'YYYY-MM-DD'), 50, 1000, 50000, 987654321, 45.67, 123.789, 456.12, TO_TIMESTAMP('2024-02-08 08:15:30', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP_TZ('2024-02-08 08:15:30 UTC', 'YYYY-MM-DD HH24:MI:SS TZR'), '08:15:30', 2022, 'char_1 ', HEXTORAW('74696e79626c6f625f646174615f31'), 'tinytext', HEXTORAW('626c6f625f646174615f31'), HEXTORAW('6d656469756d626c6f625f646174615f31'), 'mediumtext', HEXTORAW('6c6f6e67626c6f625f646174615f31'), 'longtext_content', '2', 0, 1, HEXTORAW('62696e6172795f31000000000000000000000000'), HEXTORAW('76617262696e6172795f646174615f31'), HEXTORAW('0102')); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql new file mode 100644 index 0000000000..830bc4ba29 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql @@ -0,0 +1,156 @@ + +CREATE TABLE "AllDatatypeColumns" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "other_bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "AllDatatypeColumns2" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsWithSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsReducedSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Users" ( + "user_id" NUMBER NOT NULL, + "first_name" VARCHAR2(50), + "last_name" VARCHAR2(50), + "age" NUMBER, + PRIMARY KEY ("user_id") +); + +CREATE TABLE "Authors" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +CREATE TABLE "AllDatatypeTransformation" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "blob_column" BLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Singers" ( + "singer_id" NUMBER NOT NULL , + "first_name" VARCHAR2(1024), + "last_name" VARCHAR2(1024), + PRIMARY KEY ("singer_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL , + "title" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +ALTER TABLE "AllDatatypeColumns" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeColumns2" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsWithSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsReducedSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeTransformation" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Singers" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..3ac9187f46 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,22 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, + LegacyRegion STRING(50), +) PRIMARY KEY (CustomerId); + +ALTER TABLE Customers ADD CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + LegacyOrderSystem STRING(50) NOT NULL, +) PRIMARY KEY (CustomerId, LegacyOrderSystem, OrderId); + +ALTER TABLE Orders ADD CONSTRAINT FK_CustomerOrder FOREIGN KEY (CustomerId) REFERENCES Customers(CustomerId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar2_col STRING(MAX), +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..9122439183 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,21 @@ +CREATE TABLE "Customers" ( + "CustomerId" INTEGER NOT NULL, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER NOT NULL, + "LoyaltyTier" VARCHAR2(50), + PRIMARY KEY ("CustomerId") +); + +CREATE TABLE "Orders" ( + "CustomerId" INTEGER NOT NULL, + "OrderId" INTEGER NOT NULL, + "OrderValue" NUMBER, + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" INTEGER, + "varchar2_col" VARCHAR2(1000), + PRIMARY KEY ("id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..8000ac7d0d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..0a6151cc54 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE `Customers` ( + `CustomerId` INT64 NOT NULL , + `CustomerName` STRING(255), + `CreditLimit` NUMERIC NOT NULL , + `LegacyRegion` STRING(50), + `migration_shard_id` STRING(50), + CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000) +) PRIMARY KEY (`migration_shard_id`, `CustomerId`); + +CREATE TABLE `Orders` ( + `CustomerId` INT64 NOT NULL , + `OrderId` INT64 NOT NULL , + `OrderValue` NUMERIC, + `migration_shard_id` STRING(50), + `LegacyOrderSystem` STRING(50) NOT NULL +) PRIMARY KEY (`migration_shard_id`, `CustomerId`, `LegacyOrderSystem`, `OrderId`), +INTERLEAVE IN PARENT `Customers` ON DELETE NO ACTION; + +CREATE TABLE `AllDataTypes` ( + `id` INT64 NOT NULL , + `varchar_col` STRING(1000), + `tinyint_col` INT64, + `tinyint_unsigned_col` INT64, + `text_col` STRING(MAX), + `date_col` TIMESTAMP, + `smallint_col` INT64, + `smallint_unsigned_col` INT64, + `mediumint_col` INT64, + `mediumint_unsigned_col` INT64, + `bigint_col` INT64, + `bigint_unsigned_col` INT64, + `float_col` FLOAT32, + `double_col` FLOAT64, + `decimal_col` NUMERIC, + `datetime_col` TIMESTAMP, + `time_col` STRING(50), + `year_col` STRING(4), + `char_col` STRING(255), + `tinyblob_col` BYTES(255), + `tinytext_col` STRING(MAX), + `blob_col` BYTES(65535), + `mediumblob_col` BYTES(10485760), + `mediumtext_col` STRING(MAX), + `test_json_col` STRING(MAX), + `longblob_col` BYTES(10485760), + `longtext_col` STRING(MAX), + `enum_col` STRING(MAX), + `bool_col` INT64, + `binary_col` BYTES(255), + `varbinary_col` BYTES(1000), + `bit_col` BYTES(MAX), + `bit8_col` BYTES(MAX), + `bit1_col` INT64, + `boolean_col` INT64, + `int_col` INT64, + `integer_unsigned_col` INT64, + `timestamp_col` TIMESTAMP, + `set_col` STRING(MAX), + `migration_shard_id` STRING(50) +) PRIMARY KEY (`migration_shard_id`, `id`); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..fb08ca5296 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql @@ -0,0 +1,57 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER(10, 2) NOT NULL, + "LoyaltyTier" VARCHAR2(50) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER(10, 2), + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER DEFAULT NULL, + "tinyint_unsigned_col" NUMBER DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER DEFAULT NULL, + "smallint_unsigned_col" NUMBER DEFAULT NULL, + "mediumint_col" NUMBER DEFAULT NULL, + "mediumint_unsigned_col" NUMBER DEFAULT NULL, + "bigint_col" NUMBER DEFAULT NULL, + "bigint_unsigned_col" NUMBER DEFAULT NULL, + "float_col" BINARY_FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER(38,9) DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" VARCHAR2(4) DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" RAW(255) DEFAULT NULL, + "tinytext_col" CLOB DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" VARCHAR2(4000) DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(255) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" RAW(1) DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER DEFAULT NULL, + "integer_unsigned_col" NUMBER DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(255) DEFAULT NULL +); + diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json new file mode 100644 index 0000000000..8000ac7d0d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json new file mode 100644 index 0000000000..e366c8ed21 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json @@ -0,0 +1,4272 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "XE", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "AllDataTypes", + "ColIds": [ + "c4", + "c5", + "c6", + "c7", + "c8", + "c9", + "c10", + "c11", + "c12", + "c13", + "c14", + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c51" + ], + "ShardIdColumn": "c51", + "ColDefs": { + "c10": { + "Name": "smallint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_col smallint(5)", + "Id": "c10", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c11": { + "Name": "smallint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_unsigned_col smallint(5)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c12": { + "Name": "mediumint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_col mediumint(7)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c13": { + "Name": "mediumint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_unsigned_col mediumint(7)", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c14": { + "Name": "bigint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_col bigint(19)", + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c15": { + "Name": "bigint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_unsigned_col bigint unsigned(20)", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c16": { + "Name": "float_col", + "T": { + "Name": "FLOAT32", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_col float(12)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c17": { + "Name": "double_col", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_col double(22)", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c18": { + "Name": "decimal_col", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_col decimal(65,30)", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c19": { + "Name": "datetime_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_col datetime", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c20": { + "Name": "time_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_col time", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c21": { + "Name": "year_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_col year", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c22": { + "Name": "char_col", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: char_col char(255)", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c23": { + "Name": "tinyblob_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyblob_col tinyblob(255)", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c24": { + "Name": "tinytext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinytext_col tinytext(255)", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c25": { + "Name": "blob_col", + "T": { + "Name": "BYTES", + "Len": 65535, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_col blob(65535)", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c26": { + "Name": "mediumblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumblob_col mediumblob(16777215)", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c27": { + "Name": "mediumtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumtext_col mediumtext(16777215)", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c28": { + "Name": "test_json_col", + "T": { + "Name": "JSON", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: test_json_col json", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c29": { + "Name": "longblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longblob_col longblob(4294967295)", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c30": { + "Name": "longtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longtext_col longtext(4294967295)", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c31": { + "Name": "enum_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_col enum(1)", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c32": { + "Name": "bool_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_col tinyint(1)", + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c33": { + "Name": "binary_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_col binary(255)", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c34": { + "Name": "varbinary_col", + "T": { + "Name": "BYTES", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varbinary_col varbinary(1000)", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c35": { + "Name": "bit_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_col bit(64)", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c36": { + "Name": "bit8_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit8_col bit(8)", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c37": { + "Name": "bit1_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit1_col bit(1)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c38": { + "Name": "boolean_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: boolean_col tinyint(1)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c39": { + "Name": "int_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_col int(10)", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c4": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c40": { + "Name": "integer_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: integer_unsigned_col int(10)", + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c41": { + "Name": "timestamp_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_col timestamp", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c42": { + "Name": "set_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: set_col set[]", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c5": { + "Name": "varchar_col", + "T": { + "Name": "STRING", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varchar_col varchar(1000)", + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c51": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c6": { + "Name": "tinyint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_col tinyint(3)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c7": { + "Name": "tinyint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_unsigned_col tinyint", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c8": { + "Name": "text_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_col text(65535)", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c9": { + "Name": "date_col", + "T": { + "Name": "DATE", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_col date", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c4", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c51", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table AllDataTypes", + "Id": "t1" + }, + "t2": { + "Name": "Customers", + "ColIds": [ + "c47", + "c48", + "c49", + "c50", + "c52" + ], + "ShardIdColumn": "c52", + "ColDefs": { + "c47": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c48": { + "Name": "CustomerName", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: CustomerName varchar(255)", + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c49": { + "Name": "CreditLimit", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CreditLimit decimal(10,2)", + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c50": { + "Name": "LegacyRegion", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: LoyaltyTier varchar(50)", + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c52": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c52", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": [ + { + "Id": "1", + "Name": "CHK_CreditLimit", + "Expr": "(CreditLimit > 1000)", + "ExprId": "e976" + } + ], + "Comment": "Spanner schema for source table Customers", + "Id": "t2" + }, + "t3": { + "Name": "Orders", + "ColIds": [ + "c43", + "c44", + "c45", + "c53", + "c55" + ], + "ShardIdColumn": "c53", + "ColDefs": { + "c43": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c44": { + "Name": "OrderId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: OrderId int(10)", + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c45": { + "Name": "OrderValue", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: OrderValue decimal(10,2)", + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c53": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c55": { + "Name": "LegacyOrderSystem", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": true, + "Comment": "", + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c44", + "Desc": false, + "Order": 4 + }, + { + "ColId": "c53", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c55", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "t2", + "OnDelete": "NO ACTION", + "InterleaveType": "IN PARENT" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table Orders", + "Id": "t3" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "AllDataTypes", + "Schema": "XE", + "ColIds": [ + "c4", + "c5", + "c6", + "c7", + "c8", + "c9", + "c10", + "c11", + "c12", + "c13", + "c14", + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42" + ], + "ColDefs": { + "c10": { + "Name": "smallint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c10", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c11": { + "Name": "smallint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c12": { + "Name": "mediumint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c13": { + "Name": "mediumint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c14": { + "Name": "bigint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c15": { + "Name": "bigint_unsigned_col", + "Type": { + "Name": "bigint unsigned", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c16": { + "Name": "float_col", + "Type": { + "Name": "BINARY_FLOAT", + "Mods": [ + 12 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c17": { + "Name": "double_col", + "Type": { + "Name": "BINARY_DOUBLE", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c18": { + "Name": "decimal_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c19": { + "Name": "datetime_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c20": { + "Name": "time_col", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c21": { + "Name": "year_col", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c22": { + "Name": "char_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c23": { + "Name": "tinyblob_col", + "Type": { + "Name": "RAW", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c24": { + "Name": "tinytext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c25": { + "Name": "blob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c26": { + "Name": "mediumblob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c27": { + "Name": "mediumtext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c28": { + "Name": "test_json_col", + "Type": { + "Name": "VARCHAR2", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c29": { + "Name": "longblob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c30": { + "Name": "longtext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c31": { + "Name": "enum_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c32": { + "Name": "bool_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c33": { + "Name": "binary_col", + "Type": { + "Name": "RAW", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c34": { + "Name": "varbinary_col", + "Type": { + "Name": "RAW", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c35": { + "Name": "bit_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c36": { + "Name": "bit8_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c37": { + "Name": "bit1_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c38": { + "Name": "boolean_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c39": { + "Name": "int_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c4": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c40": { + "Name": "integer_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c41": { + "Name": "timestamp_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c42": { + "Name": "set_col", + "Type": { + "Name": "VARCHAR2", + "Mods": null, + "ArrayBounds": [ + -1 + ] + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c5": { + "Name": "varchar_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c6": { + "Name": "tinyint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c7": { + "Name": "tinyint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c8": { + "Name": "text_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c9": { + "Name": "date_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c4", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t1" + }, + "t2": { + "Name": "Customers", + "Schema": "XE", + "ColIds": [ + "c47", + "c48", + "c49", + "c50" + ], + "ColDefs": { + "c47": { + "Name": "CustomerId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c48": { + "Name": "CustomerName", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c49": { + "Name": "CreditLimit", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c50": { + "Name": "LoyaltyTier", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t2" + }, + "t3": { + "Name": "Orders", + "Schema": "XE", + "ColIds": [ + "c43", + "c44", + "c45", + "c46" + ], + "ColDefs": { + "c43": { + "Name": "CustomerId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c44": { + "Name": "OrderId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c45": { + "Name": "OrderValue", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c46": { + "Name": "OrderSource", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c44", + "Desc": false, + "Order": 2 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t3" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c10": [ + 14 + ], + "c11": [ + 14 + ], + "c12": [ + 14 + ], + "c13": [ + 14 + ], + "c15": [ + 52 + ], + "c19": [ + 13 + ], + "c20": [ + 15 + ], + "c21": [ + 15 + ], + "c39": [ + 14 + ], + "c4": [ + 14 + ], + "c40": [ + 14 + ], + "c42": [ + 31 + ], + "c51": [ + 29 + ], + "c6": [ + 14 + ], + "c7": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t2": { + "ColumnLevelIssues": { + "c47": [ + 14 + ], + "c52": [ + 5 + ] + }, + "TableLevelIssues": null + }, + "t3": { + "ColumnLevelIssues": { + "c43": [ + 14 + ], + "c44": [ + 14 + ], + "c53": [ + 5 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "ToSpanner": { + "AllDataTypes": { + "Name": "AllDataTypes", + "Cols": { + "bigint_col": "bigint_col", + "bigint_unsigned_col": "bigint_unsigned_col", + "binary_col": "binary_col", + "bit1_col": "bit1_col", + "bit8_col": "bit8_col", + "bit_col": "bit_col", + "blob_col": "blob_col", + "bool_col": "bool_col", + "boolean_col": "boolean_col", + "char_col": "char_col", + "date_col": "date_col", + "datetime_col": "datetime_col", + "decimal_col": "decimal_col", + "double_col": "double_col", + "enum_col": "enum_col", + "float_col": "float_col", + "id": "id", + "int_col": "int_col", + "integer_unsigned_col": "integer_unsigned_col", + "longblob_col": "longblob_col", + "longtext_col": "longtext_col", + "mediumblob_col": "mediumblob_col", + "mediumint_col": "mediumint_col", + "mediumint_unsigned_col": "mediumint_unsigned_col", + "mediumtext_col": "mediumtext_col", + "set_col": "set_col", + "smallint_col": "smallint_col", + "smallint_unsigned_col": "smallint_unsigned_col", + "test_json_col": "test_json_col", + "text_col": "text_col", + "time_col": "time_col", + "timestamp_col": "timestamp_col", + "tinyblob_col": "tinyblob_col", + "tinyint_col": "tinyint_col", + "tinyint_unsigned_col": "tinyint_unsigned_col", + "tinytext_col": "tinytext_col", + "varbinary_col": "varbinary_col", + "varchar_col": "varchar_col", + "year_col": "year_col" + } + }, + "Customers": { + "Name": "Customers", + "Cols": { + "CreditLimit": "CreditLimit", + "CustomerId": "CustomerId", + "CustomerName": "CustomerName", + "LoyaltyTier": "LegacyRegion" + } + }, + "Orders": { + "Name": "Orders", + "Cols": { + "CustomerId": "CustomerId", + "OrderId": "OrderId", + "OrderSource": "OrderSource", + "OrderValue": "OrderValue" + } + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r54", + "Name": "r54", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": null + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "SpProjectId": "span-cloud-ck-testing-external", + "SpInstanceId": "ea-functional-tests", + "Source": "mysql", + "DatabaseOptions": { + "DbName": "", + "DefaultTimezone": "" + }, + "DefaultIdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3e8b59c4ff --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..c87cdd07ab --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500) DEFAULT NULL, + "last_name1" VARCHAR2(500) DEFAULT NULL, + "first_name2" VARCHAR2(500) DEFAULT NULL, + "last_name2" VARCHAR2(500) DEFAULT NULL, + "first_name3" VARCHAR2(500) DEFAULT NULL, + "last_name3" VARCHAR2(500) DEFAULT NULL, + "ID" NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..55f2e40bc7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,59 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, + LegacyRegion STRING(50), +) PRIMARY KEY (CustomerId); + +ALTER TABLE Customers ADD CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + LegacyOrderSystem STRING(50) NOT NULL, +) PRIMARY KEY (CustomerId, LegacyOrderSystem, OrderId); + +ALTER TABLE Orders ADD CONSTRAINT FK_CustomerOrder FOREIGN KEY (CustomerId) REFERENCES Customers(CustomerId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar_col STRING(1000), + tinyint_col INT64, + tinyint_unsigned_col INT64, + text_col STRING(MAX), + date_col DATE, + smallint_col INT64, + smallint_unsigned_col INT64, + mediumint_col INT64, + mediumint_unsigned_col INT64, + bigint_col INT64, + bigint_unsigned_col NUMERIC, + float_col FLOAT64, + double_col FLOAT64, + decimal_col NUMERIC, + datetime_col TIMESTAMP, + time_col STRING(MAX), + year_col STRING(MAX), + char_col STRING(255), + tinyblob_col BYTES(MAX), + tinytext_col STRING(MAX), + blob_col BYTES(MAX), + mediumblob_col BYTES(MAX), + mediumtext_col STRING(MAX), + test_json_col JSON, + longblob_col BYTES(MAX), + longtext_col STRING(MAX), + enum_col STRING(MAX), + bool_col BOOL, + binary_col BYTES(MAX), + varbinary_col BYTES(MAX), + bit_col BYTES(MAX), + bit8_col INT64, + bit1_col BOOL, + boolean_col BOOL, + int_col INT64, + integer_unsigned_col INT64, + timestamp_col TIMESTAMP, + set_col STRING(MAX), +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..fc5bcef3d1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,56 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER NOT NULL, + "LoyaltyTier" VARCHAR2(50) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER, + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER DEFAULT NULL, + "tinyint_unsigned_col" NUMBER DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER DEFAULT NULL, + "smallint_unsigned_col" NUMBER DEFAULT NULL, + "mediumint_col" NUMBER DEFAULT NULL, + "mediumint_unsigned_col" NUMBER DEFAULT NULL, + "bigint_col" NUMBER DEFAULT NULL, + "bigint_unsigned_col" NUMBER DEFAULT NULL, + "float_col" BINARY_FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" NUMBER DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" RAW(255) DEFAULT NULL, + "tinytext_col" VARCHAR2(255) DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" CLOB DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(50) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" NUMBER(3) DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER DEFAULT NULL, + "integer_unsigned_col" NUMBER DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(255) DEFAULT NULL +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..2c964b41cd --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..0b324f3480 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,140 @@ +CREATE TABLE `true` ( + id INT64 NOT NULL, + `ALL` STRING(MAX), + `ANALYSE` STRING(MAX), + `ANALYZE` STRING(MAX), + `AND` STRING(MAX), + `ANY` STRING(MAX), + `ARRAY` STRING(MAX), + `AS` STRING(MAX), + `ASC` STRING(MAX), + `ASSERT_ROWS_MODIFIED` STRING(MAX), + `ASYMMETRIC` STRING(MAX), + `AT` STRING(MAX), + `BETWEEN` STRING(MAX), + `BOTH` STRING(MAX), + `BY` STRING(MAX), + `CASE` STRING(MAX), + `CAST` STRING(MAX), + `CHECK` STRING(MAX), + `COLLATE` STRING(MAX), + `COLUMN` STRING(MAX), + `CONSTRAINT` STRING(MAX), + `CONTAINS` STRING(MAX), + `CREATE` STRING(MAX), + `CROSS` STRING(MAX), + `CUBE` STRING(MAX), + `CURRENT` STRING(MAX), + `CURRENT_CATALOG` STRING(MAX), + `CURRENT_DATE` STRING(MAX), + `CURRENT_ROLE` STRING(MAX), + `CURRENT_TIME` STRING(MAX), + `CURRENT_TIMESTAMP` STRING(MAX), + `CURRENT_USER` STRING(MAX), + `DEFAULT` STRING(MAX), + `DEFERRABLE` STRING(MAX), + `DEFINE` STRING(MAX), + `DESC` STRING(MAX), + `DISTINCT` STRING(MAX), + `DO` STRING(MAX), + `ELSE` STRING(MAX), + `END` STRING(MAX), + `END_P` STRING(MAX), + `ENUM` STRING(MAX), + `ESCAPE` STRING(MAX), + `EXCEPT` STRING(MAX), + `EXCLUDE` STRING(MAX), + `EXISTS` STRING(MAX), + `EXTRACT` STRING(MAX), + `FALSE` STRING(MAX), + `FALSE_P` STRING(MAX), + `FETCH` STRING(MAX), + `FOLLOWING` STRING(MAX), + `FOR` STRING(MAX), + `FOREIGN` STRING(MAX), + `FROM` STRING(MAX), + `FULL` STRING(MAX), + `GRANT` STRING(MAX), + `GRAPH_TABLE` STRING(MAX), + `GROUP` STRING(MAX), + `GROUPING` STRING(MAX), + `GROUPS` STRING(MAX), + `GROUP_P` STRING(MAX), + `HASH` STRING(MAX), + `HAVING` STRING(MAX), + `IF` STRING(MAX), + `IGNORE` STRING(MAX), + `IN` STRING(MAX), + `INITIALLY` STRING(MAX), + `INNER` STRING(MAX), + `INTERSECT` STRING(MAX), + `INTERVAL` STRING(MAX), + `INTO` STRING(MAX), + `IN_P` STRING(MAX), + `IS` STRING(MAX), + `JOIN` STRING(MAX), + `LATERAL` STRING(MAX), + `LATERAL_P` STRING(MAX), + `LEADING` STRING(MAX), + `LEFT` STRING(MAX), + `LIKE` STRING(MAX), + `LIMIT` STRING(MAX), + `LOCALTIME` STRING(MAX), + `LOCALTIMESTAMP` STRING(MAX), + `LOOKUP` STRING(MAX), + `MERGE` STRING(MAX), + `NATURAL` STRING(MAX), + `NEW` STRING(MAX), + `NO` STRING(MAX), + `NOT` STRING(MAX), + `NULL` STRING(MAX), + `NULLS` STRING(MAX), + `NULL_P` STRING(MAX), + `OF` STRING(MAX), + `OFFSET` STRING(MAX), + `ON` STRING(MAX), + `ONLY` STRING(MAX), + `OR` STRING(MAX), + `ORDER` STRING(MAX), + `OUTER` STRING(MAX), + `OVER` STRING(MAX), + `PARTITION` STRING(MAX), + `PLACING` STRING(MAX), + `PRECEDING` STRING(MAX), + `PRIMARY` STRING(MAX), + `PROTO` STRING(MAX), + `RANGE` STRING(MAX), + `RECURSIVE` STRING(MAX), + `REFERENCES` STRING(MAX), + `RESPECT` STRING(MAX), + `RETURNING` STRING(MAX), + `RIGHT` STRING(MAX), + `ROLLUP` STRING(MAX), + `ROWS` STRING(MAX), + `SELECT` STRING(MAX), + `SESSION_USER` STRING(MAX), + `SET` STRING(MAX), + `SOME` STRING(MAX), + `STRUCT` STRING(MAX), + `SYMMETRIC` STRING(MAX), + `TABLE` STRING(MAX), + `TABLESAMPLE` STRING(MAX), + `THEN` STRING(MAX), + `TO` STRING(MAX), + `TRAILING` STRING(MAX), + `TREAT` STRING(MAX), + `TRUE` STRING(MAX), + `TRUE_P` STRING(MAX), + `UNBOUNDED` STRING(MAX), + `UNION` STRING(MAX), + `UNIQUE` STRING(MAX), + `UNNEST` STRING(MAX), + `USER` STRING(MAX), + `USING` STRING(MAX), + `VARIADIC` STRING(MAX), + `WHEN` STRING(MAX), + `WHERE` STRING(MAX), + `WINDOW` STRING(MAX), + `WITH` STRING(MAX), + `WITHIN` STRING(MAX) +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql new file mode 100644 index 0000000000..11889d70cb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql @@ -0,0 +1,142 @@ +CREATE TABLE "true" ( + "id" NUMBER(38, 0) PRIMARY KEY, + "ALL" VARCHAR2(4000), + "ANALYSE" VARCHAR2(4000), + "ANALYZE" VARCHAR2(4000), + "AND" VARCHAR2(4000), + "ANY" VARCHAR2(4000), + "ARRAY" VARCHAR2(4000), + "AS" VARCHAR2(4000), + "ASC" VARCHAR2(4000), + "ASSERT_ROWS_MODIFIED" VARCHAR2(4000), + "ASYMMETRIC" VARCHAR2(4000), + "AT" VARCHAR2(4000), + "BETWEEN" VARCHAR2(4000), + "BOTH" VARCHAR2(4000), + "BY" VARCHAR2(4000), + "CASE" VARCHAR2(4000), + "CAST" VARCHAR2(4000), + "CHECK" VARCHAR2(4000), + "COLLATE" VARCHAR2(4000), + "COLUMN" VARCHAR2(4000), + "CONSTRAINT" VARCHAR2(4000), + "CONTAINS" VARCHAR2(4000), + "CREATE" VARCHAR2(4000), + "CROSS" VARCHAR2(4000), + "CUBE" VARCHAR2(4000), + "CURRENT" VARCHAR2(4000), + "CURRENT_CATALOG" VARCHAR2(4000), + "CURRENT_DATE" VARCHAR2(4000), + "CURRENT_ROLE" VARCHAR2(4000), + "CURRENT_TIME" VARCHAR2(4000), + "CURRENT_TIMESTAMP" VARCHAR2(4000), + "CURRENT_USER" VARCHAR2(4000), + "DEFAULT" VARCHAR2(4000), + "DEFERRABLE" VARCHAR2(4000), + "DEFINE" VARCHAR2(4000), + "DESC" VARCHAR2(4000), + "DISTINCT" VARCHAR2(4000), + "DO" VARCHAR2(4000), + "ELSE" VARCHAR2(4000), + "END" VARCHAR2(4000), + "END_P" VARCHAR2(4000), + "ENUM" VARCHAR2(4000), + "ESCAPE" VARCHAR2(4000), + "EXCEPT" VARCHAR2(4000), + "EXCLUDE" VARCHAR2(4000), + "EXISTS" VARCHAR2(4000), + "EXTRACT" VARCHAR2(4000), + "FALSE" VARCHAR2(4000), + "FALSE_P" VARCHAR2(4000), + "FETCH" VARCHAR2(4000), + "FOLLOWING" VARCHAR2(4000), + "FOR" VARCHAR2(4000), + "FOREIGN" VARCHAR2(4000), + "FROM" VARCHAR2(4000), + "FULL" VARCHAR2(4000), + "GRANT" VARCHAR2(4000), + "GRAPH_TABLE" VARCHAR2(4000), + "GROUP" VARCHAR2(4000), + "GROUPING" VARCHAR2(4000), + "GROUPS" VARCHAR2(4000), + "GROUP_P" VARCHAR2(4000), + "HASH" VARCHAR2(4000), + "HAVING" VARCHAR2(4000), + "IF" VARCHAR2(4000), + "IGNORE" VARCHAR2(4000), + "IN" VARCHAR2(4000), + "INITIALLY" VARCHAR2(4000), + "INNER" VARCHAR2(4000), + "INTERSECT" VARCHAR2(4000), + "INTERVAL" VARCHAR2(4000), + "INTO" VARCHAR2(4000), + "IN_P" VARCHAR2(4000), + "IS" VARCHAR2(4000), + "JOIN" VARCHAR2(4000), + "LATERAL" VARCHAR2(4000), + "LATERAL_P" VARCHAR2(4000), + "LEADING" VARCHAR2(4000), + "LEFT" VARCHAR2(4000), + "LIKE" VARCHAR2(4000), + "LIMIT" VARCHAR2(4000), + "LOCALTIME" VARCHAR2(4000), + "LOCALTIMESTAMP" VARCHAR2(4000), + "LOOKUP" VARCHAR2(4000), + "MERGE" VARCHAR2(4000), + "NATURAL" VARCHAR2(4000), + "NEW" VARCHAR2(4000), + "NO" VARCHAR2(4000), + "NOT" VARCHAR2(4000), + "NULL" VARCHAR2(4000), + "NULLS" VARCHAR2(4000), + "NULL_P" VARCHAR2(4000), + "OF" VARCHAR2(4000), + "OFFSET" VARCHAR2(4000), + "ON" VARCHAR2(4000), + "ONLY" VARCHAR2(4000), + "OR" VARCHAR2(4000), + "ORDER" VARCHAR2(4000), + "OUTER" VARCHAR2(4000), + "OVER" VARCHAR2(4000), + "PARTITION" VARCHAR2(4000), + "PLACING" VARCHAR2(4000), + "PRECEDING" VARCHAR2(4000), + "PRIMARY" VARCHAR2(4000), + "PROTO" VARCHAR2(4000), + "RANGE" VARCHAR2(4000), + "RECURSIVE" VARCHAR2(4000), + "REFERENCES" VARCHAR2(4000), + "RESPECT" VARCHAR2(4000), + "RETURNING" VARCHAR2(4000), + "RIGHT" VARCHAR2(4000), + "ROLLUP" VARCHAR2(4000), + "ROWS" VARCHAR2(4000), + "SELECT" VARCHAR2(4000), + "SESSION_USER" VARCHAR2(4000), + "SET" VARCHAR2(4000), + "SOME" VARCHAR2(4000), + "STRUCT" VARCHAR2(4000), + "SYMMETRIC" VARCHAR2(4000), + "TABLE" VARCHAR2(4000), + "TABLESAMPLE" VARCHAR2(4000), + "THEN" VARCHAR2(4000), + "TO" VARCHAR2(4000), + "TRAILING" VARCHAR2(4000), + "TREAT" VARCHAR2(4000), + "TRUE" VARCHAR2(4000), + "TRUE_P" VARCHAR2(4000), + "UNBOUNDED" VARCHAR2(4000), + "UNION" VARCHAR2(4000), + "UNIQUE" VARCHAR2(4000), + "UNNEST" VARCHAR2(4000), + "USER" VARCHAR2(4000), + "USING" VARCHAR2(4000), + "VARIADIC" VARCHAR2(4000), + "WHEN" VARCHAR2(4000), + "WHERE" VARCHAR2(4000), + "WINDOW" VARCHAR2(4000), + "WITH" VARCHAR2(4000), + "WITHIN" VARCHAR2(4000) +); +INSERT INTO "true" ("id", "ALL", "AND", "AS") VALUES (1, 'all', 'and', 'as'); +COMMIT; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json new file mode 100644 index 0000000000..229d36db8a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json @@ -0,0 +1,323 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "SP_DATABASE", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "true", + "ColIds": ["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19", "c20", "c21", "c22", "c23", "c24", "c25", "c26", "c27", "c28", "c29", "c30", "c31", "c32", "c33", "c34", "c35", "c36", "c37", "c38", "c39", "c40", "c41", "c42", "c43", "c44", "c45", "c46", "c47", "c48", "c49", "c50", "c51", "c52", "c53", "c54", "c55", "c56", "c57", "c58", "c59", "c60", "c61", "c62", "c63", "c64", "c65", "c66", "c67", "c68", "c69", "c70", "c71", "c72", "c73", "c74", "c75", "c76", "c77", "c78", "c79", "c80", "c81", "c82", "c83", "c84", "c85", "c86", "c87", "c88", "c89", "c90", "c91", "c92", "c93", "c94", "c95", "c96", "c97", "c98", "c99", "c100", "c101", "c102", "c103", "c104", "c105", "c106", "c107", "c108", "c109", "c110", "c111", "c112", "c113", "c114", "c115", "c116", "c117", "c118", "c119", "c120", "c121", "c122", "c123", "c124", "c125", "c126", "c127", "c128", "c129", "c130", "c131", "c132", "c133", "c134", "c135", "c136", "c137", "c138"], + "ShardIdColumn": "", + "ColDefs": { + "c1": {"Name": "id", "T": {"Name": "INT64"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "ALL", "T": {"Name": "STRING"}, "Id": "c2"}, + "c3": {"Name": "ANALYSE", "T": {"Name": "STRING"}, "Id": "c3"}, + "c4": {"Name": "ANALYZE", "T": {"Name": "STRING"}, "Id": "c4"}, + "c5": {"Name": "AND", "T": {"Name": "STRING"}, "Id": "c5"}, + "c6": {"Name": "ANY", "T": {"Name": "STRING"}, "Id": "c6"}, + "c7": {"Name": "ARRAY", "T": {"Name": "STRING"}, "Id": "c7"}, + "c8": {"Name": "AS", "T": {"Name": "STRING"}, "Id": "c8"}, + "c9": {"Name": "ASC", "T": {"Name": "STRING"}, "Id": "c9"}, + "c10": {"Name": "ASSERT_ROWS_MODIFIED", "T": {"Name": "STRING"}, "Id": "c10"}, + "c11": {"Name": "ASYMMETRIC", "T": {"Name": "STRING"}, "Id": "c11"}, + "c12": {"Name": "AT", "T": {"Name": "STRING"}, "Id": "c12"}, + "c13": {"Name": "BETWEEN", "T": {"Name": "STRING"}, "Id": "c13"}, + "c14": {"Name": "BOTH", "T": {"Name": "STRING"}, "Id": "c14"}, + "c15": {"Name": "BY", "T": {"Name": "STRING"}, "Id": "c15"}, + "c16": {"Name": "CASE", "T": {"Name": "STRING"}, "Id": "c16"}, + "c17": {"Name": "CAST", "T": {"Name": "STRING"}, "Id": "c17"}, + "c18": {"Name": "CHECK", "T": {"Name": "STRING"}, "Id": "c18"}, + "c19": {"Name": "COLLATE", "T": {"Name": "STRING"}, "Id": "c19"}, + "c20": {"Name": "COLUMN", "T": {"Name": "STRING"}, "Id": "c20"}, + "c21": {"Name": "CONSTRAINT", "T": {"Name": "STRING"}, "Id": "c21"}, + "c22": {"Name": "CONTAINS", "T": {"Name": "STRING"}, "Id": "c22"}, + "c23": {"Name": "CREATE", "T": {"Name": "STRING"}, "Id": "c23"}, + "c24": {"Name": "CROSS", "T": {"Name": "STRING"}, "Id": "c24"}, + "c25": {"Name": "CUBE", "T": {"Name": "STRING"}, "Id": "c25"}, + "c26": {"Name": "CURRENT", "T": {"Name": "STRING"}, "Id": "c26"}, + "c27": {"Name": "CURRENT_CATALOG", "T": {"Name": "STRING"}, "Id": "c27"}, + "c28": {"Name": "CURRENT_DATE", "T": {"Name": "STRING"}, "Id": "c28"}, + "c29": {"Name": "CURRENT_ROLE", "T": {"Name": "STRING"}, "Id": "c29"}, + "c30": {"Name": "CURRENT_TIME", "T": {"Name": "STRING"}, "Id": "c30"}, + "c31": {"Name": "CURRENT_TIMESTAMP", "T": {"Name": "STRING"}, "Id": "c31"}, + "c32": {"Name": "CURRENT_USER", "T": {"Name": "STRING"}, "Id": "c32"}, + "c33": {"Name": "DEFAULT", "T": {"Name": "STRING"}, "Id": "c33"}, + "c34": {"Name": "DEFERRABLE", "T": {"Name": "STRING"}, "Id": "c34"}, + "c35": {"Name": "DEFINE", "T": {"Name": "STRING"}, "Id": "c35"}, + "c36": {"Name": "DESC", "T": {"Name": "STRING"}, "Id": "c36"}, + "c37": {"Name": "DISTINCT", "T": {"Name": "STRING"}, "Id": "c37"}, + "c38": {"Name": "DO", "T": {"Name": "STRING"}, "Id": "c38"}, + "c39": {"Name": "ELSE", "T": {"Name": "STRING"}, "Id": "c39"}, + "c40": {"Name": "END", "T": {"Name": "STRING"}, "Id": "c40"}, + "c41": {"Name": "END_P", "T": {"Name": "STRING"}, "Id": "c41"}, + "c42": {"Name": "ENUM", "T": {"Name": "STRING"}, "Id": "c42"}, + "c43": {"Name": "ESCAPE", "T": {"Name": "STRING"}, "Id": "c43"}, + "c44": {"Name": "EXCEPT", "T": {"Name": "STRING"}, "Id": "c44"}, + "c45": {"Name": "EXCLUDE", "T": {"Name": "STRING"}, "Id": "c45"}, + "c46": {"Name": "EXISTS", "T": {"Name": "STRING"}, "Id": "c46"}, + "c47": {"Name": "EXTRACT", "T": {"Name": "STRING"}, "Id": "c47"}, + "c48": {"Name": "FALSE", "T": {"Name": "STRING"}, "Id": "c48"}, + "c49": {"Name": "FALSE_P", "T": {"Name": "STRING"}, "Id": "c49"}, + "c50": {"Name": "FETCH", "T": {"Name": "STRING"}, "Id": "c50"}, + "c51": {"Name": "FOLLOWING", "T": {"Name": "STRING"}, "Id": "c51"}, + "c52": {"Name": "FOR", "T": {"Name": "STRING"}, "Id": "c52"}, + "c53": {"Name": "FOREIGN", "T": {"Name": "STRING"}, "Id": "c53"}, + "c54": {"Name": "FROM", "T": {"Name": "STRING"}, "Id": "c54"}, + "c55": {"Name": "FULL", "T": {"Name": "STRING"}, "Id": "c55"}, + "c56": {"Name": "GRANT", "T": {"Name": "STRING"}, "Id": "c56"}, + "c57": {"Name": "GRAPH_TABLE", "T": {"Name": "STRING"}, "Id": "c57"}, + "c58": {"Name": "GROUP", "T": {"Name": "STRING"}, "Id": "c58"}, + "c59": {"Name": "GROUPING", "T": {"Name": "STRING"}, "Id": "c59"}, + "c60": {"Name": "GROUPS", "T": {"Name": "STRING"}, "Id": "c60"}, + "c61": {"Name": "GROUP_P", "T": {"Name": "STRING"}, "Id": "c61"}, + "c62": {"Name": "HASH", "T": {"Name": "STRING"}, "Id": "c62"}, + "c63": {"Name": "HAVING", "T": {"Name": "STRING"}, "Id": "c63"}, + "c64": {"Name": "IF", "T": {"Name": "STRING"}, "Id": "c64"}, + "c65": {"Name": "IGNORE", "T": {"Name": "STRING"}, "Id": "c65"}, + "c66": {"Name": "IN", "T": {"Name": "STRING"}, "Id": "c66"}, + "c67": {"Name": "INITIALLY", "T": {"Name": "STRING"}, "Id": "c67"}, + "c68": {"Name": "INNER", "T": {"Name": "STRING"}, "Id": "c68"}, + "c69": {"Name": "INTERSECT", "T": {"Name": "STRING"}, "Id": "c69"}, + "c70": {"Name": "INTERVAL", "T": {"Name": "STRING"}, "Id": "c70"}, + "c71": {"Name": "INTO", "T": {"Name": "STRING"}, "Id": "c71"}, + "c72": {"Name": "IN_P", "T": {"Name": "STRING"}, "Id": "c72"}, + "c73": {"Name": "IS", "T": {"Name": "STRING"}, "Id": "c73"}, + "c74": {"Name": "JOIN", "T": {"Name": "STRING"}, "Id": "c74"}, + "c75": {"Name": "LATERAL", "T": {"Name": "STRING"}, "Id": "c75"}, + "c76": {"Name": "LATERAL_P", "T": {"Name": "STRING"}, "Id": "c76"}, + "c77": {"Name": "LEADING", "T": {"Name": "STRING"}, "Id": "c77"}, + "c78": {"Name": "LEFT", "T": {"Name": "STRING"}, "Id": "c78"}, + "c79": {"Name": "LIKE", "T": {"Name": "STRING"}, "Id": "c79"}, + "c80": {"Name": "LIMIT", "T": {"Name": "STRING"}, "Id": "c80"}, + "c81": {"Name": "LOCALTIME", "T": {"Name": "STRING"}, "Id": "c81"}, + "c82": {"Name": "LOCALTIMESTAMP", "T": {"Name": "STRING"}, "Id": "c82"}, + "c83": {"Name": "LOOKUP", "T": {"Name": "STRING"}, "Id": "c83"}, + "c84": {"Name": "MERGE", "T": {"Name": "STRING"}, "Id": "c84"}, + "c85": {"Name": "NATURAL", "T": {"Name": "STRING"}, "Id": "c85"}, + "c86": {"Name": "NEW", "T": {"Name": "STRING"}, "Id": "c86"}, + "c87": {"Name": "NO", "T": {"Name": "STRING"}, "Id": "c87"}, + "c88": {"Name": "NOT", "T": {"Name": "STRING"}, "Id": "c88"}, + "c89": {"Name": "NULL", "T": {"Name": "STRING"}, "Id": "c89"}, + "c90": {"Name": "NULLS", "T": {"Name": "STRING"}, "Id": "c90"}, + "c91": {"Name": "NULL_P", "T": {"Name": "STRING"}, "Id": "c91"}, + "c92": {"Name": "OF", "T": {"Name": "STRING"}, "Id": "c92"}, + "c93": {"Name": "OFFSET", "T": {"Name": "STRING"}, "Id": "c93"}, + "c94": {"Name": "ON", "T": {"Name": "STRING"}, "Id": "c94"}, + "c95": {"Name": "ONLY", "T": {"Name": "STRING"}, "Id": "c95"}, + "c96": {"Name": "OR", "T": {"Name": "STRING"}, "Id": "c96"}, + "c97": {"Name": "ORDER", "T": {"Name": "STRING"}, "Id": "c97"}, + "c98": {"Name": "OUTER", "T": {"Name": "STRING"}, "Id": "c98"}, + "c99": {"Name": "OVER", "T": {"Name": "STRING"}, "Id": "c99"}, + "c100": {"Name": "PARTITION", "T": {"Name": "STRING"}, "Id": "c100"}, + "c101": {"Name": "PLACING", "T": {"Name": "STRING"}, "Id": "c101"}, + "c102": {"Name": "PRECEDING", "T": {"Name": "STRING"}, "Id": "c102"}, + "c103": {"Name": "PRIMARY", "T": {"Name": "STRING"}, "Id": "c103"}, + "c104": {"Name": "PROTO", "T": {"Name": "STRING"}, "Id": "c104"}, + "c105": {"Name": "RANGE", "T": {"Name": "STRING"}, "Id": "c105"}, + "c106": {"Name": "RECURSIVE", "T": {"Name": "STRING"}, "Id": "c106"}, + "c107": {"Name": "REFERENCES", "T": {"Name": "STRING"}, "Id": "c107"}, + "c108": {"Name": "RESPECT", "T": {"Name": "STRING"}, "Id": "c108"}, + "c109": {"Name": "RETURNING", "T": {"Name": "STRING"}, "Id": "c109"}, + "c110": {"Name": "RIGHT", "T": {"Name": "STRING"}, "Id": "c110"}, + "c111": {"Name": "ROLLUP", "T": {"Name": "STRING"}, "Id": "c111"}, + "c112": {"Name": "ROWS", "T": {"Name": "STRING"}, "Id": "c112"}, + "c113": {"Name": "SELECT", "T": {"Name": "STRING"}, "Id": "c113"}, + "c114": {"Name": "SESSION_USER", "T": {"Name": "STRING"}, "Id": "c114"}, + "c115": {"Name": "SET", "T": {"Name": "STRING"}, "Id": "c115"}, + "c116": {"Name": "SOME", "T": {"Name": "STRING"}, "Id": "c116"}, + "c117": {"Name": "STRUCT", "T": {"Name": "STRING"}, "Id": "c117"}, + "c118": {"Name": "SYMMETRIC", "T": {"Name": "STRING"}, "Id": "c118"}, + "c119": {"Name": "TABLE", "T": {"Name": "STRING"}, "Id": "c119"}, + "c120": {"Name": "TABLESAMPLE", "T": {"Name": "STRING"}, "Id": "c120"}, + "c121": {"Name": "THEN", "T": {"Name": "STRING"}, "Id": "c121"}, + "c122": {"Name": "TO", "T": {"Name": "STRING"}, "Id": "c122"}, + "c123": {"Name": "TRAILING", "T": {"Name": "STRING"}, "Id": "c123"}, + "c124": {"Name": "TREAT", "T": {"Name": "STRING"}, "Id": "c124"}, + "c125": {"Name": "TRUE", "T": {"Name": "STRING"}, "Id": "c125"}, + "c126": {"Name": "TRUE_P", "T": {"Name": "STRING"}, "Id": "c126"}, + "c127": {"Name": "UNBOUNDED", "T": {"Name": "STRING"}, "Id": "c127"}, + "c128": {"Name": "UNION", "T": {"Name": "STRING"}, "Id": "c128"}, + "c129": {"Name": "UNIQUE", "T": {"Name": "STRING"}, "Id": "c129"}, + "c130": {"Name": "UNNEST", "T": {"Name": "STRING"}, "Id": "c130"}, + "c131": {"Name": "USER", "T": {"Name": "STRING"}, "Id": "c131"}, + "c132": {"Name": "USING", "T": {"Name": "STRING"}, "Id": "c132"}, + "c133": {"Name": "VARIADIC", "T": {"Name": "STRING"}, "Id": "c133"}, + "c134": {"Name": "WHEN", "T": {"Name": "STRING"}, "Id": "c134"}, + "c135": {"Name": "WHERE", "T": {"Name": "STRING"}, "Id": "c135"}, + "c136": {"Name": "WINDOW", "T": {"Name": "STRING"}, "Id": "c136"}, + "c137": {"Name": "WITH", "T": {"Name": "STRING"}, "Id": "c137"}, + "c138": {"Name": "WITHIN", "T": {"Name": "STRING"}, "Id": "c138"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "true", + "Schema": "SRC_DATABASE", + "ColIds": ["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19", "c20", "c21", "c22", "c23", "c24", "c25", "c26", "c27", "c28", "c29", "c30", "c31", "c32", "c33", "c34", "c35", "c36", "c37", "c38", "c39", "c40", "c41", "c42", "c43", "c44", "c45", "c46", "c47", "c48", "c49", "c50", "c51", "c52", "c53", "c54", "c55", "c56", "c57", "c58", "c59", "c60", "c61", "c62", "c63", "c64", "c65", "c66", "c67", "c68", "c69", "c70", "c71", "c72", "c73", "c74", "c75", "c76", "c77", "c78", "c79", "c80", "c81", "c82", "c83", "c84", "c85", "c86", "c87", "c88", "c89", "c90", "c91", "c92", "c93", "c94", "c95", "c96", "c97", "c98", "c99", "c100", "c101", "c102", "c103", "c104", "c105", "c106", "c107", "c108", "c109", "c110", "c111", "c112", "c113", "c114", "c115", "c116", "c117", "c118", "c119", "c120", "c121", "c122", "c123", "c124", "c125", "c126", "c127", "c128", "c129", "c130", "c131", "c132", "c133", "c134", "c135", "c136", "c137", "c138"], + "ColDefs": { + "c1": {"Name": "id", "T": {"Name": "INT64"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "ALL", "T": {"Name": "STRING"}, "Id": "c2"}, + "c3": {"Name": "ANALYSE", "T": {"Name": "STRING"}, "Id": "c3"}, + "c4": {"Name": "ANALYZE", "T": {"Name": "STRING"}, "Id": "c4"}, + "c5": {"Name": "AND", "T": {"Name": "STRING"}, "Id": "c5"}, + "c6": {"Name": "ANY", "T": {"Name": "STRING"}, "Id": "c6"}, + "c7": {"Name": "ARRAY", "T": {"Name": "STRING"}, "Id": "c7"}, + "c8": {"Name": "AS", "T": {"Name": "STRING"}, "Id": "c8"}, + "c9": {"Name": "ASC", "T": {"Name": "STRING"}, "Id": "c9"}, + "c10": {"Name": "ASSERT_ROWS_MODIFIED", "T": {"Name": "STRING"}, "Id": "c10"}, + "c11": {"Name": "ASYMMETRIC", "T": {"Name": "STRING"}, "Id": "c11"}, + "c12": {"Name": "AT", "T": {"Name": "STRING"}, "Id": "c12"}, + "c13": {"Name": "BETWEEN", "T": {"Name": "STRING"}, "Id": "c13"}, + "c14": {"Name": "BOTH", "T": {"Name": "STRING"}, "Id": "c14"}, + "c15": {"Name": "BY", "T": {"Name": "STRING"}, "Id": "c15"}, + "c16": {"Name": "CASE", "T": {"Name": "STRING"}, "Id": "c16"}, + "c17": {"Name": "CAST", "T": {"Name": "STRING"}, "Id": "c17"}, + "c18": {"Name": "CHECK", "T": {"Name": "STRING"}, "Id": "c18"}, + "c19": {"Name": "COLLATE", "T": {"Name": "STRING"}, "Id": "c19"}, + "c20": {"Name": "COLUMN", "T": {"Name": "STRING"}, "Id": "c20"}, + "c21": {"Name": "CONSTRAINT", "T": {"Name": "STRING"}, "Id": "c21"}, + "c22": {"Name": "CONTAINS", "T": {"Name": "STRING"}, "Id": "c22"}, + "c23": {"Name": "CREATE", "T": {"Name": "STRING"}, "Id": "c23"}, + "c24": {"Name": "CROSS", "T": {"Name": "STRING"}, "Id": "c24"}, + "c25": {"Name": "CUBE", "T": {"Name": "STRING"}, "Id": "c25"}, + "c26": {"Name": "CURRENT", "T": {"Name": "STRING"}, "Id": "c26"}, + "c27": {"Name": "CURRENT_CATALOG", "T": {"Name": "STRING"}, "Id": "c27"}, + "c28": {"Name": "CURRENT_DATE", "T": {"Name": "STRING"}, "Id": "c28"}, + "c29": {"Name": "CURRENT_ROLE", "T": {"Name": "STRING"}, "Id": "c29"}, + "c30": {"Name": "CURRENT_TIME", "T": {"Name": "STRING"}, "Id": "c30"}, + "c31": {"Name": "CURRENT_TIMESTAMP", "T": {"Name": "STRING"}, "Id": "c31"}, + "c32": {"Name": "CURRENT_USER", "T": {"Name": "STRING"}, "Id": "c32"}, + "c33": {"Name": "DEFAULT", "T": {"Name": "STRING"}, "Id": "c33"}, + "c34": {"Name": "DEFERRABLE", "T": {"Name": "STRING"}, "Id": "c34"}, + "c35": {"Name": "DEFINE", "T": {"Name": "STRING"}, "Id": "c35"}, + "c36": {"Name": "DESC", "T": {"Name": "STRING"}, "Id": "c36"}, + "c37": {"Name": "DISTINCT", "T": {"Name": "STRING"}, "Id": "c37"}, + "c38": {"Name": "DO", "T": {"Name": "STRING"}, "Id": "c38"}, + "c39": {"Name": "ELSE", "T": {"Name": "STRING"}, "Id": "c39"}, + "c40": {"Name": "END", "T": {"Name": "STRING"}, "Id": "c40"}, + "c41": {"Name": "END_P", "T": {"Name": "STRING"}, "Id": "c41"}, + "c42": {"Name": "ENUM", "T": {"Name": "STRING"}, "Id": "c42"}, + "c43": {"Name": "ESCAPE", "T": {"Name": "STRING"}, "Id": "c43"}, + "c44": {"Name": "EXCEPT", "T": {"Name": "STRING"}, "Id": "c44"}, + "c45": {"Name": "EXCLUDE", "T": {"Name": "STRING"}, "Id": "c45"}, + "c46": {"Name": "EXISTS", "T": {"Name": "STRING"}, "Id": "c46"}, + "c47": {"Name": "EXTRACT", "T": {"Name": "STRING"}, "Id": "c47"}, + "c48": {"Name": "FALSE", "T": {"Name": "STRING"}, "Id": "c48"}, + "c49": {"Name": "FALSE_P", "T": {"Name": "STRING"}, "Id": "c49"}, + "c50": {"Name": "FETCH", "T": {"Name": "STRING"}, "Id": "c50"}, + "c51": {"Name": "FOLLOWING", "T": {"Name": "STRING"}, "Id": "c51"}, + "c52": {"Name": "FOR", "T": {"Name": "STRING"}, "Id": "c52"}, + "c53": {"Name": "FOREIGN", "T": {"Name": "STRING"}, "Id": "c53"}, + "c54": {"Name": "FROM", "T": {"Name": "STRING"}, "Id": "c54"}, + "c55": {"Name": "FULL", "T": {"Name": "STRING"}, "Id": "c55"}, + "c56": {"Name": "GRANT", "T": {"Name": "STRING"}, "Id": "c56"}, + "c57": {"Name": "GRAPH_TABLE", "T": {"Name": "STRING"}, "Id": "c57"}, + "c58": {"Name": "GROUP", "T": {"Name": "STRING"}, "Id": "c58"}, + "c59": {"Name": "GROUPING", "T": {"Name": "STRING"}, "Id": "c59"}, + "c60": {"Name": "GROUPS", "T": {"Name": "STRING"}, "Id": "c60"}, + "c61": {"Name": "GROUP_P", "T": {"Name": "STRING"}, "Id": "c61"}, + "c62": {"Name": "HASH", "T": {"Name": "STRING"}, "Id": "c62"}, + "c63": {"Name": "HAVING", "T": {"Name": "STRING"}, "Id": "c63"}, + "c64": {"Name": "IF", "T": {"Name": "STRING"}, "Id": "c64"}, + "c65": {"Name": "IGNORE", "T": {"Name": "STRING"}, "Id": "c65"}, + "c66": {"Name": "IN", "T": {"Name": "STRING"}, "Id": "c66"}, + "c67": {"Name": "INITIALLY", "T": {"Name": "STRING"}, "Id": "c67"}, + "c68": {"Name": "INNER", "T": {"Name": "STRING"}, "Id": "c68"}, + "c69": {"Name": "INTERSECT", "T": {"Name": "STRING"}, "Id": "c69"}, + "c70": {"Name": "INTERVAL", "T": {"Name": "STRING"}, "Id": "c70"}, + "c71": {"Name": "INTO", "T": {"Name": "STRING"}, "Id": "c71"}, + "c72": {"Name": "IN_P", "T": {"Name": "STRING"}, "Id": "c72"}, + "c73": {"Name": "IS", "T": {"Name": "STRING"}, "Id": "c73"}, + "c74": {"Name": "JOIN", "T": {"Name": "STRING"}, "Id": "c74"}, + "c75": {"Name": "LATERAL", "T": {"Name": "STRING"}, "Id": "c75"}, + "c76": {"Name": "LATERAL_P", "T": {"Name": "STRING"}, "Id": "c76"}, + "c77": {"Name": "LEADING", "T": {"Name": "STRING"}, "Id": "c77"}, + "c78": {"Name": "LEFT", "T": {"Name": "STRING"}, "Id": "c78"}, + "c79": {"Name": "LIKE", "T": {"Name": "STRING"}, "Id": "c79"}, + "c80": {"Name": "LIMIT", "T": {"Name": "STRING"}, "Id": "c80"}, + "c81": {"Name": "LOCALTIME", "T": {"Name": "STRING"}, "Id": "c81"}, + "c82": {"Name": "LOCALTIMESTAMP", "T": {"Name": "STRING"}, "Id": "c82"}, + "c83": {"Name": "LOOKUP", "T": {"Name": "STRING"}, "Id": "c83"}, + "c84": {"Name": "MERGE", "T": {"Name": "STRING"}, "Id": "c84"}, + "c85": {"Name": "NATURAL", "T": {"Name": "STRING"}, "Id": "c85"}, + "c86": {"Name": "NEW", "T": {"Name": "STRING"}, "Id": "c86"}, + "c87": {"Name": "NO", "T": {"Name": "STRING"}, "Id": "c87"}, + "c88": {"Name": "NOT", "T": {"Name": "STRING"}, "Id": "c88"}, + "c89": {"Name": "NULL", "T": {"Name": "STRING"}, "Id": "c89"}, + "c90": {"Name": "NULLS", "T": {"Name": "STRING"}, "Id": "c90"}, + "c91": {"Name": "NULL_P", "T": {"Name": "STRING"}, "Id": "c91"}, + "c92": {"Name": "OF", "T": {"Name": "STRING"}, "Id": "c92"}, + "c93": {"Name": "OFFSET", "T": {"Name": "STRING"}, "Id": "c93"}, + "c94": {"Name": "ON", "T": {"Name": "STRING"}, "Id": "c94"}, + "c95": {"Name": "ONLY", "T": {"Name": "STRING"}, "Id": "c95"}, + "c96": {"Name": "OR", "T": {"Name": "STRING"}, "Id": "c96"}, + "c97": {"Name": "ORDER", "T": {"Name": "STRING"}, "Id": "c97"}, + "c98": {"Name": "OUTER", "T": {"Name": "STRING"}, "Id": "c98"}, + "c99": {"Name": "OVER", "T": {"Name": "STRING"}, "Id": "c99"}, + "c100": {"Name": "PARTITION", "T": {"Name": "STRING"}, "Id": "c100"}, + "c101": {"Name": "PLACING", "T": {"Name": "STRING"}, "Id": "c101"}, + "c102": {"Name": "PRECEDING", "T": {"Name": "STRING"}, "Id": "c102"}, + "c103": {"Name": "PRIMARY", "T": {"Name": "STRING"}, "Id": "c103"}, + "c104": {"Name": "PROTO", "T": {"Name": "STRING"}, "Id": "c104"}, + "c105": {"Name": "RANGE", "T": {"Name": "STRING"}, "Id": "c105"}, + "c106": {"Name": "RECURSIVE", "T": {"Name": "STRING"}, "Id": "c106"}, + "c107": {"Name": "REFERENCES", "T": {"Name": "STRING"}, "Id": "c107"}, + "c108": {"Name": "RESPECT", "T": {"Name": "STRING"}, "Id": "c108"}, + "c109": {"Name": "RETURNING", "T": {"Name": "STRING"}, "Id": "c109"}, + "c110": {"Name": "RIGHT", "T": {"Name": "STRING"}, "Id": "c110"}, + "c111": {"Name": "ROLLUP", "T": {"Name": "STRING"}, "Id": "c111"}, + "c112": {"Name": "ROWS", "T": {"Name": "STRING"}, "Id": "c112"}, + "c113": {"Name": "SELECT", "T": {"Name": "STRING"}, "Id": "c113"}, + "c114": {"Name": "SESSION_USER", "T": {"Name": "STRING"}, "Id": "c114"}, + "c115": {"Name": "SET", "T": {"Name": "STRING"}, "Id": "c115"}, + "c116": {"Name": "SOME", "T": {"Name": "STRING"}, "Id": "c116"}, + "c117": {"Name": "STRUCT", "T": {"Name": "STRING"}, "Id": "c117"}, + "c118": {"Name": "SYMMETRIC", "T": {"Name": "STRING"}, "Id": "c118"}, + "c119": {"Name": "TABLE", "T": {"Name": "STRING"}, "Id": "c119"}, + "c120": {"Name": "TABLESAMPLE", "T": {"Name": "STRING"}, "Id": "c120"}, + "c121": {"Name": "THEN", "T": {"Name": "STRING"}, "Id": "c121"}, + "c122": {"Name": "TO", "T": {"Name": "STRING"}, "Id": "c122"}, + "c123": {"Name": "TRAILING", "T": {"Name": "STRING"}, "Id": "c123"}, + "c124": {"Name": "TREAT", "T": {"Name": "STRING"}, "Id": "c124"}, + "c125": {"Name": "TRUE", "T": {"Name": "STRING"}, "Id": "c125"}, + "c126": {"Name": "TRUE_P", "T": {"Name": "STRING"}, "Id": "c126"}, + "c127": {"Name": "UNBOUNDED", "T": {"Name": "STRING"}, "Id": "c127"}, + "c128": {"Name": "UNION", "T": {"Name": "STRING"}, "Id": "c128"}, + "c129": {"Name": "UNIQUE", "T": {"Name": "STRING"}, "Id": "c129"}, + "c130": {"Name": "UNNEST", "T": {"Name": "STRING"}, "Id": "c130"}, + "c131": {"Name": "USER", "T": {"Name": "STRING"}, "Id": "c131"}, + "c132": {"Name": "USING", "T": {"Name": "STRING"}, "Id": "c132"}, + "c133": {"Name": "VARIADIC", "T": {"Name": "STRING"}, "Id": "c133"}, + "c134": {"Name": "WHEN", "T": {"Name": "STRING"}, "Id": "c134"}, + "c135": {"Name": "WHERE", "T": {"Name": "STRING"}, "Id": "c135"}, + "c136": {"Name": "WINDOW", "T": {"Name": "STRING"}, "Id": "c136"}, + "c137": {"Name": "WITH", "T": {"Name": "STRING"}, "Id": "c137"}, + "c138": {"Name": "WITHIN", "T": {"Name": "STRING"}, "Id": "c138"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro new file mode 100644 index 0000000000..01421913ae Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro new file mode 100644 index 0000000000..0b050282ec Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro new file mode 100644 index 0000000000..79c690553a Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro new file mode 100644 index 0000000000..195a491cd8 Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro new file mode 100644 index 0000000000..3996dd9a66 Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro new file mode 100644 index 0000000000..844e0ec20a Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..60816e9bc9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,40 @@ +CREATE TABLE Movie ( + id INT64 NOT NULL, + name STRING(200), + actor NUMERIC, + startTime TIMESTAMP, +) PRIMARY KEY(id); + +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + subscribed BOOL, + plan STRING(1), + startDate TIMESTAMP, +) PRIMARY KEY (id); + +CREATE TABLE Authors ( + author_id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (author_id); + +CREATE TABLE Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, +) PRIMARY KEY (id); + +CREATE INDEX author_id_6 ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id); + +CREATE TABLE Articles ( + id INT64 NOT NULL, + name STRING(200), + published_date TIMESTAMP, + author_id INT64 NOT NULL, +) PRIMARY KEY (author_id, id), +INTERLEAVE IN PARENT Authors; + +CREATE INDEX author_id ON Articles (author_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql new file mode 100644 index 0000000000..dcdbc6bf55 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql @@ -0,0 +1,44 @@ +CREATE TABLE "Movie" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "actor" NUMBER, + "startTime" TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE "Users" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "age" INTEGER, + "subscribed" NUMBER(1), + "plan" CHAR(1), + "startDate" DATE, + PRIMARY KEY ("id") +); + +CREATE TABLE "Authors" ( + "author_id" INTEGER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Articles" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "published_date" DATE, + "author_id" INTEGER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Articles" add FOREIGN KEY ("author_id") references "Authors"("author_id"); +CREATE INDEX "author_id" ON "Articles" ("author_id"); + +CREATE TABLE "Books" ( + "id" INTEGER NOT NULL, + "title" VARCHAR2(200), + "author_id" INTEGER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" add FOREIGN KEY ("author_id") references "Authors"("author_id"); +CREATE INDEX "author_id_6" ON "Books" ("author_id"); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..71ee384cbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..fd61478626 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500), + "last_name1" VARCHAR2(500), + "first_name2" VARCHAR2(500), + "last_name2" VARCHAR2(500), + "first_name3" VARCHAR2(500), + "last_name3" VARCHAR2(500), + "ID" NUMBER NOT NULL PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json new file mode 100644 index 0000000000..c0b5fc0009 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json @@ -0,0 +1,10 @@ +{ + "renamedTables":{ + "person1":"human1" + }, + "renamedColumns":{ + "person1":{ + "first_name1":"name1" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..62f8aaa748 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS Authors ( + author_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY (author_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), +) PRIMARY KEY (synth_id); + +CREATE INDEX author_id ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id) ON DELETE NO ACTION; + +CREATE TABLE IF NOT EXISTS Genre ( + name STRING(200), + genre_id INT64 NOT NULL, +) PRIMARY KEY (genre_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql new file mode 100644 index 0000000000..c2bb9f0fca --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE "Authors" ( + "author_id" NUMBER(10,0) NOT NULL, + "name" VARCHAR2(25), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER(10,0) NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER(10,0), + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD CONSTRAINT "Books_ibfk_1" FOREIGN KEY ("author_id") REFERENCES "Authors" ("author_id"); + +CREATE TABLE "Genre" ( + "name" VARCHAR2(200), + "genre_id" NUMBER(10,0) NOT NULL, + PRIMARY KEY ("genre_id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json new file mode 100644 index 0000000000..b7ff136a40 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json @@ -0,0 +1,506 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "e2e-it", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t34": { + "Name": "Authors", + "ColIds": [ + "c37", + "c38" + ], + "ShardIdColumn": "", + "ColDefs": { + "c37": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: author_id int(10)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Authors", + "Id": "t34" + }, + "t35": { + "Name": "Books", + "ColIds": [ + "c41", + "c42", + "c43", + "c47" + ], + "ShardIdColumn": "", + "ColDefs": { + "c41": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c47": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "Id": "f40", + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION" + } + ], + "Indexes": [ + { + "Name": "author_id", + "TableId": "t35", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t35" + } + }, + "SyntheticPKeys": { + "t35": { + "ColId": "c47", + "Sequence": 0 + } + }, + "SrcSchema": { + "t34": { + "Name": "Authors", + "Schema": "e2e-it", + "ColIds": [ + "c37", + "c38", + "c39" + ], + "ColDefs": { + "c37": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c39": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t34" + }, + "t35": { + "Name": "Books", + "Schema": "e2e-it", + "ColIds": [ + "c41", + "c42", + "c43" + ], + "ColDefs": { + "c41": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION", + "Id": "f40" + } + ], + "Indexes": [ + { + "Name": "author_id", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "Id": "t35" + }, + "t36": { + "Name": "Genre", + "Schema": "e2e-it", + "ColIds": [ + "c44", + "c45" + ], + "ColDefs": { + "c44": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c45": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c45", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t36" + } + }, + "SchemaIssues": { + "t34": { + "ColumnLevelIssues": { + "c37": [ + 14 + ], + "c38": [] + }, + "TableLevelIssues": null + }, + "t35": { + "ColumnLevelIssues": { + "c41": [ + 14 + ], + "c42": [], + "c43": [ + 14, + 24 + ], + "c47": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t36": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..d57b1f3c9e --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS Category ( + category_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY(category_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), + extraCol1 INT64, +) PRIMARY KEY(synth_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql new file mode 100644 index 0000000000..6de37949db --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql @@ -0,0 +1,13 @@ +CREATE TABLE "Category" ( + "category_id" NUMBER PRIMARY KEY, + "name" VARCHAR2(25), + "last_update" TIMESTAMP +); +ALTER TABLE "Category" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +CREATE TABLE "Books" ( + "id" NUMBER, + "title" VARCHAR2(200), + "author_id" NUMBER +); +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json new file mode 100644 index 0000000000..43305ab260 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json @@ -0,0 +1,456 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "mysql-schema.sql", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t15515": { + "Name": "Category", + "ColIds": [ + "c15516", + "c15517" + ], + "ShardIdColumn": "", + "ColDefs": { + "c15516": { + "Name": "category_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: category_id tinyint(4)", + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Category", + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "ColIds": [ + "c15520", + "c15521", + "c15522", + "c15523", + "c1" + ], + "ShardIdColumn": "", + "ColDefs": { + "c1": { + "Name": "extraCol1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c1", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15520": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(11)", + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15523": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c15523", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15523", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t15519" + } + }, + "SyntheticPKeys": { + "t15519": { + "ColId": "c15523", + "Sequence": 0 + } + }, + "SrcSchema": { + "t15515": { + "Name": "Category", + "Schema": "", + "ColIds": [ + "c15516", + "c15517", + "c15518" + ], + "ColDefs": { + "c15516": { + "Name": "category_id", + "Type": { + "Name": "tinyint", + "Mods": [ + 4 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15518": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15518", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "Schema": "", + "ColIds": [ + "c15520", + "c15521", + "c15522" + ], + "ColDefs": { + "c15520": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "Type": { + "Name": "varchar", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": null, + "Indexes": null, + "Id": "t15519" + }, + "t15520": { + "Name": "Genre", + "Schema": "", + "ColIds": [ + "c13", + "c14" + ], + "ColDefs": { + "c13": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + }, + "c14": { + "Name": "genre_id", + "Type": { + "Name": "decimal", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14" + } + }, + "PrimaryKeys": [ + { + "ColId": "c14", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15520" + } + }, + "SchemaIssues": { + "t15515": { + "ColumnLevelIssues": { + "c15516": [ + 14 + ], + "c15517": [] + }, + "TableLevelIssues": null + }, + "t15519": { + "ColumnLevelIssues": { + "c15520": [ + 14 + ], + "c15521": [], + "c15522": [ + 14 + ], + "c15523": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t15520": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..f677c0086c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,7 @@ +ALTER DATABASE db SET OPTIONS (default_time_zone = 'Australia/Brisbane'); + +CREATE TABLE IF NOT EXISTS DateData ( + id INT64 NOT NULL, + timestamp_column TIMESTAMP, + datetime_column TIMESTAMP, +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql new file mode 100644 index 0000000000..2a3ecbf1b2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE "DateData" ( + "id" NUMBER PRIMARY KEY, + "timestamp_column" TIMESTAMP, + "datetime_column" TIMESTAMP +); + +ALTER TABLE "DateData" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..8e5f5c2236 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,71 @@ +CREATE TABLE T_SPRG_R0_VARCHAR2 (ID INT64 NOT NULL, COL_0 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R1_VARCHAR (ID INT64 NOT NULL, COL_1 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R2_CHAR (ID INT64 NOT NULL, COL_2 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R3_CHARACTER (ID INT64 NOT NULL, COL_3 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID INT64 NOT NULL, COL_4 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R5_NCHAR (ID INT64 NOT NULL, COL_5 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID INT64 NOT NULL, COL_6 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R7_NATIONAL_C (ID INT64 NOT NULL, COL_7 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R8_NATIONAL_C (ID INT64 NOT NULL, COL_8 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R9_NATIONAL_C (ID INT64 NOT NULL, COL_9 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R10_NATIONAL_C (ID INT64 NOT NULL, COL_10 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R11_NUMBER (ID INT64 NOT NULL, COL_11 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R12_NUMERIC (ID INT64 NOT NULL, COL_12 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R13_DECIMAL (ID INT64 NOT NULL, COL_13 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R14_DEC (ID INT64 NOT NULL, COL_14 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R15_FLOAT (ID INT64 NOT NULL, COL_15 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID INT64 NOT NULL, COL_16 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R17_REAL (ID INT64 NOT NULL, COL_17 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R18_BINARY_FLO (ID INT64 NOT NULL, COL_18 FLOAT32) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R19_BINARY_DOU (ID INT64 NOT NULL, COL_19 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R20_INTEGER (ID INT64 NOT NULL, COL_20 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R21_INT (ID INT64 NOT NULL, COL_21 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R22_SMALLINT (ID INT64 NOT NULL, COL_22 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R23_DATE (ID INT64 NOT NULL, COL_23 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R24_TIMESTAMP (ID INT64 NOT NULL, COL_24 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID INT64 NOT NULL, COL_25 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID INT64 NOT NULL, COL_26 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R28_INTERVAL_D (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R29_RAW (ID INT64 NOT NULL, COL_29 BYTES(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R30_LONG_RAW (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R31_BLOB (ID INT64 NOT NULL, COL_31 BYTES(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R32_CLOB (ID INT64 NOT NULL, COL_32 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R33_NCLOB (ID INT64 NOT NULL, COL_33 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R34_BFILE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R35_LONG (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R36_ROWID (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R37_UROWID (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R38_BOOLEAN (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R39_JSON (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R40_XMLTYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R44_VECTOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R45_ANYDATA (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R46_ANYTYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R47_ANYDATASET (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R48_URITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R49_DBURITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R50_XDBURITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R51_HTTPURITYP (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R52_EXPRESSION (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R53_ORDAUDIO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R54_ORDIMAGE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R55_ORDVIDEO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R56_ORDDOC (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R57_SI_STILLIM (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R58_SI_COLOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R60_SI_COLORHI (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R61_SI_POSITIO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R63_SI_FEATURE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R64_MLSLABEL (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R65_REF (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R66_REF_CURSOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R67_VARRAY (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R68_NESTED_TAB (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql new file mode 100644 index 0000000000..b6a89adea2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql @@ -0,0 +1,142 @@ +CREATE TABLE T_RZFA_VARCHAR2_TABLE (ID NUMBER PRIMARY KEY, VARCHAR2_COL VARCHAR2(255)); +INSERT INTO T_RZFA_VARCHAR2_TABLE (ID, VARCHAR2_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_VARCHAR_TABLE (ID NUMBER PRIMARY KEY, VARCHAR_COL VARCHAR(255)); +INSERT INTO T_RZFA_VARCHAR_TABLE (ID, VARCHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CHAR_TABLE (ID NUMBER PRIMARY KEY, CHAR_COL CHAR(255)); +INSERT INTO T_RZFA_CHAR_TABLE (ID, CHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CHARACTER_TABLE (ID NUMBER PRIMARY KEY, CHARACTER_COL CHARACTER(255)); +INSERT INTO T_RZFA_CHARACTER_TABLE (ID, CHARACTER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NVARCHAR2_TABLE (ID NUMBER PRIMARY KEY, NVARCHAR2_COL NVARCHAR2(255)); +INSERT INTO T_RZFA_NVARCHAR2_TABLE (ID, NVARCHAR2_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCHAR_TABLE (ID NUMBER PRIMARY KEY, NCHAR_COL NCHAR(255)); +INSERT INTO T_RZFA_NCHAR_TABLE (ID, NCHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCHAR_VARYING_TABLE (ID NUMBER PRIMARY KEY, NCHAR_VARYING_COL NCHAR VARYING(255)); +INSERT INTO T_RZFA_NCHAR_VARYING_TABLE (ID, NCHAR_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHARACTER_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHARACTER_COL NATIONAL CHARACTER(255)); +INSERT INTO T_RZFA_NATIONAL_CHARACTER_TABLE (ID, NATIONAL_CHARACTER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHAR_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHAR_COL NATIONAL CHAR(255)); +INSERT INTO T_RZFA_NATIONAL_CHAR_TABLE (ID, NATIONAL_CHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHARACTER_VARYING_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHARACTER_VARYING_COL NATIONAL CHARACTER VARYING(255)); +INSERT INTO T_RZFA_NATIONAL_CHARACTER_VARYING_TABLE (ID, NATIONAL_CHARACTER_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHAR_VARYING_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHAR_VARYING_COL NATIONAL CHAR VARYING(255)); +INSERT INTO T_RZFA_NATIONAL_CHAR_VARYING_TABLE (ID, NATIONAL_CHAR_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NUMBER_TABLE (ID NUMBER PRIMARY KEY, NUMBER_COL NUMBER); +INSERT INTO T_RZFA_NUMBER_TABLE (ID, NUMBER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NUMERIC_TABLE (ID NUMBER PRIMARY KEY, NUMERIC_COL NUMERIC); +INSERT INTO T_RZFA_NUMERIC_TABLE (ID, NUMERIC_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DECIMAL_TABLE (ID NUMBER PRIMARY KEY, DECIMAL_COL DECIMAL); +INSERT INTO T_RZFA_DECIMAL_TABLE (ID, DECIMAL_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DEC_TABLE (ID NUMBER PRIMARY KEY, DEC_COL DEC); +INSERT INTO T_RZFA_DEC_TABLE (ID, DEC_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_FLOAT_TABLE (ID NUMBER PRIMARY KEY, FLOAT_COL FLOAT); +INSERT INTO T_RZFA_FLOAT_TABLE (ID, FLOAT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DOUBLE_PRECISION_TABLE (ID NUMBER PRIMARY KEY, DOUBLE_PRECISION_COL DOUBLE PRECISION); +INSERT INTO T_RZFA_DOUBLE_PRECISION_TABLE (ID, DOUBLE_PRECISION_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_REAL_TABLE (ID NUMBER PRIMARY KEY, REAL_COL REAL); +INSERT INTO T_RZFA_REAL_TABLE (ID, REAL_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BINARY_FLOAT_TABLE (ID NUMBER PRIMARY KEY, BINARY_FLOAT_COL BINARY_FLOAT); +INSERT INTO T_RZFA_BINARY_FLOAT_TABLE (ID, BINARY_FLOAT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BINARY_DOUBLE_TABLE (ID NUMBER PRIMARY KEY, BINARY_DOUBLE_COL BINARY_DOUBLE); +INSERT INTO T_RZFA_BINARY_DOUBLE_TABLE (ID, BINARY_DOUBLE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INTEGER_TABLE (ID NUMBER PRIMARY KEY, INTEGER_COL INTEGER); +INSERT INTO T_RZFA_INTEGER_TABLE (ID, INTEGER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INT_TABLE (ID NUMBER PRIMARY KEY, INT_COL INT); +INSERT INTO T_RZFA_INT_TABLE (ID, INT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_SMALLINT_TABLE (ID NUMBER PRIMARY KEY, SMALLINT_COL SMALLINT); +INSERT INTO T_RZFA_SMALLINT_TABLE (ID, SMALLINT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DATE_TABLE (ID NUMBER PRIMARY KEY, DATE_COL DATE); +INSERT INTO T_RZFA_DATE_TABLE (ID, DATE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_COL TIMESTAMP); +INSERT INTO T_RZFA_TIMESTAMP_TABLE (ID, TIMESTAMP_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_WITH_TIME_ZONE_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_WITH_TIME_ZONE_COL TIMESTAMP WITH TIME ZONE); +INSERT INTO T_RZFA_TIMESTAMP_WITH_TIME_ZONE_TABLE (ID, TIMESTAMP_WITH_TIME_ZONE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_WITH_LOCAL_TIME_ZONE_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_WITH_LOCAL_TIME_ZONE_COL TIMESTAMP WITH LOCAL TIME ZONE); +INSERT INTO T_RZFA_TIMESTAMP_WITH_LOCAL_TIME_ZONE_TABLE (ID, TIMESTAMP_WITH_LOCAL_TIME_ZONE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INTERVAL_YEAR_TO_MONTH_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_INTERVAL_YEAR_TO_MONTH_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_INTERVAL_DAY_TO_SECOND_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_INTERVAL_DAY_TO_SECOND_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_RAW_TABLE (ID NUMBER PRIMARY KEY, RAW_COL RAW(255)); +INSERT INTO T_RZFA_RAW_TABLE (ID, RAW_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_LONG_RAW_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_LONG_RAW_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_BLOB_TABLE (ID NUMBER PRIMARY KEY, BLOB_COL BLOB); +INSERT INTO T_RZFA_BLOB_TABLE (ID, BLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CLOB_TABLE (ID NUMBER PRIMARY KEY, CLOB_COL CLOB); +INSERT INTO T_RZFA_CLOB_TABLE (ID, CLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCLOB_TABLE (ID NUMBER PRIMARY KEY, NCLOB_COL NCLOB); +INSERT INTO T_RZFA_NCLOB_TABLE (ID, NCLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BFILE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_BFILE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_LONG_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_LONG_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ROWID_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ROWID_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_UROWID_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_UROWID_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_BOOLEAN_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_BOOLEAN_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_JSON_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_JSON_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_XMLTYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_XMLTYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_GEOMETRY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_GEOMETRY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_TOPO_GEOMETRY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_TOPO_GEOMETRY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_GEORASTER_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_GEORASTER_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_VECTOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_VECTOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYDATA_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYDATA_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYTYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYTYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYDATASET_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYDATASET_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_URITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_URITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_DBURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_DBURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_XDBURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_XDBURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_HTTPURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_HTTPURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_EXPRESSION_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_EXPRESSION_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDAUDIO_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDAUDIO_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDIMAGE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDIMAGE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDVIDEO_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDVIDEO_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDDOC_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDDOC_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_STILLIMAGE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_STILLIMAGE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_COLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_COLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_AVERAGECOLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_AVERAGECOLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_COLORHISTOGRAM_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_COLORHISTOGRAM_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_POSITIONALCOLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_POSITIONALCOLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_TEXTURE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_TEXTURE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_FEATURELIST_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_FEATURELIST_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_MLSLABEL_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_MLSLABEL_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_REF_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_REF_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_REF_CURSOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_REF_CURSOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_VARRAY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_VARRAY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_NESTED_TABLE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_NESTED_TABLE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ASSOCIATIVE_ARRAY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ASSOCIATIVE_ARRAY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_OBJECT_TYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_OBJECT_TYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql new file mode 100644 index 0000000000..3c8cf1eee1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..7c5c7a9e65 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,181 @@ +CREATE TABLE `LargeKey` ( + `pk_col1` STRING(255) NOT NULL , + `pk_col2` STRING(255) NOT NULL , + `pk_col3` STRING(255) NOT NULL , + `col1` STRING(255), + `col2` STRING(255), + `col3` STRING(255), + `value_col` STRING(MAX), +) PRIMARY KEY (`pk_col1`, `pk_col2`, `pk_col3`); + +CREATE INDEX `large_index` ON `LargeKey` (`col1`, `col2`, `col3`); + +CREATE TABLE `LargeCell` ( + `id` INT64 NOT NULL , + `max_string_col_to_bytes` BYTES(MAX), + `max_string_col_to_str` STRING(MAX), +) PRIMARY KEY (`id`); + +CREATE TABLE `WideRow` ( + `id` INT64 NOT NULL , + `col1` BYTES(MAX), + `col2` BYTES(MAX), + `col3` BYTES(MAX), + `col4` BYTES(MAX), + `col5` BYTES(MAX), + `col6` BYTES(MAX), + `col7` BYTES(MAX), + `col8` BYTES(MAX), + `col9` BYTES(MAX), + `col10` BYTES(MAX), + `col11` BYTES(MAX), + `col12` BYTES(MAX), + `col13` BYTES(MAX), + `col14` BYTES(MAX), + `col15` BYTES(MAX), + `col16` BYTES(MAX), + `col17` BYTES(MAX), + `col18` BYTES(MAX), + `col19` BYTES(MAX), + `col20` BYTES(MAX), + `col21` BYTES(MAX), + `col22` BYTES(MAX), + `col23` BYTES(MAX), + `col24` BYTES(MAX), + `col25` BYTES(MAX), + `col26` BYTES(MAX), + `col27` BYTES(MAX), + `col28` BYTES(MAX), + `col29` BYTES(MAX), + `col30` BYTES(MAX), + `col31` BYTES(MAX), + `col32` BYTES(MAX), + `col33` BYTES(MAX), + `col34` BYTES(MAX), + `col35` BYTES(MAX), + `col36` BYTES(MAX), + `col37` BYTES(MAX), + `col38` BYTES(MAX), + `col39` BYTES(MAX), + `col40` BYTES(MAX), + `col41` BYTES(MAX), + `col42` BYTES(MAX), + `col43` BYTES(MAX), + `col44` BYTES(MAX), + `col45` BYTES(MAX), + `col46` BYTES(MAX), + `col47` BYTES(MAX), + `col48` BYTES(MAX), + `col49` BYTES(MAX), + `col50` BYTES(MAX), + `col51` BYTES(MAX), + `col52` BYTES(MAX), + `col53` BYTES(MAX), + `col54` BYTES(MAX), + `col55` BYTES(MAX), + `col56` BYTES(MAX), + `col57` BYTES(MAX), + `col58` BYTES(MAX), + `col59` BYTES(MAX), + `col60` BYTES(MAX), + `col61` BYTES(MAX), + `col62` BYTES(MAX), + `col63` BYTES(MAX), + `col64` BYTES(MAX), + `col65` BYTES(MAX), + `col66` BYTES(MAX), + `col67` BYTES(MAX), + `col68` BYTES(MAX), + `col69` BYTES(MAX), + `col70` BYTES(MAX), + `col71` BYTES(MAX), + `col72` BYTES(MAX), + `col73` BYTES(MAX), + `col74` BYTES(MAX), + `col75` BYTES(MAX), + `col76` BYTES(MAX), + `col77` BYTES(MAX), + `col78` BYTES(MAX), + `col79` BYTES(MAX), + `col80` BYTES(MAX), + `col81` BYTES(MAX), + `col82` BYTES(MAX), + `col83` BYTES(MAX), + `col84` BYTES(MAX), + `col85` BYTES(MAX), + `col86` BYTES(MAX), + `col87` BYTES(MAX), + `col88` BYTES(MAX), + `col89` BYTES(MAX), + `col90` BYTES(MAX), + `col91` BYTES(MAX), + `col92` BYTES(MAX), + `col93` BYTES(MAX), + `col94` BYTES(MAX), + `col95` BYTES(MAX), + `col96` BYTES(MAX), + `col97` BYTES(MAX), + `col98` BYTES(MAX), + `col99` BYTES(MAX), + `col100` BYTES(MAX), + `col101` BYTES(MAX), + `col102` BYTES(MAX), + `col103` BYTES(MAX), + `col104` BYTES(MAX), + `col105` BYTES(MAX), + `col106` BYTES(MAX), + `col107` BYTES(MAX), + `col108` BYTES(MAX), + `col109` BYTES(MAX), + `col110` BYTES(MAX), + `col111` BYTES(MAX), + `col112` BYTES(MAX), + `col113` BYTES(MAX), + `col114` BYTES(MAX), + `col115` BYTES(MAX), + `col116` BYTES(MAX), + `col117` BYTES(MAX), + `col118` BYTES(MAX), + `col119` BYTES(MAX), + `col120` BYTES(MAX), + `col121` BYTES(MAX), + `col122` BYTES(MAX), + `col123` BYTES(MAX), + `col124` BYTES(MAX), + `col125` BYTES(MAX), + `col126` BYTES(MAX), + `col127` BYTES(MAX), + `col128` BYTES(MAX), + `col129` BYTES(MAX), + `col130` BYTES(MAX), + `col131` BYTES(MAX), + `col132` BYTES(MAX), + `col133` BYTES(MAX), + `col134` BYTES(MAX), + `col135` BYTES(MAX), + `col136` BYTES(MAX), + `col137` BYTES(MAX), + `col138` BYTES(MAX), + `col139` BYTES(MAX), + `col140` BYTES(MAX), + `col141` BYTES(MAX), + `col142` BYTES(MAX), + `col143` BYTES(MAX), + `col144` BYTES(MAX), + `col145` BYTES(MAX), + `col146` BYTES(MAX), + `col147` BYTES(MAX), + `col148` BYTES(MAX), + `col149` BYTES(MAX), + `col150` BYTES(MAX), + `col151` BYTES(MAX), + `col152` BYTES(MAX), + `col153` BYTES(MAX), + `col154` BYTES(MAX), + `col155` BYTES(MAX), + `col156` BYTES(MAX), + `col157` BYTES(MAX), + `col158` BYTES(MAX), + `col159` BYTES(MAX), + `col160` BYTES(MAX), +) PRIMARY KEY (`id`); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql new file mode 100644 index 0000000000..e18c79f955 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql @@ -0,0 +1,28 @@ +CREATE TABLE "LargeKey" ( + "pk_col1" VARCHAR2(255) NOT NULL, + "pk_col2" VARCHAR2(255) NOT NULL, + "pk_col3" VARCHAR2(255) NOT NULL, + "col1" VARCHAR2(255), + "col2" VARCHAR2(255), + "col3" VARCHAR2(255), + "value_col" CLOB, + PRIMARY KEY ("pk_col1", "pk_col2", "pk_col3") +); + +CREATE INDEX "large_index" ON "LargeKey" ("col1", "col2", "col3"); + +INSERT INTO "LargeKey" VALUES ( + RPAD('A', 255, 'A'), + RPAD('B', 255, 'B'), + RPAD('C', 255, 'C'), + RPAD('A', 255, 'A'), + RPAD('B', 255, 'B'), + RPAD('C', 255, 'C'), + '3072 bytes of total size of table key as per the mysql limitation' +); + +CREATE TABLE "LargeCell" ( + "id" NUMBER PRIMARY KEY, + "max_string_col_to_bytes" BLOB, + "max_string_col_to_str" CLOB +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json new file mode 100644 index 0000000000..d613fee945 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json @@ -0,0 +1,11391 @@ +{ + "SpSchema": { + "t1": { + "Name": "LargeKey", + "ColIds": [ + "c2", + "c3", + "c4", + "c5", + "c6", + "c7", + "c8" + ], + "ShardIdColumn": "", + "ColDefs": { + "c2": { + "Name": "pk_col1", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col1 varchar(255)", + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c3": { + "Name": "pk_col2", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col2 varchar(255)", + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c4": { + "Name": "pk_col3", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col3 varchar(255)", + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c5": { + "Name": "col1", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col1 varchar(255)", + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c6": { + "Name": "col2", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col2 varchar(255)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c7": { + "Name": "col3", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col3 varchar(255)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c8": { + "Name": "value_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: value_col text", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c3", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c4", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "Indexes": [ + { + "Name": "large_index", + "TableId": "t1", + "Unique": false, + "Keys": [ + { + "ColId": "c5", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 3 + } + ], + "Id": "i9", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table LargeKey", + "Id": "t1" + }, + "t10": { + "Name": "LargeCell", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ShardIdColumn": "", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c12": { + "Name": "max_string_col_to_bytes", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: max_string_col_to_bytes longtext", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c13": { + "Name": "max_string_col_to_str", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: max_string_col_to_str mediumtext", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table LargeCell", + "Id": "t10" + }, + "t14": { + "Name": "WideRow", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53", + "c54", + "c55", + "c56", + "c57", + "c58", + "c59", + "c60", + "c61", + "c62", + "c63", + "c64", + "c65", + "c66", + "c67", + "c68", + "c69", + "c70", + "c71", + "c72", + "c73", + "c74", + "c75", + "c76", + "c77", + "c78", + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107", + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135", + "c136", + "c137", + "c138", + "c139", + "c140", + "c141", + "c142", + "c143", + "c144", + "c145", + "c146", + "c147", + "c148", + "c149", + "c150", + "c151", + "c152", + "c153", + "c154", + "c155", + "c156", + "c157", + "c158", + "c159", + "c160", + "c161", + "c162", + "c163", + "c164", + "c165", + "c166", + "c167", + "c168", + "c169", + "c170", + "c171", + "c172", + "c173", + "c174", + "c175" + ], + "ShardIdColumn": "", + "ColDefs": { + "c100": { + "Name": "col85", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col85 mediumtext", + "Id": "c100", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c101": { + "Name": "col86", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col86 mediumtext", + "Id": "c101", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c102": { + "Name": "col87", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col87 mediumtext", + "Id": "c102", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c103": { + "Name": "col88", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col88 mediumtext", + "Id": "c103", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c104": { + "Name": "col89", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col89 mediumtext", + "Id": "c104", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c105": { + "Name": "col90", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col90 mediumtext", + "Id": "c105", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c106": { + "Name": "col91", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col91 mediumtext", + "Id": "c106", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c107": { + "Name": "col92", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col92 mediumtext", + "Id": "c107", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c108": { + "Name": "col93", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col93 mediumtext", + "Id": "c108", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c109": { + "Name": "col94", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col94 mediumtext", + "Id": "c109", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c110": { + "Name": "col95", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col95 mediumtext", + "Id": "c110", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c111": { + "Name": "col96", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col96 mediumtext", + "Id": "c111", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c112": { + "Name": "col97", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col97 mediumtext", + "Id": "c112", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c113": { + "Name": "col98", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col98 mediumtext", + "Id": "c113", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c114": { + "Name": "col99", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col99 mediumtext", + "Id": "c114", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c115": { + "Name": "col100", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col100 mediumtext", + "Id": "c115", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c116": { + "Name": "col101", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col101 mediumtext", + "Id": "c116", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c117": { + "Name": "col102", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col102 mediumtext", + "Id": "c117", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c118": { + "Name": "col103", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col103 mediumtext", + "Id": "c118", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c119": { + "Name": "col104", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col104 mediumtext", + "Id": "c119", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c120": { + "Name": "col105", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col105 mediumtext", + "Id": "c120", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c121": { + "Name": "col106", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col106 mediumtext", + "Id": "c121", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c122": { + "Name": "col107", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col107 mediumtext", + "Id": "c122", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c123": { + "Name": "col108", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col108 mediumtext", + "Id": "c123", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c124": { + "Name": "col109", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col109 mediumtext", + "Id": "c124", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c125": { + "Name": "col110", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col110 mediumtext", + "Id": "c125", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c126": { + "Name": "col111", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col111 mediumtext", + "Id": "c126", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c127": { + "Name": "col112", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col112 mediumtext", + "Id": "c127", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c128": { + "Name": "col113", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col113 mediumtext", + "Id": "c128", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c129": { + "Name": "col114", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col114 mediumtext", + "Id": "c129", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c130": { + "Name": "col115", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col115 mediumtext", + "Id": "c130", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c131": { + "Name": "col116", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col116 mediumtext", + "Id": "c131", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c132": { + "Name": "col117", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col117 mediumtext", + "Id": "c132", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c133": { + "Name": "col118", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col118 mediumtext", + "Id": "c133", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c134": { + "Name": "col119", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col119 mediumtext", + "Id": "c134", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c135": { + "Name": "col120", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col120 mediumtext", + "Id": "c135", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c136": { + "Name": "col121", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col121 mediumtext", + "Id": "c136", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c137": { + "Name": "col122", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col122 mediumtext", + "Id": "c137", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c138": { + "Name": "col123", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col123 mediumtext", + "Id": "c138", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c139": { + "Name": "col124", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col124 mediumtext", + "Id": "c139", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c140": { + "Name": "col125", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col125 mediumtext", + "Id": "c140", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c141": { + "Name": "col126", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col126 mediumtext", + "Id": "c141", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c142": { + "Name": "col127", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col127 mediumtext", + "Id": "c142", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c143": { + "Name": "col128", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col128 mediumtext", + "Id": "c143", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c144": { + "Name": "col129", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col129 mediumtext", + "Id": "c144", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c145": { + "Name": "col130", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col130 mediumtext", + "Id": "c145", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c146": { + "Name": "col131", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col131 mediumtext", + "Id": "c146", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c147": { + "Name": "col132", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col132 mediumtext", + "Id": "c147", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c148": { + "Name": "col133", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col133 mediumtext", + "Id": "c148", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c149": { + "Name": "col134", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col134 mediumtext", + "Id": "c149", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c15": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c150": { + "Name": "col135", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col135 mediumtext", + "Id": "c150", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c151": { + "Name": "col136", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col136 mediumtext", + "Id": "c151", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c152": { + "Name": "col137", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col137 mediumtext", + "Id": "c152", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c153": { + "Name": "col138", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col138 mediumtext", + "Id": "c153", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c154": { + "Name": "col139", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col139 mediumtext", + "Id": "c154", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c155": { + "Name": "col140", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col140 mediumtext", + "Id": "c155", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c156": { + "Name": "col141", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col141 mediumtext", + "Id": "c156", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c157": { + "Name": "col142", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col142 mediumtext", + "Id": "c157", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c158": { + "Name": "col143", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col143 mediumtext", + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c159": { + "Name": "col144", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col144 mediumtext", + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c16": { + "Name": "col1", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col1 mediumtext", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c160": { + "Name": "col145", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col145 mediumtext", + "Id": "c160", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c161": { + "Name": "col146", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col146 mediumtext", + "Id": "c161", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c162": { + "Name": "col147", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col147 mediumtext", + "Id": "c162", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c163": { + "Name": "col148", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col148 mediumtext", + "Id": "c163", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c164": { + "Name": "col149", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col149 mediumtext", + "Id": "c164", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c165": { + "Name": "col150", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col150 mediumtext", + "Id": "c165", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c166": { + "Name": "col151", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col151 mediumtext", + "Id": "c166", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c167": { + "Name": "col152", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col152 mediumtext", + "Id": "c167", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c168": { + "Name": "col153", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col153 mediumtext", + "Id": "c168", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c169": { + "Name": "col154", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col154 mediumtext", + "Id": "c169", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c17": { + "Name": "col2", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col2 mediumtext", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c170": { + "Name": "col155", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col155 mediumtext", + "Id": "c170", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c171": { + "Name": "col156", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col156 mediumtext", + "Id": "c171", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c172": { + "Name": "col157", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col157 mediumtext", + "Id": "c172", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c173": { + "Name": "col158", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col158 mediumtext", + "Id": "c173", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c174": { + "Name": "col159", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col159 mediumtext", + "Id": "c174", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c175": { + "Name": "col160", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col160 mediumtext", + "Id": "c175", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c18": { + "Name": "col3", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col3 mediumtext", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c19": { + "Name": "col4", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col4 mediumtext", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c20": { + "Name": "col5", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col5 mediumtext", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c21": { + "Name": "col6", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col6 mediumtext", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c22": { + "Name": "col7", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col7 mediumtext", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c23": { + "Name": "col8", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col8 mediumtext", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c24": { + "Name": "col9", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col9 mediumtext", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c25": { + "Name": "col10", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col10 mediumtext", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c26": { + "Name": "col11", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col11 mediumtext", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c27": { + "Name": "col12", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col12 mediumtext", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c28": { + "Name": "col13", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col13 mediumtext", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c29": { + "Name": "col14", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col14 mediumtext", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c30": { + "Name": "col15", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col15 mediumtext", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c31": { + "Name": "col16", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col16 mediumtext", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c32": { + "Name": "col17", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col17 mediumtext", + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c33": { + "Name": "col18", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col18 mediumtext", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c34": { + "Name": "col19", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col19 mediumtext", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c35": { + "Name": "col20", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col20 mediumtext", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c36": { + "Name": "col21", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col21 mediumtext", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c37": { + "Name": "col22", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col22 mediumtext", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c38": { + "Name": "col23", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col23 mediumtext", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c39": { + "Name": "col24", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col24 mediumtext", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c40": { + "Name": "col25", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col25 mediumtext", + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c41": { + "Name": "col26", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col26 mediumtext", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c42": { + "Name": "col27", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col27 mediumtext", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c43": { + "Name": "col28", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col28 mediumtext", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c44": { + "Name": "col29", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col29 mediumtext", + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c45": { + "Name": "col30", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col30 mediumtext", + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c46": { + "Name": "col31", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col31 mediumtext", + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c47": { + "Name": "col32", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col32 mediumtext", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c48": { + "Name": "col33", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col33 mediumtext", + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c49": { + "Name": "col34", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col34 mediumtext", + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c50": { + "Name": "col35", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col35 mediumtext", + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c51": { + "Name": "col36", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col36 mediumtext", + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c52": { + "Name": "col37", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col37 mediumtext", + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c53": { + "Name": "col38", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col38 mediumtext", + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c54": { + "Name": "col39", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col39 mediumtext", + "Id": "c54", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c55": { + "Name": "col40", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col40 mediumtext", + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c56": { + "Name": "col41", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col41 mediumtext", + "Id": "c56", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c57": { + "Name": "col42", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col42 mediumtext", + "Id": "c57", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c58": { + "Name": "col43", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col43 mediumtext", + "Id": "c58", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c59": { + "Name": "col44", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col44 mediumtext", + "Id": "c59", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c60": { + "Name": "col45", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col45 mediumtext", + "Id": "c60", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c61": { + "Name": "col46", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col46 mediumtext", + "Id": "c61", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c62": { + "Name": "col47", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col47 mediumtext", + "Id": "c62", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c63": { + "Name": "col48", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col48 mediumtext", + "Id": "c63", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c64": { + "Name": "col49", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col49 mediumtext", + "Id": "c64", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c65": { + "Name": "col50", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col50 mediumtext", + "Id": "c65", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c66": { + "Name": "col51", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col51 mediumtext", + "Id": "c66", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c67": { + "Name": "col52", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col52 mediumtext", + "Id": "c67", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c68": { + "Name": "col53", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col53 mediumtext", + "Id": "c68", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c69": { + "Name": "col54", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col54 mediumtext", + "Id": "c69", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c70": { + "Name": "col55", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col55 mediumtext", + "Id": "c70", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c71": { + "Name": "col56", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col56 mediumtext", + "Id": "c71", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c72": { + "Name": "col57", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col57 mediumtext", + "Id": "c72", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c73": { + "Name": "col58", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col58 mediumtext", + "Id": "c73", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c74": { + "Name": "col59", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col59 mediumtext", + "Id": "c74", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c75": { + "Name": "col60", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col60 mediumtext", + "Id": "c75", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c76": { + "Name": "col61", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col61 mediumtext", + "Id": "c76", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c77": { + "Name": "col62", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col62 mediumtext", + "Id": "c77", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c78": { + "Name": "col63", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col63 mediumtext", + "Id": "c78", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c79": { + "Name": "col64", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col64 mediumtext", + "Id": "c79", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c80": { + "Name": "col65", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col65 mediumtext", + "Id": "c80", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c81": { + "Name": "col66", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col66 mediumtext", + "Id": "c81", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c82": { + "Name": "col67", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col67 mediumtext", + "Id": "c82", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c83": { + "Name": "col68", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col68 mediumtext", + "Id": "c83", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c84": { + "Name": "col69", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col69 mediumtext", + "Id": "c84", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c85": { + "Name": "col70", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col70 mediumtext", + "Id": "c85", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c86": { + "Name": "col71", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col71 mediumtext", + "Id": "c86", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c87": { + "Name": "col72", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col72 mediumtext", + "Id": "c87", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c88": { + "Name": "col73", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col73 mediumtext", + "Id": "c88", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c89": { + "Name": "col74", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col74 mediumtext", + "Id": "c89", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c90": { + "Name": "col75", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col75 mediumtext", + "Id": "c90", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c91": { + "Name": "col76", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col76 mediumtext", + "Id": "c91", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c92": { + "Name": "col77", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col77 mediumtext", + "Id": "c92", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c93": { + "Name": "col78", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col78 mediumtext", + "Id": "c93", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c94": { + "Name": "col79", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col79 mediumtext", + "Id": "c94", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c95": { + "Name": "col80", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col80 mediumtext", + "Id": "c95", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c96": { + "Name": "col81", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col81 mediumtext", + "Id": "c96", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c97": { + "Name": "col82", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col82 mediumtext", + "Id": "c97", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c98": { + "Name": "col83", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col83 mediumtext", + "Id": "c98", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c99": { + "Name": "col84", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col84 mediumtext", + "Id": "c99", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table WideRow", + "Id": "t14" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "LargeKey", + "Schema": "", + "ColIds": [ + "c2", + "c3", + "c4", + "c5", + "c6", + "c7", + "c8" + ], + "ColDefs": { + "c2": { + "Name": "pk_col1", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c3": { + "Name": "pk_col2", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c4": { + "Name": "pk_col3", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c5": { + "Name": "col1", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c6": { + "Name": "col2", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c7": { + "Name": "col3", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c8": { + "Name": "value_col", + "Type": { + "Name": "text", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c3", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c4", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": [ + { + "Name": "large_index", + "Unique": false, + "Keys": [ + { + "ColId": "c5", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 3 + } + ], + "Id": "i9", + "StoredColumnIds": null + } + ], + "Id": "t1" + }, + "t10": { + "Name": "LargeCell", + "Schema": "", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c12": { + "Name": "max_string_col_to_bytes", + "Type": { + "Name": "longtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c13": { + "Name": "max_string_col_to_str", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t10" + }, + "t14": { + "Name": "WideRow", + "Schema": "", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53", + "c54", + "c55", + "c56", + "c57", + "c58", + "c59", + "c60", + "c61", + "c62", + "c63", + "c64", + "c65", + "c66", + "c67", + "c68", + "c69", + "c70", + "c71", + "c72", + "c73", + "c74", + "c75", + "c76", + "c77", + "c78", + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107", + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135", + "c136", + "c137", + "c138", + "c139", + "c140", + "c141", + "c142", + "c143", + "c144", + "c145", + "c146", + "c147", + "c148", + "c149", + "c150", + "c151", + "c152", + "c153", + "c154", + "c155", + "c156", + "c157", + "c158", + "c159", + "c160", + "c161", + "c162", + "c163", + "c164", + "c165", + "c166", + "c167", + "c168", + "c169", + "c170", + "c171", + "c172", + "c173", + "c174", + "c175" + ], + "ColDefs": { + "c100": { + "Name": "col85", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c100", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c101": { + "Name": "col86", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c101", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c102": { + "Name": "col87", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c102", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c103": { + "Name": "col88", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c103", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c104": { + "Name": "col89", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c104", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c105": { + "Name": "col90", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c105", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c106": { + "Name": "col91", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c106", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c107": { + "Name": "col92", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c107", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c108": { + "Name": "col93", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c108", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c109": { + "Name": "col94", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c109", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c110": { + "Name": "col95", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c110", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c111": { + "Name": "col96", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c111", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c112": { + "Name": "col97", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c112", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c113": { + "Name": "col98", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c113", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c114": { + "Name": "col99", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c114", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c115": { + "Name": "col100", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c115", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c116": { + "Name": "col101", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c116", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c117": { + "Name": "col102", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c117", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c118": { + "Name": "col103", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c118", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c119": { + "Name": "col104", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c119", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c120": { + "Name": "col105", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c120", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c121": { + "Name": "col106", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c121", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c122": { + "Name": "col107", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c122", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c123": { + "Name": "col108", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c123", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c124": { + "Name": "col109", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c124", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c125": { + "Name": "col110", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c125", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c126": { + "Name": "col111", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c126", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c127": { + "Name": "col112", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c127", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c128": { + "Name": "col113", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c128", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c129": { + "Name": "col114", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c129", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c130": { + "Name": "col115", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c130", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c131": { + "Name": "col116", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c131", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c132": { + "Name": "col117", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c132", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c133": { + "Name": "col118", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c133", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c134": { + "Name": "col119", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c134", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c135": { + "Name": "col120", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c135", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c136": { + "Name": "col121", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c136", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c137": { + "Name": "col122", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c137", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c138": { + "Name": "col123", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c138", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c139": { + "Name": "col124", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c139", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c140": { + "Name": "col125", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c140", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c141": { + "Name": "col126", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c141", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c142": { + "Name": "col127", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c142", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c143": { + "Name": "col128", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c143", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c144": { + "Name": "col129", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c144", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c145": { + "Name": "col130", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c145", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c146": { + "Name": "col131", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c146", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c147": { + "Name": "col132", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c147", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c148": { + "Name": "col133", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c148", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c149": { + "Name": "col134", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c149", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c15": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c150": { + "Name": "col135", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c150", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c151": { + "Name": "col136", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c151", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c152": { + "Name": "col137", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c152", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c153": { + "Name": "col138", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c153", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c154": { + "Name": "col139", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c154", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c155": { + "Name": "col140", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c155", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c156": { + "Name": "col141", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c156", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c157": { + "Name": "col142", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c157", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c158": { + "Name": "col143", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c159": { + "Name": "col144", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c16": { + "Name": "col1", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c160": { + "Name": "col145", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c160", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c161": { + "Name": "col146", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c161", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c162": { + "Name": "col147", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c162", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c163": { + "Name": "col148", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c163", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c164": { + "Name": "col149", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c164", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c165": { + "Name": "col150", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c165", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c166": { + "Name": "col151", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c166", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c167": { + "Name": "col152", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c167", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c168": { + "Name": "col153", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c168", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c169": { + "Name": "col154", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c169", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c17": { + "Name": "col2", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c170": { + "Name": "col155", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c170", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c171": { + "Name": "col156", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c171", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c172": { + "Name": "col157", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c172", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c173": { + "Name": "col158", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c173", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c174": { + "Name": "col159", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c174", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c175": { + "Name": "col160", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c175", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c18": { + "Name": "col3", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c19": { + "Name": "col4", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c20": { + "Name": "col5", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c21": { + "Name": "col6", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c22": { + "Name": "col7", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c23": { + "Name": "col8", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c24": { + "Name": "col9", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c25": { + "Name": "col10", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c26": { + "Name": "col11", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c27": { + "Name": "col12", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c28": { + "Name": "col13", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c29": { + "Name": "col14", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c30": { + "Name": "col15", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c31": { + "Name": "col16", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c32": { + "Name": "col17", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c33": { + "Name": "col18", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c34": { + "Name": "col19", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c35": { + "Name": "col20", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c36": { + "Name": "col21", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c37": { + "Name": "col22", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c38": { + "Name": "col23", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c39": { + "Name": "col24", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c40": { + "Name": "col25", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c41": { + "Name": "col26", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c42": { + "Name": "col27", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c43": { + "Name": "col28", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c44": { + "Name": "col29", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c45": { + "Name": "col30", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c46": { + "Name": "col31", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c47": { + "Name": "col32", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c48": { + "Name": "col33", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c49": { + "Name": "col34", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c50": { + "Name": "col35", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c51": { + "Name": "col36", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c52": { + "Name": "col37", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c53": { + "Name": "col38", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c54": { + "Name": "col39", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c54", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c55": { + "Name": "col40", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c56": { + "Name": "col41", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c56", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c57": { + "Name": "col42", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c57", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c58": { + "Name": "col43", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c58", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c59": { + "Name": "col44", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c59", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c60": { + "Name": "col45", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c60", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c61": { + "Name": "col46", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c61", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c62": { + "Name": "col47", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c62", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c63": { + "Name": "col48", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c63", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c64": { + "Name": "col49", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c64", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c65": { + "Name": "col50", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c65", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c66": { + "Name": "col51", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c66", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c67": { + "Name": "col52", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c67", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c68": { + "Name": "col53", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c68", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c69": { + "Name": "col54", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c69", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c70": { + "Name": "col55", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c70", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c71": { + "Name": "col56", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c71", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c72": { + "Name": "col57", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c72", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c73": { + "Name": "col58", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c73", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c74": { + "Name": "col59", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c74", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c75": { + "Name": "col60", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c75", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c76": { + "Name": "col61", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c76", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c77": { + "Name": "col62", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c77", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c78": { + "Name": "col63", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c78", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c79": { + "Name": "col64", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c79", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c80": { + "Name": "col65", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c80", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c81": { + "Name": "col66", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c81", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c82": { + "Name": "col67", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c82", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c83": { + "Name": "col68", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c83", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c84": { + "Name": "col69", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c84", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c85": { + "Name": "col70", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c85", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c86": { + "Name": "col71", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c86", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c87": { + "Name": "col72", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c87", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c88": { + "Name": "col73", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c88", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c89": { + "Name": "col74", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c89", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c90": { + "Name": "col75", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c90", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c91": { + "Name": "col76", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c91", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c92": { + "Name": "col77", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c92", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c93": { + "Name": "col78", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c93", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c94": { + "Name": "col79", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c94", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c95": { + "Name": "col80", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c95", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c96": { + "Name": "col81", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c96", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c97": { + "Name": "col82", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c97", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c98": { + "Name": "col83", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c98", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c99": { + "Name": "col84", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c99", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t14" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + }, + "t10": { + "ColumnLevelIssues": { + "c11": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t14": { + "ColumnLevelIssues": { + "c15": [ + 14 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "ToSpanner": { + "LargeKey": { + "Name": "LargeKey", + "Cols": { + "col1": "col1", + "col2": "col2", + "col3": "col3", + "pk_col1": "pk_col1", + "pk_col2": "pk_col2", + "pk_col3": "pk_col3", + "value_col": "value_col" + } + }, + "LargeCell": { + "Name": "LargeCell", + "Cols": { + "id": "id", + "max_string_col_to_bytes": "max_string_col_to_bytes", + "max_string_col_to_str": "max_string_col_to_str" + } + }, + "WideRow": { + "Name": "WideRow", + "Cols": { + "col1": "col1", + "col10": "col10", + "col100": "col100", + "col101": "col101", + "col102": "col102", + "col103": "col103", + "col104": "col104", + "col105": "col105", + "col106": "col106", + "col107": "col107", + "col108": "col108", + "col109": "col109", + "col11": "col11", + "col110": "col110", + "col111": "col111", + "col112": "col112", + "col113": "col113", + "col114": "col114", + "col115": "col115", + "col116": "col116", + "col117": "col117", + "col118": "col118", + "col119": "col119", + "col12": "col12", + "col120": "col120", + "col121": "col121", + "col122": "col122", + "col123": "col123", + "col124": "col124", + "col125": "col125", + "col126": "col126", + "col127": "col127", + "col128": "col128", + "col129": "col129", + "col13": "col13", + "col130": "col130", + "col131": "col131", + "col132": "col132", + "col133": "col133", + "col134": "col134", + "col135": "col135", + "col136": "col136", + "col137": "col137", + "col138": "col138", + "col139": "col139", + "col14": "col14", + "col140": "col140", + "col141": "col141", + "col142": "col142", + "col143": "col143", + "col144": "col144", + "col145": "col145", + "col146": "col146", + "col147": "col147", + "col148": "col148", + "col149": "col149", + "col15": "col15", + "col150": "col150", + "col151": "col151", + "col152": "col152", + "col153": "col153", + "col154": "col154", + "col155": "col155", + "col156": "col156", + "col157": "col157", + "col158": "col158", + "col159": "col159", + "col16": "col16", + "col160": "col160", + "col17": "col17", + "col18": "col18", + "col19": "col19", + "col2": "col2", + "col20": "col20", + "col21": "col21", + "col22": "col22", + "col23": "col23", + "col24": "col24", + "col25": "col25", + "col26": "col26", + "col27": "col27", + "col28": "col28", + "col29": "col29", + "col3": "col3", + "col30": "col30", + "col31": "col31", + "col32": "col32", + "col33": "col33", + "col34": "col34", + "col35": "col35", + "col36": "col36", + "col37": "col37", + "col38": "col38", + "col39": "col39", + "col4": "col4", + "col40": "col40", + "col41": "col41", + "col42": "col42", + "col43": "col43", + "col44": "col44", + "col45": "col45", + "col46": "col46", + "col47": "col47", + "col48": "col48", + "col49": "col49", + "col5": "col5", + "col50": "col50", + "col51": "col51", + "col52": "col52", + "col53": "col53", + "col54": "col54", + "col55": "col55", + "col56": "col56", + "col57": "col57", + "col58": "col58", + "col59": "col59", + "col6": "col6", + "col60": "col60", + "col61": "col61", + "col62": "col62", + "col63": "col63", + "col64": "col64", + "col65": "col65", + "col66": "col66", + "col67": "col67", + "col68": "col68", + "col69": "col69", + "col7": "col7", + "col70": "col70", + "col71": "col71", + "col72": "col72", + "col73": "col73", + "col74": "col74", + "col75": "col75", + "col76": "col76", + "col77": "col77", + "col78": "col78", + "col79": "col79", + "col8": "col8", + "col80": "col80", + "col81": "col81", + "col82": "col82", + "col83": "col83", + "col84": "col84", + "col85": "col85", + "col86": "col86", + "col87": "col87", + "col88": "col88", + "col89": "col89", + "col9": "col9", + "col90": "col90", + "col91": "col91", + "col92": "col92", + "col93": "col93", + "col94": "col94", + "col95": "col95", + "col96": "col96", + "col97": "col97", + "col98": "col98", + "col99": "col99", + "id": "id" + } + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "Source": "mysqldump" +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..6b6168912a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,131 @@ +CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsWithSizes ( + varchar_column STRING(30) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(50), + bool_column BOOL, + binary_column BYTES(30), + varbinary_column BYTES(30), + bit_column BYTES(10), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsReducedSizes ( + varchar_column STRING(10) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(20), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS Users ( + user_id INT64 NOT NULL, + first_name STRING(50), + last_name STRING(50), + age INT64, + full_name STRING(100) AS (ARRAY_TO_STRING([first_name, last_name], " ")) STORED, +) PRIMARY KEY (user_id); + +CREATE TABLE IF NOT EXISTS Authors ( + id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (id); + +CREATE TABLE IF NOT EXISTS AllDatatypeTransformation ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + blob_column BYTES(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY (varchar_column); + +CREATE SEQUENCE sequence_id OPTIONS (sequence_kind='bit_reversed_positive', skip_range_min = 0, skip_range_max = 3); + +CREATE TABLE IF NOT EXISTS Singers ( + singer_id INT64 NOT NULL DEFAULT (GET_NEXT_SEQUENCE_VALUE(SEQUENCE sequence_id)) , + first_name STRING(1024), + last_name STRING(1024), +) PRIMARY KEY (singer_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE), + title STRING(200), +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql new file mode 100644 index 0000000000..fc1f4a088c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql @@ -0,0 +1,4 @@ +INSERT INTO "AllDatatypeColumns" +("varchar_column", "tinyint_column", "text_column", "date_column", "smallint_column", "mediumint_column", "int_column", "bigint_column", "float_column", "double_column", "decimal_column", "datetime_column", "timestamp_column", "time_column", "year_column", "char_column", "tinyblob_column", "tinytext_column", "blob_column", "mediumblob_column", "mediumtext_column", "longblob_column", "longtext_column", "enum_column", "bool_column", "other_bool_column", "binary_column", "varbinary_column", "bit_column") +VALUES +('value1', 10, 'text_content', TO_DATE('2024-02-08', 'YYYY-MM-DD'), 50, 1000, 50000, 987654321, 45.67, 123.789, 456.12, TO_TIMESTAMP('2024-02-08 08:15:30', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP_TZ('2024-02-08 08:15:30 UTC', 'YYYY-MM-DD HH24:MI:SS TZR'), '08:15:30', 2022, 'char_1 ', HEXTORAW('74696e79626c6f625f646174615f31'), 'tinytext', HEXTORAW('626c6f625f646174615f31'), HEXTORAW('6d656469756d626c6f625f646174615f31'), 'mediumtext', HEXTORAW('6c6f6e67626c6f625f646174615f31'), 'longtext_content', '2', 0, 1, HEXTORAW('62696e6172795f31000000000000000000000000'), HEXTORAW('76617262696e6172795f646174615f31'), HEXTORAW('0102')); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql new file mode 100644 index 0000000000..830bc4ba29 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql @@ -0,0 +1,156 @@ + +CREATE TABLE "AllDatatypeColumns" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "other_bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "AllDatatypeColumns2" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsWithSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsReducedSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Users" ( + "user_id" NUMBER NOT NULL, + "first_name" VARCHAR2(50), + "last_name" VARCHAR2(50), + "age" NUMBER, + PRIMARY KEY ("user_id") +); + +CREATE TABLE "Authors" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +CREATE TABLE "AllDatatypeTransformation" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "blob_column" BLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Singers" ( + "singer_id" NUMBER NOT NULL , + "first_name" VARCHAR2(1024), + "last_name" VARCHAR2(1024), + PRIMARY KEY ("singer_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL , + "title" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +ALTER TABLE "AllDatatypeColumns" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeColumns2" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsWithSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsReducedSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeTransformation" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Singers" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro new file mode 100644 index 0000000000..01421913ae Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Authors.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Authors.avro new file mode 100644 index 0000000000..0b050282ec Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Authors.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Books.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Books.avro new file mode 100644 index 0000000000..79c690553a Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Books.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..7af0944350 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,40 @@ +CREATE TABLE IF NOT EXISTS Movie ( + id INT64 NOT NULL, + name STRING(200), + actor NUMERIC, + startTime TIMESTAMP, +) PRIMARY KEY(id); + +CREATE TABLE IF NOT EXISTS Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + subscribed BOOL, + plan STRING(1), + startDate DATE +) PRIMARY KEY (id); + +CREATE TABLE IF NOT EXISTS Authors ( + author_id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (author_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, +) PRIMARY KEY (id); + +CREATE INDEX author_id_6 ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id); + +CREATE TABLE IF NOT EXISTS Articles ( + id INT64 NOT NULL, + name STRING(200), + published_date DATE, + author_id INT64 NOT NULL, +) PRIMARY KEY (author_id, id), +INTERLEAVE IN PARENT Authors; + +CREATE INDEX author_id ON Articles (author_id); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Movie.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Movie.avro new file mode 100644 index 0000000000..195a491cd8 Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Movie.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Users.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Users.avro new file mode 100644 index 0000000000..3996dd9a66 Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Users.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-cdc-Users.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-cdc-Users.avro new file mode 100644 index 0000000000..844e0ec20a Binary files /dev/null and b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-cdc-Users.avro differ diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql new file mode 100644 index 0000000000..76cdeafdbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql @@ -0,0 +1,52 @@ +CREATE TABLE "Movie" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "actor" NUMBER, + "startTime" TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + "subscribed" NUMBER(1), + "plan" CHAR(1), + "startDate" DATE, + PRIMARY KEY ("id") +); + +CREATE TABLE "Authors" ( + "author_id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Articles" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "published_date" DATE, + "author_id" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Articles" ADD FOREIGN KEY ("author_id") REFERENCES "Authors"("author_id"); + +CREATE INDEX author_id ON "Articles" ("author_id"); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD FOREIGN KEY ("author_id") REFERENCES "Authors"("author_id"); + +CREATE INDEX author_id_6 ON "Books" ("author_id"); + +ALTER TABLE "Movie" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Articles" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..71ee384cbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..fd61478626 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500), + "last_name1" VARCHAR2(500), + "first_name2" VARCHAR2(500), + "last_name2" VARCHAR2(500), + "first_name3" VARCHAR2(500), + "last_name3" VARCHAR2(500), + "ID" NUMBER NOT NULL PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json new file mode 100644 index 0000000000..c0b5fc0009 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json @@ -0,0 +1,10 @@ +{ + "renamedTables":{ + "person1":"human1" + }, + "renamedColumns":{ + "person1":{ + "first_name1":"name1" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..62f8aaa748 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS Authors ( + author_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY (author_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), +) PRIMARY KEY (synth_id); + +CREATE INDEX author_id ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id) ON DELETE NO ACTION; + +CREATE TABLE IF NOT EXISTS Genre ( + name STRING(200), + genre_id INT64 NOT NULL, +) PRIMARY KEY (genre_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql new file mode 100644 index 0000000000..f58350d4a9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql @@ -0,0 +1,26 @@ +CREATE TABLE "Authors" ( + "author_id" NUMBER(10,0) NOT NULL, + "name" VARCHAR2(25), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER(10,0) NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER(10,0), + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD CONSTRAINT "Books_ibfk_1" FOREIGN KEY ("author_id") REFERENCES "Authors" ("author_id"); + +CREATE TABLE "Genre" ( + "name" VARCHAR2(200), + "genre_id" NUMBER(10,0) NOT NULL, + PRIMARY KEY ("genre_id") +); + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Genre" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json new file mode 100644 index 0000000000..b7ff136a40 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json @@ -0,0 +1,506 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "e2e-it", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t34": { + "Name": "Authors", + "ColIds": [ + "c37", + "c38" + ], + "ShardIdColumn": "", + "ColDefs": { + "c37": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: author_id int(10)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Authors", + "Id": "t34" + }, + "t35": { + "Name": "Books", + "ColIds": [ + "c41", + "c42", + "c43", + "c47" + ], + "ShardIdColumn": "", + "ColDefs": { + "c41": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c47": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "Id": "f40", + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION" + } + ], + "Indexes": [ + { + "Name": "author_id", + "TableId": "t35", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t35" + } + }, + "SyntheticPKeys": { + "t35": { + "ColId": "c47", + "Sequence": 0 + } + }, + "SrcSchema": { + "t34": { + "Name": "Authors", + "Schema": "e2e-it", + "ColIds": [ + "c37", + "c38", + "c39" + ], + "ColDefs": { + "c37": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c39": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t34" + }, + "t35": { + "Name": "Books", + "Schema": "e2e-it", + "ColIds": [ + "c41", + "c42", + "c43" + ], + "ColDefs": { + "c41": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION", + "Id": "f40" + } + ], + "Indexes": [ + { + "Name": "author_id", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "Id": "t35" + }, + "t36": { + "Name": "Genre", + "Schema": "e2e-it", + "ColIds": [ + "c44", + "c45" + ], + "ColDefs": { + "c44": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c45": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c45", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t36" + } + }, + "SchemaIssues": { + "t34": { + "ColumnLevelIssues": { + "c37": [ + 14 + ], + "c38": [] + }, + "TableLevelIssues": null + }, + "t35": { + "ColumnLevelIssues": { + "c41": [ + 14 + ], + "c42": [], + "c43": [ + 14, + 24 + ], + "c47": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t36": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..af959aef6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,28 @@ + +CREATE TABLE IF NOT EXISTS + MY_TABLE ( ID INT64 NOT NULL, + VAL INT64, + ) +PRIMARY KEY + (ID); + +CREATE TABLE IF NOT EXISTS + ALLTYPES ( BOOL_FIELD STRING(MAX) NOT NULL, + INT64_FIELD INT64 NOT NULL, + FLOAT64_FIELD FLOAT64 NOT NULL, + STRING_FIELD STRING(MAX) NOT NULL, + BYTES_FIELD BYTES(MAX) NOT NULL, + TIMESTAMP_FIELD TIMESTAMP NOT NULL, + DATE_FIELD DATE NOT NULL, + NUMERIC_FIELD NUMERIC NOT NULL, + VAL INT64, + ) +PRIMARY KEY + (BOOL_FIELD, + INT64_FIELD, + FLOAT64_FIELD, + STRING_FIELD, + BYTES_FIELD, + TIMESTAMP_FIELD, + DATE_FIELD, + NUMERIC_FIELD); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql new file mode 100644 index 0000000000..3560e1d435 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql @@ -0,0 +1,20 @@ + +CREATE TABLE MY_TABLE ( + ID NUMBER NOT NULL PRIMARY KEY, + VAL NUMBER +); +ALTER TABLE MY_TABLE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +CREATE TABLE ALLTYPES ( + BOOL_FIELD VARCHAR2(10) NOT NULL, + INT64_FIELD NUMBER NOT NULL, + FLOAT64_FIELD BINARY_DOUBLE NOT NULL, + STRING_FIELD VARCHAR2(4000) NOT NULL, + BYTES_FIELD RAW(2000) NOT NULL, + TIMESTAMP_FIELD TIMESTAMP NOT NULL, + DATE_FIELD DATE NOT NULL, + NUMERIC_FIELD NUMBER NOT NULL, + VAL NUMBER, + PRIMARY KEY (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD) +); +ALTER TABLE ALLTYPES ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql new file mode 100644 index 0000000000..7447a310a2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE "Category" ( + "category_id" NUMBER NOT NULL PRIMARY KEY, + "full_name" VARCHAR2(25), + "last_update" TIMESTAMP +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json new file mode 100644 index 0000000000..f6e317ed00 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json @@ -0,0 +1,448 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "XEPDB1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t15515": { + "Name": "Category", + "ColIds": [ + "c15516", + "c15517" + ], + "ShardIdColumn": "", + "ColDefs": { + "c15516": { + "Name": "category_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: category_id tinyint(4)", + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Category", + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "ColIds": [ + "c15520", + "c15521", + "c15522", + "c15523", + "c1" + ], + "ShardIdColumn": "", + "ColDefs": { + "c1": { + "Name": "extraCol1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c1", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15520": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(11)", + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15523": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c15523", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15523", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t15519" + } + }, + "SyntheticPKeys": { + "t15519": { + "ColId": "c15523", + "Sequence": 0 + } + }, + "SrcSchema": { + "t15515": { + "Name": "Category", + "Schema": "", + "ColIds": [ + "c15516", + "c15517", + "c15518" + ], + "ColDefs": { + "c15516": { + "Name": "category_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15518": { + "Name": "last_update", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15518", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "Schema": "", + "ColIds": [ + "c15520", + "c15521", + "c15522" + ], + "ColDefs": { + "c15520": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": null, + "Indexes": null, + "Id": "t15519" + }, + "t15520": { + "Name": "Genre", + "Schema": "", + "ColIds": [ + "c13", + "c14" + ], + "ColDefs": { + "c13": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + }, + "c14": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14" + } + }, + "PrimaryKeys": [ + { + "ColId": "c14", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15520" + } + }, + "SchemaIssues": { + "t15515": { + "ColumnLevelIssues": { + "c15516": [ + 14 + ], + "c15517": [] + }, + "TableLevelIssues": null + }, + "t15519": { + "ColumnLevelIssues": { + "c15520": [ + 14 + ], + "c15521": [], + "c15522": [ + 14 + ], + "c15523": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t15520": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql new file mode 100644 index 0000000000..d57b1f3c9e --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS Category ( + category_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY(category_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), + extraCol1 INT64, +) PRIMARY KEY(synth_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..921e1b3013 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64 +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3e8b59c4ff --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..c87cdd07ab --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500) DEFAULT NULL, + "last_name1" VARCHAR2(500) DEFAULT NULL, + "first_name2" VARCHAR2(500) DEFAULT NULL, + "last_name2" VARCHAR2(500) DEFAULT NULL, + "first_name3" VARCHAR2(500) DEFAULT NULL, + "last_name3" VARCHAR2(500) DEFAULT NULL, + "ID" NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java index 132b3bc84f..b2e0448fe2 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java @@ -59,7 +59,8 @@ public static ISpannerMigrationTransformer getApplyTransformationImpl( URL[] classLoaderUrls = JarFileReader.saveFilesLocally(customTransformation.jarPath()); // Create a new URLClassLoader - URLClassLoader urlClassLoader = new URLClassLoader(classLoaderUrls); + URLClassLoader urlClassLoader = + new URLClassLoader(classLoaderUrls, Thread.currentThread().getContextClassLoader()); // Load the target class Class customTransformationClass =