From 3759e32d5c4236efc8abd51a7f2a4889eeb8587f Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Mon, 24 Aug 2026 10:45:33 +0000 Subject: [PATCH 01/13] Add oracle live ITs --- .github/workflows/spanner-pr.yml | 8 +- cicd/cmd/run-it-smoke-tests/main.go | 1 + cicd/cmd/run-it-tests/main.go | 1 + cicd/internal/flags/it-flags.go | 6 + .../cloudsql/CloudOracleResourceManager.java | 2 +- .../v2/templates/DataStreamToSpannerIT.java | 4 +- .../templates/DataStreamToSpannerITBase.java | 41 + .../DataStreamToSpannerOracleDDLIT.java | 663 + ...ataStreamToSpannerOracleRetryAllDLQIT.java | 348 + .../DataStreamToSpannerOracleRetryDLQIT.java | 417 + ...treamToSpannerOracleStringOverridesIT.java | 189 + ...OracleWideRowForMaxColumnsPerTablesIT.java | 347 + ...amToSpannerShardedOracleRetryAllDLQIT.java | 399 + ...reamToSpannerReservedKeywordsOracleIT.java | 192 + .../OracleDataStreamToSpannerEventsIT.java | 388 + ...cleDataStreamToSpannerFileOverridesIT.java | 267 + .../oracle/OracleDataStreamToSpannerIT.java | 477 + .../OracleDataStreamToSpannerIT.java.orig | 455 + .../OracleDataStreamToSpannerIT.java.rej | 33 + .../OracleDataStreamToSpannerMixedIT.java | 298 + .../OracleDataStreamToSpannerSessionIT.java | 315 + .../OracleDataStreamToSpannerTimezoneIT.java | 206 + ...rWideRowForMax16KeyTablePerDatabaseIT.java | 378 + ...RowForMaxTableNameWithMaxColumnNameIT.java | 357 + .../OracleDatastreamToSpannerDataTypesIT.java | 575 + ...amToSpannerSingleDFShardedMigrationIT.java | 352 + ...astreamToSpannerTableAndIndexLimitsIT.java | 252 + ...erWideRowForMax9MibTablePerDatabaseIT.java | 332 + ...racleSeparateShadowTableDatabaseDDLIT.java | 671 + ...leSeparateShadowTableDatabaseEventsIT.java | 420 + ...ateShadowTableDatabaseFileOverridesIT.java | 273 + ...cleSeparateShadowTableDatabaseMixedIT.java | 277 + ...eparateShadowTableDatabasePKFocusedIT.java | 439 + ...eSeparateShadowTableDatabaseSessionIT.java | 411 + ...MigrationWithMigrationShardIdColumnIT.java | 301 + ...rationWithoutMigrationShardIdColumnIT.java | 299 + ...bleDatabaseSingleDFShardedMigrationIT.java | 298 + ...eShadowTableDatabaseStringOverridesIT.java | 239 + ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 131 + .../oracle-inserts.sql | 4 + .../oracle-schema.sql | 156 + ...cle-google_standard_sql-spanner-schema.sql | 22 + .../oracle-schema.sql | 21 + .../overrides.json | 8 + ...cle-google_standard_sql-spanner-schema.sql | 60 + .../oracle-schema.sql | 57 + .../overrides.json | 8 + .../session.json | 4272 ++++++ ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 9 + .../oracle-schema.sql | 10 + ...cle-google_standard_sql-spanner-schema.sql | 59 + .../oracle-schema.sql | 56 + .../overrides.json | 8 + ...cle-google_standard_sql-spanner-schema.sql | 140 + .../oracle-schema.sql | 142 + .../session.json | 323 + .../oracle-Articles.avro | Bin 0 -> 2484 bytes .../oracle-Authors.avro | Bin 0 -> 2154 bytes .../oracle-Books.avro | Bin 0 -> 2304 bytes .../oracle-backfill-Movie.avro | Bin 0 -> 1938 bytes .../oracle-backfill-Users.avro | Bin 0 -> 1927 bytes .../oracle-cdc-Users.avro | Bin 0 -> 2592 bytes ...cle-google_standard_sql-spanner-schema.sql | 40 + .../oracle-schema.sql | 44 + ...cle-google_standard_sql-spanner-schema.sql | 9 + .../oracle-schema.sql | 10 + .../override.json | 10 + ...cle-google_standard_sql-spanner-schema.sql | 20 + .../oracle-schema.sql | 20 + .../oracle-session.json | 506 + ...cle-google_standard_sql-spanner-schema.sql | 12 + .../oracle-schema.sql | 13 + .../oracle-session.json | 456 + ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 7 + .../oracle-schema.sql | 7 + ...cle-google_standard_sql-spanner-schema.sql | 71 + .../oracle-schema.sql | 142 + ...cle-google_standard_sql-spanner-schema.sql | 6 + .../oracle-schema.sql | 6 + .../oracle-session.json | 215 + ...cle-google_standard_sql-spanner-schema.sql | 181 + .../oracle-schema.sql | 28 + .../session.json | 11391 ++++++++++++++++ ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 131 + .../oracle-inserts.sql | 4 + .../oracle-schema.sql | 156 + .../oracle-Articles.avro | Bin 0 -> 2484 bytes .../oracle-Authors.avro | Bin 0 -> 2154 bytes .../oracle-Books.avro | Bin 0 -> 2304 bytes ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 40 + .../oracle-backfill-Movie.avro | Bin 0 -> 1938 bytes .../oracle-backfill-Users.avro | Bin 0 -> 1927 bytes .../oracle-cdc-Users.avro | Bin 0 -> 2592 bytes .../oracle-schema.sql | 52 + ...cle-google_standard_sql-spanner-schema.sql | 9 + .../oracle-schema.sql | 10 + .../override.json | 10 + ...cle-google_standard_sql-spanner-schema.sql | 20 + .../oracle-schema.sql | 26 + .../oracle-session.json | 506 + ...cle-google_standard_sql-spanner-schema.sql | 28 + .../oracle-schema.sql | 20 + .../oracle-schema.sql | 11 + .../oracle-session.json | 448 + .../spanner-schema.sql | 12 + ...cle-google_standard_sql-spanner-schema.sql | 6 + .../oracle-schema.sql | 8 + .../oracle-session.json | 215 + ...cle-google_standard_sql-spanner-schema.sql | 5 + .../oracle-schema.sql | 8 + .../oracle-session.json | 215 + ...cle-google_standard_sql-spanner-schema.sql | 6 + .../oracle-schema.sql | 8 + .../oracle-session.json | 215 + ...cle-GOOGLE_STANDARD_SQL-spanner-schema.sql | 9 + .../oracle-schema.sql | 10 + .../CustomTransformationImplFetcher.java | 2 +- 117 files changed, 31796 insertions(+), 5 deletions(-) create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Authors.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Books.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Movie.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-backfill-Users.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-cdc-Users.avro create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index 0bb8633866..8b035e4b4d 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -173,7 +173,10 @@ jobs: --it-project="span-cloud-teleport-testing" \ --it-artifact-bucket="span-cloud-teleport-testing-it-gitactions" \ --it-private-connectivity="datastream-connect-2" \ - --it-cloud-proxy-host="10.128.0.16" + --it-cloud-proxy-host="10.128.0.16" \ + --it-oracle-host="10.128.0.108" \ + --it-oracle-sys-password="TestPassword123" \ + --it-oracle-password="TestPassword123" - name: Upload Smoke Tests Report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() # always run even if the previous step fails @@ -217,6 +220,9 @@ jobs: --it-artifact-bucket="span-cloud-teleport-testing-it-gitactions" \ --it-private-connectivity="datastream-connect-2" \ --it-cloud-proxy-host="10.128.0.16" \ + --it-oracle-host="10.128.0.108" \ + --it-oracle-sys-password="TestPassword123" \ + --it-oracle-password="TestPassword123" \ --test="$SPECIFIC_TEST" - name: Upload Integration Tests Report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/cicd/cmd/run-it-smoke-tests/main.go b/cicd/cmd/run-it-smoke-tests/main.go index 7290477c44..65c6810759 100644 --- a/cicd/cmd/run-it-smoke-tests/main.go +++ b/cicd/cmd/run-it-smoke-tests/main.go @@ -69,6 +69,7 @@ func main() { flags.RetryFailures(), flags.StaticOracleHost(), flags.StaticOracleSysPassword(), + flags.StaticOraclePassword(), flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index e4fe0669e4..f654b3a9b6 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -70,6 +70,7 @@ func main() { flags.RetryFailures(), flags.StaticOracleHost(), flags.StaticOracleSysPassword(), + flags.StaticOraclePassword(), flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7fd488819a..acbda5bec5 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -39,6 +39,7 @@ var ( dCloudProxyPassword string dOracleHost string dCloudOracleSysPassword string + dCloudOraclePassword string dUnifiedWorkerHarnessContainerImage string dIntegrationTestParallelism string dThreadCount string @@ -61,6 +62,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") + flag.StringVar(&dCloudOraclePassword, "it-oracle-password", "oracle", "app password of static Oracle DB") flag.StringVar(&dUnifiedWorkerHarnessContainerImage, "it-unified-worker-harness-container-image", "", "Runner harness image to run tests against") flag.StringVar(&dIntegrationTestParallelism, "it-integration-test-parallelism", "3", "The level of parallelism for integration tests") flag.StringVar(&dThreadCount, "it-thread-count", "4", "The IT thread count to use for maven, which is the number of threads per core") @@ -149,6 +151,10 @@ func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } +func StaticOraclePassword() string { + return "-DcloudOraclePassword=" + dCloudOraclePassword +} + func UnifiedWorkerHarnessContainerImage() string { if dUnifiedWorkerHarnessContainerImage != "" { return "-DunifiedWorkerHarnessContainerImage=" + dUnifiedWorkerHarnessContainerImage diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java index 1e5081a77b..391eb2fe41 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java @@ -61,7 +61,7 @@ public static Builder builder(String testId) { @Override public synchronized @NonNull String getUri() { return String.format( - "jdbc:%s:thin:@%s:%d:%s", + "jdbc:%s:thin:@//%s:%d/%s", getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java index 94ac8b7013..75cfd32028 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java @@ -507,7 +507,7 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + flushOracleRedoLogs(cloudSqlResourceManager); } return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); } @@ -578,7 +578,7 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + flushOracleRedoLogs(cloudSqlResourceManager); } return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index dbb355be98..bc61635ed2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -89,6 +89,19 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { .maybeUseStaticInstance() .build(); } + public org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager setUpOracleResourceManager() { + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(System.getProperty("cloudProxyUsername", "system")); + builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); + builder.setDatabaseName("XEPDB1"); + } + return (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + } public String generateSessionFile( int numOfTables, String srcDb, String spannerDb, List tableNames, String sessionFile) @@ -614,4 +627,32 @@ protected ConditionCheck buildBaseConditionCheck( } return combinedCondition; } + + public void flushOracleRedoLogs(org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager) { + boolean success = false; + if (oracleResourceManager != null) { + try { + oracleResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).info("Successfully flushed Oracle redo logs natively."); + success = true; + } catch (Exception e) { + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).warn("Failed to switch Oracle log via ResourceManager, attempting raw JDBC fallback...", e); + } + } + + if (!success && System.getProperty("hostIp") != null) { + String url = "jdbc:oracle:thin:@//" + System.getProperty("hostIp") + ":1521/XE"; + String user = System.getProperty("cloudProxyUsername", "system"); + String pass = System.getProperty("cloudProxyPassword", "TestPassword123"); + try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); + java.sql.Statement stmt = conn.createStatement()) { + stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).info("Successfully flushed Oracle redo logs via raw JDBC fallback."); + } catch (Exception ex) { + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).error("Raw JDBC fallback log flush also failed.", ex); + } + } + } + + } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java new file mode 100644 index 0000000000..4ebebed7ce --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -0,0 +1,663 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck.CheckResult; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for datatype conversions for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleDDLIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(DataStreamToSpannerOracleDDLIT.class); + + private static final String SPANNER_DDL_RESOURCE = "oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static final String TABLE1 = "AllDatatypeColumns"; + private static final String TABLE2 = "AllDatatypeColumns2"; + private static final String TABLE3 = "DatatypeColumnsWithSizes"; + private static final String TABLE4 = "DatatypeColumnsReducedSizes"; + private static final String TABLE5 = "Users"; + private static final String TABLE6 = "Books"; + private static final String TABLE7 = "Authors"; + private static final String TABLE8 = "Singers"; + + private static final String TRANSFORMATION_TABLE = "AllDatatypeTransformation"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; + + + private void flushOracleLogs() { + try { + Class.forName("oracle.jdbc.OracleDriver"); + try (Connection conn = DriverManager.getConnection("jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + } catch (Exception e) { + throw new RuntimeException("Failed to switch logfile", e); + } + } + + @Before + + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleDDLIT.class) { + testInstances.add(this); + if (jobInfo == null) { + oracleResourceManager = setUpOracleResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + createAndUploadJarToGcs("DatatypeIT", gcsResourceManager); + CustomTransformation customTransformation = + CustomTransformation.builder( + "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") + .build(); + + String[] tables = {"AllDatatypeColumns", "AllDatatypeColumns2", "DatatypeColumnsWithSizes", "DatatypeColumnsReducedSizes", "Users", "Authors", "AllDatatypeTransformation", "Singers", "Books"}; + for (String table : tables) { + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); + } catch (Exception e) { + } +executeSqlScript(oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql"); + flushOracleLogs(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DatatypeIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + customTransformation, // customTransformation + null, // shardingContextFileResourceName + gcsResourceManager, + datastreamResourceManager, + null, // sessionResourceContent + org.apache.beam.it.gcp.datastream.OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables(java.util.Map.of(oracleResourceManager.getUsername().toUpperCase(), java.util.Arrays.asList(TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + .build()); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleDDLIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager); + } + + @Test + public void migrationTestWithAllDatatypeConversionMapping() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + @Override + protected String getDescription() { return "Run SQL inserts"; } + @Override + protected CheckResult check() { + try { + executeSqlScript(oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql"); + flushOracleLogs(); + return new CheckResult(true, "Success"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(15)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertAllDatatypeColumnsTableBackfillContents(); + } + + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + // Only asserting primary key + simple int column to verify end-to-end sync without complex base64 logic + row.put("int_column", "50000"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, int_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("date_column", "2024-02-08T00:00:00Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("decimal_column", "456.12"); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + // text, char, tinytext, mediumtext, longtext are BYTE columns + row.put("text_column", "/u/9n58P"); + row.put("char_column", "v58P"); + row.put("tinytext_column", "7+/+7/2fnw8="); + row.put("mediumtext_column", "/+3v79/v2vrx"); + row.put("longtext_column", "/+/v3+/a+vE="); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f3100000000000000000..."); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column" + + ", tinyblob_column, blob_column, mediumblob_column" + + ", longblob_column, enum_column, bool_column, other_bool_column" + + ", varbinary_column, bit_column, decimal_column, text_column, binary_column" + + ", char_column, tinytext_column, mediumtext_column, longtext_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("decimal_column", 123.45); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "15:30:45"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" + + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 21); + row.put("text_column", "Some text 2 append"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 201); + row.put("bigint_column", 987655); + row.put("float_column", 24.45); + row.put("double_column", 235.567); + row.put("decimal_column", 23457.78); + row.put("datetime_column", "2022-12-31T23:59:58Z"); + row.put("timestamp_column", "2022-12-31T23:59:58Z"); + row.put("time_column", "00:59:59"); + row.put("year_column", "2023"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 31); + row.put("text_column", "Some text 3 append"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 301); + row.put("bigint_column", 112234); + row.put("float_column", 35.56); + row.put("double_column", 346.678); + row.put("decimal_column", 34568.89); + row.put("datetime_column", "2023-12-31T23:59:59Z"); + row.put("timestamp_column", "2023-12-31T23:59:59Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example4"); + row.put("tinyint_column", 41); + row.put("text_column", "Some text 4 append"); + row.put("date_column", "2021-11-12"); + row.put("int_column", 401); + row.put("bigint_column", 223345); + row.put("float_column", 46.67); + row.put("double_column", 457.789); + row.put("decimal_column", 45679.90); + row.put("datetime_column", "2021-11-11T11:11:10Z"); + row.put("timestamp_column", "2021-11-11T11:11:10Z"); + row.put("time_column", "12:11:11"); + row.put("year_column", "2022"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, binary_column, bit_column FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 25); + row.put("text_column", "Updated text 2"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 250); + row.put("bigint_column", 56789); + row.put("float_column", 25.45); + row.put("double_column", 345.678); + row.put("decimal_column", 23456.79); + row.put("datetime_column", "2023-01-01T12:00:00Z"); + row.put("timestamp_column", "2023-01-01T12:00:00Z"); + row.put("time_column", "12:00:00"); + row.put("year_column", "2023"); + row.put("blob_column", "EjRWeJCrze8="); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "EjRWeJCrze8SNFZ4kKvN7xI0Vng="); + row.put("bit_column", "fw=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 35); + row.put("text_column", "Updated text 3"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 350); + row.put("bigint_column", 88000); + row.put("float_column", 35.67); + row.put("double_column", 456.789); + row.put("decimal_column", 34567.90); + row.put("datetime_column", "2024-01-02T00:00:00Z"); + row.put("timestamp_column", "2024-01-02T00:00:00Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "q83vEjRWeJA="); + row.put("enum_column", "1"); + row.put("bool_column", false); + row.put("binary_column", "q83vEjRWeJCrze8SNFZ4kKvN7xI="); + row.put("bit_column", "AA=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("SELECT * FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" + + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDataTypeSizeConversionBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("float_column", 45.67); + row.put("decimal_column", 456.12); + row.put("char_column", "char_1"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column" + + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsWithSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOException { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column" + + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsReducedSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertUsersBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("user_id", 1); + row.put("first_name", "Lorem"); + row.put("last_name", "Epsum"); + row.put("full_name", "Lorem Epsum"); + row.put("age", 20); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 2); + row.put("first_name", "Jane"); + row.put("last_name", "Doe"); + row.put("full_name", "Jane Doe"); + row.put("age", 21); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 9); + row.put("first_name", "James"); + row.put("last_name", "Dove"); + row.put("full_name", "James Dove"); + row.put("age", 22); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select user_id, first_name, last_name, full_name, age from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAuthorsBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "J.R.R. Tolkien"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Jane Austen"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Douglas Adams"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, name from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhiker's Guide to the Galaxy"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..7c890f0d8a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -0,0 +1,348 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleRetryAllDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-retryalldlq-test"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // Setup Oracle Source + CloudOracleResourceManager sysUser = setUpOracleResourceManager(); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + CloudOracleResourceManager.builder(testName); + sysBuilder.setHost(sysUser.getHost()); + sysBuilder.setPort(sysUser.getPort()); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setDatabaseName(sysUser.getDatabaseName()); + CloudOracleResourceManager trueSysUser = (CloudOracleResourceManager) sysBuilder.build(); + + String oracleUser = System.getProperty("cloudProxyUsername", "system"); + String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + CloudOracleResourceManager.builder(testName); + builder.setHost(sysUser.getHost()); + builder.setPort(sysUser.getPort()); + builder.setUsername(oracleUser); + builder.setPassword(oraclePassword); + builder.setDatabaseName("XEPDB1"); + + jdbcResourceManager = (CloudOracleResourceManager) builder.build(); + + // Create Oracle Schema using helper + executeSqlScript(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE); + + // Add Supplemental Log Data required by Datastream CDC + List.of("Customers", "Orders", "AllDataTypes").forEach( + tableName -> { + jdbcResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); + }); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + // Upload overrides file + gcsResourceManager.uploadArtifact( + "input/overrides.json", Resources.getResource(OVERRIDES_FILE_RESOURCE).getPath()); + + // Upload custom transformation jar + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + // Prepare job parameters + Map jobParameters = new HashMap<>(); + jobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "20"); + jobParameters.put("dlqRetryMinutes", "1"); + + OracleSource oracleSource = + OracleSource.builder( + jdbcResourceManager.getHost(), + jdbcResourceManager.getUsername(), + jdbcResourceManager.getPassword(), + jdbcResourceManager.getPort(), + jdbcResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + jdbcResourceManager.getUsername().toUpperCase(), + List.of("Customers", "Orders", "AllDataTypes"))) + .build(); + + // Launch regular pipeline using ITBase method + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + null, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, jdbcResourceManager, gcsResourceManager, datastreamResourceManager); + } + + @Test + public void testDataStreamToSpannerRetryAllDLQ() throws Exception { + LOG.info("Starting testDataStreamToSpannerRetryAllDLQ"); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Inserting parent rows directly into Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .build())); + + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + LOG.info("Waiting for DLQ events to appear in retry and severe buckets"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder( + gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) // id = 102 + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) // id= 2 + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) // id = 1 + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); + assertTrue("id=999 should NOT exist yet in AllDataTypes", !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); + assertTrue("id=1 should NOT exist yet in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); + assertTrue("id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(15))); + + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Parent Customer") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .build())); + + Map retryJobParameters = new HashMap<>(); + retryJobParameters.put("runMode", "retryAllDLQ"); + retryJobParameters.put("schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryJobParameters.put("transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryJobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryJobParameters.put("transformationCustomParameters", "mode=good"); + retryJobParameters.put("dlqMaxRetryCount", "20"); + retryJobParameters.put("dlqRetryMinutes", "1"); + retryJobParameters.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + null, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryJobParameters, + null, + null, + gcsResourceManager, + null, + null, + null); + + assertThatPipeline(retryJobInfo).isRunning(); + + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(1) + .setMaxEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .setMaxEvents(1) + .build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes").setMinRows(2).build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(30)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + + assertTrue("id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + + List rows = + spannerResourceManager.runQuery("SELECT * FROM AllDataTypes WHERE id = 999"); + Map expectedRow = createExpectedRowFor999(); + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(List.of(expectedRow)); + + assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); + assertTrue("id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + } + + private boolean rowExistsInSpanner(String tableName, String idColumnName, long id) { + List rows = + spannerResourceManager.readTableRecords(tableName, List.of(idColumnName)); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id) { + return true; + } + } + return false; + } + + private void insertDataInOracle() { + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar2_col\") VALUES (1, 'test1')"); + jdbcResourceManager.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar2_col\") VALUES (999, 'test999')"); + } + + private Map createExpectedRowFor999() { + Map row = new HashMap<>(); + row.put("id", 999L); + row.put("varchar2_col", "test999"); + return row; + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java new file mode 100644 index 0000000000..08a0aee6cc --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -0,0 +1,417 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.io.Resources; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleRetryDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerOracleRetryDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/DataStreamToSpannerOracleRetryDLQIT/session.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-sharded-retrydlq"; + + private static final HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static PubsubResourceManager pubsubResourceManager; + + private static String streamNameA; + + private CloudOracleResourceManager setUpOracleResourceManagerForShard(String shardName) { + if (System.getProperty("hostIp") != null) { + CloudOracleResourceManager.Builder shardBuilder = CloudOracleResourceManager.builder(testName + shardName); + shardBuilder.setUsername(System.getProperty("cloudProxyUsername", "system")); + shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + shardBuilder.setHost(System.getProperty("hostIp")); + shardBuilder.setPort(1521); + shardBuilder.setDatabaseName("XEPDB1"); + return shardBuilder.build(); + } else { + return CloudOracleResourceManager.builder(testName + shardName).build(); + } + } + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleRetryDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + jdbcResourceManagerShardA = setUpOracleResourceManagerForShard("A"); + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) {} + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) {} + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) {} + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + pubsubResourceManager = setUpPubSubResourceManager(); + + gcsResourceManager.uploadArtifact( + "input/session.json", Resources.getResource(SESSION_FILE_RESOURCE).getPath()); + + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); + + String gcsPrefix = + getGcsPath(GCS_PATH_PREFIX + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + + OracleSource oracleSourceA = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), "XEPDB1") + .setAllowedTables( + Map.of( + jdbcResourceManagerShardA.getUsername().toUpperCase(), + List.of("Customers", "Orders", "AllDataTypes"))) // Assuming schemas are username + .build(); + + SourceConfig sourceConfigA = + datastreamResourceManager.buildJDBCSourceConfig("oracle-profile-shardA", oracleSourceA); + DestinationConfig destinationConfigA = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile-shardA", + gcsResourceManager.getBucket(), + gcsPrefix, + DestinationOutputFormat.AVRO_FILE_FORMAT); + Stream streamA = + datastreamResourceManager.createStream( + "stream-shardA", sourceConfigA, destinationConfigA); + datastreamResourceManager.startStream(streamA); + streamNameA = streamA.getName().substring(streamA.getName().lastIndexOf('/') + 1); + + String shardConfig = + generateSourceConfig( + streamNameA, + jdbcResourceManagerShardA.getUsername().toUpperCase(), + "shard1", + streamNameA, + "DUMMY_DB_B", + "shard2"); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", shardConfig.getBytes(StandardCharsets.UTF_8)); + + Map jobParameters = new HashMap<>(); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "1000"); + jobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParameters.put( + "inputFilePattern", getGcsPath(GCS_PATH_PREFIX + "/cdc/", gcsResourceManager)); + jobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + jobParameters.put("datastreamSourceType", "oracle"); + jobParameters.put("dlqRetryMinutes", "1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + null); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleRetryDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + jdbcResourceManagerShardA, + jdbcResourceManagerShardA, + gcsResourceManager, + datastreamResourceManager, + pubsubResourceManager); + } + + @Test + public void testDataStreamToSpannerShardedRetryDLQ() throws Exception { + LOG.info("Starting testDataStreamToSpannerShardedRetryDLQ"); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Inserting parent rows directly into Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .set("migration_shard_id") + .to("shard1") + .build())); + + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + LOG.info("Waiting for DLQ events to appear in severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(15)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build() + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + + assertTrue("id=1 should exist in AllDataTypes on shard1", + rowExistsInSpanner("AllDataTypes", "id", 1, "shard1")); + assertTrue("id=999 should NOT exist yet in AllDataTypes on shard1", + !rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); + + assertTrue("id=2 should exist in Customers on shard1", + rowExistsInSpanner("Customers", "CustomerId", 2, "shard1")); + assertTrue("id=1 should NOT exist yet in Customers on shard1", + !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); + + assertTrue("id=102 should exist in Orders on shard1", + rowExistsInSpanner("Orders", "OrderId", 102, "shard1")); + assertTrue("id=101 should NOT exist yet in Orders on shard1", + !rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); + assertTrue("id=103 should NOT exist yet in Orders on shard1", + !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); + + LOG.info("Launching retryDLQ job with session file to process DLQ"); + Map retryJobParameters = new HashMap<>(); + retryJobParameters.put("runMode", "retryDLQ"); + retryJobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryJobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryJobParameters.put("transformationCustomParameters", "mode=good"); + retryJobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryJobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + retryJobParameters.put("datastreamSourceType", "oracle"); + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + SESSION_FILE_RESOURCE, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryJobParameters, + null, + null, + gcsResourceManager, + null, + null, + null); + LOG.info("RetryDLQ job launched: {}", retryJobInfo.jobId()); + + assertThatPipeline(retryJobInfo).isRunning(); + + LOG.info("Applying partial fixes in Spanner"); + spannerResourceManager.write( + List.of( + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Parent Customer") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .set("migration_shard_id") + .to("shard1") + .build())); + + LOG.info("Waiting for the retryDLQ job to complete automatically"); + PipelineOperator.Result retryJobResult = + pipelineOperator().waitUntilDone(createConfig(retryJobInfo, Duration.ofMinutes(15))); + assertThatResult(retryJobResult).isLaunchFinished(); + + assertTrue( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .build() + .get()); + + LOG.info("Waiting for fixed rows to appear in Spanner"); + PipelineOperator.Result finalWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(2) + .build() + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build())); + assertThatResult(finalWaitResult).meetsConditions(); + + assertTrue("id=999 should exist", rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); + assertTrue("id=3 should exist", rowExistsInSpanner("Customers", "CustomerId", 3, "shard1")); + assertTrue("id=1 should NOT exist", !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); + assertTrue("id=101 should exist", rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); + assertTrue("id=103 should NOT exist", !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); + + LOG.info("Stopping the regular pipeline: {}", jobInfo.jobId()); + pipelineLauncher.cancelJob(PROJECT, REGION, jobInfo.jobId()); + } + + private boolean rowExistsInSpanner( + String tableName, String idColumnName, long id, String shardId) { + List rows = + spannerResourceManager.readTableRecords( + tableName, List.of(idColumnName, "migration_shard_id")); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id && row.getString("migration_shard_id").equals(shardId)) { + return true; + } + } + return false; + } + + private void insertDataInOracle() { + LOG.info("Inserting data in Oracle Shard A"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + + LOG.info("Inserting data in Oracle Shard B"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (1, 'test1')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (999, 'test999')"); + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private void executeSqlScript(CloudOracleResourceManager resourceManager, String resourceName) + throws IOException { + String sql = Resources.toString(Resources.getResource(resourceName), StandardCharsets.UTF_8); + for (String statement : sql.split(";")) { + if (!statement.trim().isEmpty()) { + resourceManager.runSQLUpdate(statement); + } + } + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA, String streamB, String dbB, String shardB) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " },\n" + + " {\n" + + " \"logicalShardId\": \"" + shardB + "\",\n" + + " \"dbName\": \"" + dbB + "\",\n" + + " \"streamId\": \"" + streamB + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java new file mode 100644 index 0000000000..9fafdbe667 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java @@ -0,0 +1,189 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleStringOverridesIT extends DataStreamToSpannerITBase { + + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + + private static HashSet testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static String gcsPrefix; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleStringOverridesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + + try { oracleResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch(Exception e) {} + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + try { + oracleResourceManager.runSQLUpdate("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL"); + } catch(Exception e) { + // Might exist or fail if we are not SYS, ignore wrapper + } + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource oracleSource = OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("tableOverrides", "[{person1, human1}]"); + overridesMap.put("columnOverrides", "[{person1.first_name1, person1.name1}]"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "oracleStringOverridesIT", + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerOracleStringOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager, oracleResourceManager, datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new org.apache.beam.it.conditions.ConditionCheck() { + @Override + protected String getDescription() { + return "Insert records into Oracle"; + } + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); + return new CheckResult(true, "Inserted successfully"); + } catch (Exception e) { + return new CheckResult(false, "Failed to insert"); + } + } + }, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java new file mode 100644 index 0000000000..a8fc8d5a31 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -0,0 +1,347 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + private static final Integer NUM_COLUMNS = 1000; + + private static CloudOracleResourceManager cloudOracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + + private static final List COLUMNS = new ArrayList<>(); + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + // According to instructions on quoting/identifiers: Oracle often folds to uppercase but Datastream captures whatever it is. Oh wait, Oracle tables are usually uppercase or need quotes. + // Let's use uppercase to avoid case folding issues if unquoted + TABLE_NAMES.add("DATASTREAMTOSPANNER_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + } + for (int i = 1; i <= NUM_COLUMNS; i++) { + COLUMNS.add("COL_" + i); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(System.getProperty("cloudProxyUsername")); + builder.setDatabaseName("XEPDB1"); + } + cloudOracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + + String sessionContent = generateBaseSchema(); + sessionContent = sessionContent.replaceAll("SRC_DATABASE", cloudOracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()) + .replaceAll("\"mysql\"", "\"oracle\""); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + cloudOracleResourceManager.getHost(), + cloudOracleResourceManager.getUsername(), + cloudOracleResourceManager.getPassword(), + cloudOracleResourceManager.getPort(), + cloudOracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudOracleResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudOracleResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + cloudOracleResourceManager.runSQLUpdate(getJDBCSchema(tableName)); + cloudOracleResourceManager.runSQLUpdate( + String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerForMaxColumnsPerTables() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + PipelineOperator.Config.builder() + .setJobId(jobInfo.jobId()) + .setProject(PROJECT) + .setRegion(REGION) + .setTimeoutAfter(Duration.ofMinutes(20)) + .setCheckAfter(Duration.ofSeconds(5)) + .build(), + conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + return gson.toJson(sessionTemplate); + } + + public static Map createSessionTemplate() { + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + return createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "VARCHAR2"); + colType.put("Len", 0); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + " VARCHAR2(5)"); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)).append(" VARCHAR2(5) NOT NULL"); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(", PRIMARY KEY (").append(COLUMNS.get(0)).append("))"); + return sb.toString(); + } + + private void createSpannerTables() { + for (String tableName : TABLE_NAMES) { + List columns = new ArrayList<>(); + columns.add(COLUMNS.get(0) + " STRING(10) NOT NULL"); + + for (int i = 2; i <= NUM_COLUMNS; i++) { + columns.add(COLUMNS.get(i - 1) + " STRING(10)"); + } + + String ddlStatement = + String.format( + "CREATE TABLE %s (%s) PRIMARY KEY (%s)", + tableName, String.join(", ", columns), COLUMNS.get(0)); + + spannerResourceManager.executeDdlStatement(ddlStatement); + } + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : TABLE_NAMES) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + values.put(COLUMNS.get(ci - 1), String.valueOf(ci)); + } + rows.add(values); + } + cdcEvents.put(tableName, rows); + success &= cloudOracleResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java new file mode 100644 index 0000000000..c218e40fa8 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -0,0 +1,399 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.conditions.DlqEventsCountCheck; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for live replication from Oracle to Spanner using the retryAllDLQ mode for a + * sharded setup. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerShardedOracleRetryAllDLQIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DataStreamToSpannerShardedOracleRetryAllDLQIT.class); + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql"; + private static final String OVERRIDES_FILE_RESOURCE = + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json"; + private static final String GCS_PATH_PREFIX = "oracle-datastream-to-spanner-sharded-retryall"; + + private static final HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + public static SpannerResourceManager spannerResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + private static String streamNameA; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (DataStreamToSpannerShardedOracleRetryAllDLQIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + String oracleUser = System.getProperty("cloudProxyUsername", "system"); + String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + + + jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) {} + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) {} + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) {} + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + datastreamResourceManager = + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = + org.apache.beam.it.gcp.datastream.OracleSource.builder( + System.getProperty("hostIp"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleUser, + java.util.List.of("Customers", "Orders", "AllDataTypes"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), GCS_PATH_PREFIX + "/cdc/", org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_"+ org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + String shardConfig = + generateSourceConfig( + streamNameA, + oracleUser, + "shard1", + streamNameA, + oracleUser, + "shard2"); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + gcsResourceManager.uploadArtifact( + "input/overrides.json", + com.google.common.io.Resources.getResource("oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json").getPath()); + gcsResourceManager.uploadArtifact( + "input/customShard.jar", + getCustomShardJarPath()); + + java.util.Map jobParameters = new java.util.HashMap<>(); + jobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + jobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + jobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + jobParameters.put("transformationCustomParameters", "mode=bad"); + jobParameters.put("dlqMaxRetryCount", "20"); + jobParameters.put("dlqRetryMinutes", "60"); + jobParameters.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + jobParameters.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + // Explicitly set inputFilePattern for manual stream + jobParameters.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + GCS_PATH_PREFIX + "/cdc/"); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParameters.put("jdbcDriverJars", driverPath); + } + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + null, + jobParameters, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + null + ); + + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerShardedOracleRetryAllDLQIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + jdbcResourceManagerShardA, + gcsResourceManager, + datastreamResourceManager); + } + + @Test + public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + // 1. Insert parent rows directly into Spanner. + com.google.cloud.spanner.Mutation customer2 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(2) + .set("CustomerName") + .to("Customer 2") + .set("CreditLimit") + .to(1500) + .set("LegacyRegion") + .to("Silver") + .build(); + spannerResourceManager.write(customer2); + + // 2. Insert all test data into Oracle. + insertDataInOracle(); + LOG.info("Data inserted into Oracle successfully"); + + // 3. Wait for DLQ events to appear in corresponding buckets. + LOG.info("Waiting for DLQ events to appear in shared severe bucket"); + PipelineOperator.Result dlqWaitResult = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(25)), + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(2) + .build() + .and( + DlqEventsCountCheck.builder( + gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(2) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Orders") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "Customers") + .setMinRows(1) + .setMaxRows(1) + .build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(1) + .setMaxRows(1) + .build())); + assertThatResult(dlqWaitResult).meetsConditions(); + LOG.info("DLQ events successfully generated in corresponding buckets"); + + // Verify Spanner state + LOG.info("Verifying Spanner state before retry job runs"); + assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); + assertTrue("id=999 should NOT exist yet in AllDataTypes", !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); + assertTrue("id=1 should NOT exist yet in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); + assertTrue("id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + + // 4. Stop the regular pipeline + LOG.info("Stopping regular pipeline"); + pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(5))); + LOG.info("Regular pipeline stopped successfully"); + + // 5. Apply partial fixes + LOG.info("Applying partial fixes in Spanner"); + com.google.cloud.spanner.Mutation customer3 = + com.google.cloud.spanner.Mutation.newInsertOrUpdateBuilder("Customers") + .set("CustomerId") + .to(3) + .set("CustomerName") + .to("Customer 3") + .set("CreditLimit") + .to(2000) + .set("LegacyRegion") + .to("Gold") + .build(); + spannerResourceManager.write(customer3); + + // 6. Launch retryAllDLQ job + LOG.info("Launching retryAllDLQ job"); + Map retryParams = new HashMap<>(); + retryParams.put("runMode", "retryAllDLQ"); + retryParams.put("schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryParams.put("transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryParams.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); + retryParams.put("transformationCustomParameters", "mode=good"); + retryParams.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryParams.put("dlqMaxRetryCount", "20"); + retryParams.put("dlqRetryMinutes", "60"); + retryParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + + // Proprietary Drivers + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + retryParams.put("jdbcDriverJars", driverPath); + } + + PipelineLauncher.LaunchInfo retryJobInfo = + launchDataflowJob( + getClass().getSimpleName() + "-retry", + null, + null, + GCS_PATH_PREFIX + "-retry", + spannerResourceManager, + null, + retryParams, + null, + null, + gcsResourceManager, + null, + null, + null + ); + LOG.info("RetryAllDLQ job launched: {}", retryJobInfo.jobId()); + assertThatPipeline(retryJobInfo).isRunning(); + + // 7. Wait for the retry job + LOG.info("Waiting for DLQ events to appear in retry and severe buckets after retry"); + ConditionCheck dlqConditionCheck = + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/retry/") + .setMinEvents(1) + .build() + .and( + DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") + .setMinEvents(1) + .build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes").setMinRows(2).build()) + .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); + + PipelineOperator.Result retryResult = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(retryJobInfo, Duration.ofMinutes(15)), dlqConditionCheck); + + assertThatResult(retryResult).meetsConditions(); + LOG.info("Retry job completed processing successfully"); + + // 8. Verify Spanner state + assertTrue("id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); + assertTrue("id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + } + + private boolean rowExistsInSpanner(String tableName, String idColumnName, long id) { + List rows = + spannerResourceManager.readTableRecords(tableName, List.of(idColumnName)); + for (com.google.cloud.spanner.Struct row : rows) { + if (row.getLong(idColumnName) == id) { + return true; + } + } + return false; + } + + private String getCustomShardJarPath() { + return "/home/dhwanilpatel_google_com/MyStorage/OracleSupport/DataflowTemplates/v2/spanner-custom-shard/target/spanner-custom-shard-1.0-SNAPSHOT.jar"; + } + + private void insertDataInOracle() { + LOG.info("Inserting data in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (1, 'test1')"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (999, 'test999')"); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA, String streamB, String dbB, String shardB) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " },\n" + + " {\n" + + " \"logicalShardId\": \"" + shardB + "\",\n" + + " \"dbName\": \"" + dbB + "\",\n" + + " \"streamId\": \"" + streamB + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java new file mode 100644 index 0000000000..832db20580 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -0,0 +1,192 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static java.util.Map.entry; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DatastreamToSpannerReservedKeywordsOracleIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(DatastreamToSpannerReservedKeywordsOracleIT.class); + + private static final String ORACLE_DDL_RESOURCE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE = + "oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json"; + + private CloudOracleResourceManager oracleResourceManager; + private SpannerResourceManager spannerResourceManager; + private GcsResourceManager gcsResourceManager; + private PubsubResourceManager pubsubResourceManager; + private DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException { + LOG.info("Setting up Oracle resource manager..."); + oracleResourceManager = setUpOracleResourceManager(); + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + LOG.info( + "Spanner resource manager created with instance ID: {}", + spannerResourceManager.getInstanceId()); + + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + LOG.info("GCS resource manager created with bucket: {}", gcsResourceManager.getBucket()); + + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + LOG.info("Pub/Sub resource manager created."); + + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") /* from original */ + .build(); + LOG.info("Datastream resource manager created"); + } + + @After + public void cleanUp() { + LOG.info("Cleaning up resources..."); + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + private void flushOracleLogs() { + LOG.info("Flushing Oracle logs via raw JDBC..."); + try (Connection conn = DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + LOG.info("Successfully switched Oracle log file."); + } catch (Exception e) { + LOG.error("Failed to switch Oracle logfile", e); + } + } + + @Test + public void testOracleReservedKeywords() throws Exception { + LOG.info("Executing Oracle DDL script..."); + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"true\" CASCADE CONSTRAINTS PURGE"); + } catch (Exception e) { + LOG.info("Table true does not exist or could not be dropped: " + e.getMessage()); + } + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + oracleResourceManager.runSQLUpdate("ALTER TABLE \"true\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + flushOracleLogs(); + + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + LOG.info("Generating session file content..."); + String sessionFileContent = + generateSessionFile( + 1, + oracleResourceManager.getDatabaseName(), + spannerResourceManager.getDatabaseId(), + List.of("true"), // Table name + SESSION_FILE); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("true"))) + .build(); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + + LOG.info("Launching Dataflow job..."); + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + "oracle-reserved-keywords", + null, + null, + "datastream-to-spanner-reserved-keywords", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + oracleSource); + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(20)), + SpannerRowsCheck.builder(spannerResourceManager, "`true`").setMinRows(1).build()); + assertThatResult(result).meetsConditions(); + + List rows = null; + try { + rows = spannerResourceManager.readTableRecords("true", List.of("id", "ALL", "AND", "AS")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from `true`", e); + throw e; + } + + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns( + List.of( + Map.ofEntries( + entry("id", 1), entry("ALL", "all"), entry("AND", "and"), entry("AS", "as")))); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java new file mode 100644 index 0000000000..efdffc3bda --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java @@ -0,0 +1,388 @@ +/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerEventsIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerEventsIT.class); + + private static final String TABLE1 = "Users"; + private static final String TABLE2 = "Movie"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerEventsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleEventsIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerEventsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager); + } + + @Test + public void migrationTestWithUpdatesAndDeletes() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE1, + "backfill_users.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Users.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build(), + uploadDataStreamFile( + jobInfo, + TABLE1, + "cdc_users.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-cdc-Users.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertUsersTableContents(); + } + + @Test + public void migrationTestWithInsertsOnly() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE2, + "backfill_movie.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-backfill-Movie.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertMovieTableContents(); + } + + @Test + public void interleavedAndFKAndIndexTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + "Articles", + "mysql_articles.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro", + gcsResourceManager), + uploadDataStreamFile( + jobInfo, + "Authors", + "mysql_authors.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Authors.avro", + gcsResourceManager), + uploadDataStreamFile( + jobInfo, + "Books", + "mysql_books.avro", + "oracle/OracleDataStreamToSpannerEventsIT/oracle-Books.avro", + gcsResourceManager), + SpannerRowsCheck.builder(spannerResourceManager, "Articles") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(18)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTable(); + assertBooksTable(); + assertArticlesTable(); + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("id", 1); + row1.put("name", "Tester Kumar"); + row1.put("age", 30); + row1.put("subscribed", false); + row1.put("plan", "A"); + row1.put("startDate", Timestamp.parseTimestamp("2023-01-01T00:00:00Z")); + + Map row2 = new HashMap<>(); + row2.put("id", 3); + row2.put("name", "Tester Gupta"); + row2.put("age", 50); + row2.put("subscribed", false); + row2.put("plan", "Z"); + row2.put("startDate", Timestamp.parseTimestamp("2023-06-07T00:00:00Z")); + + Map row3 = new HashMap<>(); + row3.put("id", 4); + row3.put("name", "Tester"); + row3.put("age", 38); + row3.put("subscribed", true); + row3.put("plan", "D"); + row3.put("startDate", Timestamp.parseTimestamp("2023-09-10T00:00:00Z")); + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertMovieTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("id", 1); + row1.put("name", "movie1"); + row1.put("startTime", Timestamp.parseTimestamp("2023-01-01T12:12:12.000Z")); + + Map row2 = new HashMap<>(); + row2.put("id", 2); + row2.put("name", "movie2"); + row2.put("startTime", Timestamp.parseTimestamp("2023-11-25T17:10:12.000Z")); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, name, startTime from Movie where id in (1, 2)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + + ImmutableList numericVals = + spannerResourceManager.runQuery("select actor from Movie order by id"); + Assert.assertEquals(12345.09876, numericVals.get(0).getBigDecimal(0).doubleValue(), 0.00000001); + Assert.assertEquals(931.5123, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.00000001); + } + + private void assertAuthorsTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("author_id", 1); + row.put("name", "a1"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 2); + row.put("name", "a2"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 3); + row.put("name", "a3"); + events.add(row); + + row = new HashMap<>(); + row.put("author_id", 4); + row.put("name", "a4"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "Book005"); + row.put("author_id", 3); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Book002"); + row.put("author_id", 3); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "Book004"); + row.put("author_id", 4); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("title", "Book005"); + row.put("author_id", 2); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertArticlesTable() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "Article001"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Article002"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Article004"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 4); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("name", "Article005"); + row.put("published_date", Timestamp.parseTimestamp("2024-01-01T00:00:00Z")); + row.put("author_id", 3); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java new file mode 100644 index 0000000000..85a051b683 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -0,0 +1,267 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.com.google.common.io.Resources; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerFileOverridesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerFileOverridesIT.class); + + private static final String ORACLE_SCHEMA = "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = "oracle/OracleDataStreamToSpannerFileOverridesIT/override.json"; + private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = new HashSet<>(); + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerFileOverridesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + gcsResourceManager.uploadArtifact( + GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerFileOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() throws Exception { + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put( + "schemaOverridesFilePath", + getGcsPath(GCS_PATH_PREFIX + "/override.json", gcsResourceManager)); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck sendDataCondition = new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + sendDataCondition, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java new file mode 100644 index 0000000000..994304eaa9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -0,0 +1,477 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.function.Function; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerTemplateITBase; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(Parameterized.class) +public class OracleDataStreamToSpannerIT extends SpannerTemplateITBase { + + private static final Integer NUM_EVENTS = 10; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + + private String gcsPrefix; + private String dlqGcsPrefix; + + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private static final List COLUMNS = List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED); + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + dlqGcsPrefix = + getGcsPath(testName + "/dlq/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + @Test + public void testDataStreamOracleToSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToPostgresSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.POSTGRESQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToSpannerJson() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.JSON_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + private void simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat fileFormat, + Dialect spannerDialect, + Function paramsAdder) + throws IOException { + + cloudOracleSysUser = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudProxyPassword")) + .setDatabaseName("XE").setHost(System.getProperty("hostIp")).setPort(1521) + .build(); + String oracleUser = System.getProperty("cloudProxyUsername"); + String oraclePassword = System.getProperty("cloudProxyPassword"); + //setUpOracleUser(oracleUser, oraclePassword); +// setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1").setHost(System.getProperty("hostIp")).setPort(1521) + .build(); + + SpannerResourceManager.Builder spannerResourceManagerBuilder = + SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) + .maybeUseStaticInstance() + .useCustomHost(spannerHost) + .setCredentials(credentials); + spannerResourceManager = spannerResourceManagerBuilder.build(); + + List tableNames = + List.of( + ("DatastreamToSpanner_1_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase(), + ("DatastreamToSpanner_2_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase()); + + tableNames.forEach( + tableName -> { + cloudSqlResourceManager.createTable( + tableName, createJdbcSchema()); + cloudSqlResourceManager.runSQLUpdate( + String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); + }); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of(tableNames.get(0), tableNames.get(1)))) + .build(); + + createSpannerTables(tableNames, spannerDialect); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, fileFormat); + + Stream stream = + datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + + createPubSubNotifications(); + String jobName = PipelineUtils.createJobName(testName); + PipelineLauncher.LaunchConfig.Builder options = + paramsAdder + .apply( + PipelineLauncher.LaunchConfig.builder(jobName, specPath) + .addParameter("gcsPubSubSubscription", subscription.toString()) + .addParameter("dlqGcsPubSubSubscription", dlqSubscription.toString()) + .addParameter("streamName", stream.getName()) + .addParameter("instanceId", spannerResourceManager.getInstanceId()) + .addParameter("databaseId", spannerResourceManager.getDatabaseId()) + .addParameter("projectId", PROJECT) + .addParameter( + "deadLetterQueueDirectory", + getGcsPath(testName, gcsResourceManager) + "/dlq/") + .addParameter("spannerHost", spannerResourceManager.getSpannerHost()) + .addParameter( + "inputFileFormat", + fileFormat.equals( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT) + ? "avro" + : "json")) + .addParameter("workerMachineType", "n2-standard-4"); + + PipelineLauncher.LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(tableNames, cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(1)) + .setMinRows(NUM_EVENTS) + .build(), + changeJdbcData(tableNames, cdcEvents), + checkDestinationRows(tableNames, cdcEvents))) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel(createConfig(info, Duration.ofMinutes(20)), conditionCheck); + + checkSpannerTables(tableNames, cdcEvents); + assertThatResult(result).meetsConditions(); + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "INTEGER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "INTEGER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createPubSubNotifications() throws IOException { + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + + TopicName topic = pubsubResourceManager.createTopic("it"); + TopicName dlqTopic = pubsubResourceManager.createTopic("dlq"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "dlq-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + gcsResourceManager.createNotification(dlqTopic.toString(), dlqGcsPrefix.substring(1)); + } + + private void createSpannerTables(List tableNames, Dialect spannerDialect) { + boolean usingPg = Dialect.POSTGRESQL.equals(spannerDialect); + tableNames.forEach( + tableName -> { + String q = usingPg ? "\"" : "`"; + spannerResourceManager.executeDdlStatement( + "CREATE TABLE " + + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") + + " (" + + q + ROW_ID + q + + (usingPg ? " bigint " : " INT64 ") + + "NOT NULL, " + + q + NAME + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + AGE + q + + (usingPg ? " bigint, " : " INT64, ") + + q + MEMBER + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + ENTRY_ADDED + q + + (usingPg ? " character varying(200)" : " STRING(MAX)") + + (usingPg ? ", " : ") ") + + "PRIMARY KEY (" + + q + ROW_ID + q + + ")" + + (usingPg ? ")" : "")); + }); + } + + private ConditionCheck checkDestinationRows( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : tableNames) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + try { + checkSpannerTables(tableNames, cdcEvents); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + private void checkSpannerTables( + List tableNames, Map>> cdcEvents) { + tableNames.forEach( + tableName -> + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(tableName, COLUMNS)) + .hasRecordsUnorderedCaseInsensitiveColumns(cdcEvents.get(tableName))); + } + + private ConditionCheck writeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + values.put(ROW_ID, i); + values.put(NAME, RandomStringUtils.randomAlphabetic(10)); + values.put(AGE, new Random().nextInt(100)); + values.put(MEMBER, new Random().nextInt() % 2 == 0 ? "Y" : "N"); + values.put(ENTRY_ADDED, Instant.now().toString()); + rows.add(values); + } + + List> cdcRows = new ArrayList<>(); + for (Map row : rows) { + Map cdcRow = new HashMap<>(); + cdcRow.put(ROW_ID, row.get(ROW_ID)); + cdcRow.put(NAME, row.get(NAME)); + cdcRow.put(AGE, row.get(AGE)); + cdcRow.put(MEMBER, row.get(MEMBER)); + cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); + cdcRows.add(cdcRow); + } + cdcEvents.put(tableName, cdcRows); + + success &= cloudSqlResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } + + private ConditionCheck changeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send JDBC changes."; + } + + @Override + protected CheckResult check() { + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> newCdcEvents = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + if (i % 2 == 0) { + Map values = cdcEvents.get(tableName).get(i); + + String newName = values.get(NAME).toString().toUpperCase(); + int newAge = new Random().nextInt(100); + String newMember = Objects.equals(values.get(MEMBER).toString(), "Y") ? "N" : "Y"; + + values.put(NAME, newName); + values.put(AGE, newAge); + values.put(MEMBER, newMember); + + String updateSql = + "UPDATE " + + tableName + + " SET " + NAME + " = '" + + newName + + "', " + AGE + " = " + + newAge + + ", " + MEMBER + " = '" + + newMember + + "' WHERE " + ROW_ID + " = " + + i; + cloudSqlResourceManager.runSQLUpdate(updateSql); + newCdcEvents.add(values); + } else { + cloudSqlResourceManager.runSQLUpdate( + "DELETE FROM " + tableName + " WHERE " + ROW_ID + "=" + i); + } + } + cdcEvents.put(tableName, newCdcEvents); + messages.add(String.format("%d changes to %s", newCdcEvents.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig new file mode 100644 index 0000000000..8f97be3766 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.orig @@ -0,0 +1,455 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.function.Function; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerTemplateITBase; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(Parameterized.class) +public class OracleDataStreamToSpannerIT extends SpannerTemplateITBase { + + private static final Integer NUM_EVENTS = 10; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + + private String gcsPrefix; + private String dlqGcsPrefix; + + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private static final List COLUMNS = List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED); + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + dlqGcsPrefix = + getGcsPath(testName + "/dlq/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + @Test + public void testDataStreamOracleToSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToPostgresSpanner() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT, + Dialect.POSTGRESQL, + Function.identity()); + } + + @Test + public void testDataStreamOracleToSpannerJson() throws IOException { + simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat.JSON_FILE_FORMAT, + Dialect.GOOGLE_STANDARD_SQL, + Function.identity()); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s", user)); + } + + private void simpleOracleToSpannerTest( + DatastreamResourceManager.DestinationOutputFormat fileFormat, + Dialect spannerDialect, + Function paramsAdder) + throws IOException { + + cloudOracleSysUser = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword")) + .setDatabaseName("XEPDB1") + .build(); + String oracleUser = "U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .build(); + + SpannerResourceManager.Builder spannerResourceManagerBuilder = + SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) + .maybeUseStaticInstance() + .useCustomHost(spannerHost) + .setCredentials(credentials); + spannerResourceManager = spannerResourceManagerBuilder.build(); + + List tableNames = + List.of( + ("DatastreamToSpanner_1_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase(), + ("DatastreamToSpanner_2_" + RandomStringUtils.randomAlphanumeric(5)).toUpperCase()); + + tableNames.forEach( + tableName -> + cloudSqlResourceManager.createTable( + tableName, createJdbcSchema())); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of(tableNames.get(0), tableNames.get(1)))) + .build(); + + createSpannerTables(tableNames, spannerDialect); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, fileFormat); + + Stream stream = + datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + + createPubSubNotifications(); + String jobName = PipelineUtils.createJobName(testName); + PipelineLauncher.LaunchConfig.Builder options = + paramsAdder + .apply( + PipelineLauncher.LaunchConfig.builder(jobName, specPath) + .addParameter("gcsPubSubSubscription", subscription.toString()) + .addParameter("dlqGcsPubSubSubscription", dlqSubscription.toString()) + .addParameter("streamName", stream.getName()) + .addParameter("instanceId", spannerResourceManager.getInstanceId()) + .addParameter("databaseId", spannerResourceManager.getDatabaseId()) + .addParameter("projectId", PROJECT) + .addParameter( + "deadLetterQueueDirectory", + getGcsPath(testName, gcsResourceManager) + "/dlq/") + .addParameter("spannerHost", spannerResourceManager.getSpannerHost()) + .addParameter( + "inputFileFormat", + fileFormat.equals( + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT) + ? "avro" + : "json")) + .addParameter("workerMachineType", "n2-standard-4"); + + PipelineLauncher.LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(tableNames, cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, tableNames.get(1)) + .setMinRows(NUM_EVENTS) + .build(), + changeJdbcData(tableNames, cdcEvents), + checkDestinationRows(tableNames, cdcEvents))) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel(createConfig(info, Duration.ofMinutes(20)), conditionCheck); + + checkSpannerTables(tableNames, cdcEvents); + assertThatResult(result).meetsConditions(); + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "INTEGER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "INTEGER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createPubSubNotifications() throws IOException { + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + + TopicName topic = pubsubResourceManager.createTopic("it"); + TopicName dlqTopic = pubsubResourceManager.createTopic("dlq"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "dlq-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + gcsResourceManager.createNotification(dlqTopic.toString(), dlqGcsPrefix.substring(1)); + } + + private void createSpannerTables(List tableNames, Dialect spannerDialect) { + boolean usingPg = Dialect.POSTGRESQL.equals(spannerDialect); + tableNames.forEach( + tableName -> { + String q = usingPg ? "\"" : "`"; + spannerResourceManager.executeDdlStatement( + "CREATE TABLE " + + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") + + " (" + + q + ROW_ID + q + + (usingPg ? " bigint " : " INT64 ") + + "NOT NULL, " + + q + NAME + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + AGE + q + + (usingPg ? " bigint, " : " INT64, ") + + q + MEMBER + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + ENTRY_ADDED + q + + (usingPg ? " character varying(200)" : " STRING(MAX)") + + (usingPg ? ", " : ") ") + + "PRIMARY KEY (" + + q + ROW_ID + q + + ")" + + (usingPg ? ")" : "")); + }); + } + + private ConditionCheck checkDestinationRows( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : tableNames) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + try { + checkSpannerTables(tableNames, cdcEvents); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + private void checkSpannerTables( + List tableNames, Map>> cdcEvents) { + tableNames.forEach( + tableName -> + SpannerAsserts.assertThatStructs( + spannerResourceManager.readTableRecords(tableName, COLUMNS)) + .hasRecordsUnorderedCaseInsensitiveColumns(cdcEvents.get(tableName))); + } + + private ConditionCheck writeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + values.put(ROW_ID, i); + values.put(NAME, RandomStringUtils.randomAlphabetic(10)); + values.put(AGE, new Random().nextInt(100)); + values.put(MEMBER, new Random().nextInt() % 2 == 0 ? "Y" : "N"); + values.put(ENTRY_ADDED, Instant.now().toString()); + rows.add(values); + } + + List> cdcRows = new ArrayList<>(); + for (Map row : rows) { + Map cdcRow = new HashMap<>(); + cdcRow.put(ROW_ID, row.get(ROW_ID)); + cdcRow.put(NAME, row.get(NAME)); + cdcRow.put(AGE, row.get(AGE)); + cdcRow.put(MEMBER, row.get(MEMBER)); + cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); + cdcRows.add(cdcRow); + } + cdcEvents.put(tableName, cdcRows); + + success &= cloudSqlResourceManager.write(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } + + private ConditionCheck changeJdbcData( + List tableNames, Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send JDBC changes."; + } + + @Override + protected CheckResult check() { + List messages = new ArrayList<>(); + for (String tableName : tableNames) { + + List> newCdcEvents = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + if (i % 2 == 0) { + Map values = cdcEvents.get(tableName).get(i); + + String newName = values.get(NAME).toString().toUpperCase(); + int newAge = new Random().nextInt(100); + String newMember = Objects.equals(values.get(MEMBER).toString(), "Y") ? "N" : "Y"; + + values.put(NAME, newName); + values.put(AGE, newAge); + values.put(MEMBER, newMember); + + String updateSql = + "UPDATE " + + tableName + + " SET " + NAME + " = '" + + newName + + "', " + AGE + " = " + + newAge + + ", " + MEMBER + " = '" + + newMember + + "' WHERE " + ROW_ID + " = " + + i; + cloudSqlResourceManager.runSQLUpdate(updateSql); + newCdcEvents.add(values); + } else { + cloudSqlResourceManager.runSQLUpdate( + "DELETE FROM " + tableName + " WHERE " + ROW_ID + "=" + i); + } + } + cdcEvents.put(tableName, newCdcEvents); + messages.add(String.format("%d changes to %s", newCdcEvents.size(), tableName)); + } + + cloudSqlResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej new file mode 100644 index 0000000000..3416446f98 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej @@ -0,0 +1,33 @@ +--- v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java ++++ v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +@@ -134,16 +134,19 @@ + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( +- String.format("CREATE USER %s IDENTIFIED BY %s", user, password)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s", user)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s", user)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s", user)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s", user)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s", user)); +- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s", user)); +- cloudOracleSysUser.runSQLUpdate( +- String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s", user)); +- cloudOracleSysUser.runSQLUpdate( +- String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s", user)); ++ String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); ++ cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + } + diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java new file mode 100644 index 0000000000..a13bc64745 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -0,0 +1,298 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerMixedIT extends DataStreamToSpannerITBase { + + private String gcsPrefix; + private SubscriptionName subscription; + private SubscriptionName dlqSubscription; + + private CloudOracleResourceManager cloudOracleSysUser; + private CloudOracleResourceManager cloudSqlResourceManager; + private DatastreamResourceManager datastreamResourceManager; + private SpannerResourceManager spannerResourceManager; + private PubsubResourceManager pubsubResourceManager; + private GcsResourceManager gcsResourceManager; + + @Before + public void setUp() throws IOException { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + gcsPrefix = + getGcsPath(testName + "/cdc/", gcsResourceManager) + .replace("gs://" + gcsResourceManager.getBucket(), ""); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources( + cloudOracleSysUser, + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + @Test + public void mixedMigrationTest() throws IOException { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + executeSqlScript(cloudSqlResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql"); + cloudSqlResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Authors")); + cloudSqlResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Books")); + cloudSqlResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Genre")); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Authors", "Books", "Genre"))) + .build(); + + SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + + Stream stream = + datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + TopicName topic = pubsubResourceManager.createTopic("it-topic"); + subscription = pubsubResourceManager.createSubscription(topic, "it-sub"); + gcsResourceManager.createNotification(topic.toString(), gcsPrefix.substring(1)); + + TopicName dlqTopic = pubsubResourceManager.createTopic("it-dlq"); + dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "it-dlq-sub"); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("gcsPubSubSubscription", subscription.toString()); + jobParams.put("dlqGcsPubSubSubscription", dlqSubscription.toString()); + jobParams.put("streamName", stream.getName()); + jobParams.put("instanceId", spannerResourceManager.getInstanceId()); + jobParams.put("databaseId", spannerResourceManager.getDatabaseId()); + jobParams.put("projectId", PROJECT); + jobParams.put("deadLetterQueueDirectory", getGcsPath(testName, gcsResourceManager) + "/dlq/"); + jobParams.put("workerMachineType", "n2-standard-4"); + jobParams.put("datastreamSourceType", "oracle"); + + PipelineLauncher.LaunchConfig.Builder options = + PipelineLauncher.LaunchConfig.builder(PipelineUtils.createJobName(testName), specPath) + .setParameters(jobParams) + .addEnvironment("ipConfiguration", "WORKER_IP_PRIVATE"); + + PipelineLauncher.LaunchInfo jobInfo = launchDataflowJob( + testName, + "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json", + null, + testName, + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager); + + assertThatPipeline(jobInfo).isRunning(); + + // Condition checks + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors").setMinRows(4).build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books").setMinRows(3).build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES + 10)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTableContents(); + assertBooksTableContents(); + } + + private ConditionCheck writeInitialData() { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Write JDBC data"; + } + + @Override + protected CheckResult check() { + List> authorRows = new ArrayList<>(); + authorRows.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + + List> bookRows = new ArrayList<>(); + bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); + bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); + bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); + + List> genreRows = new ArrayList<>(); + genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); + + for (Map r : authorRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", r.get("author_id"), r.get("full_name"))); + } + for (Map r : bookRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", r.get("id"), r.get("title"), r.get("author_id"))); + } + for (Map r : genreRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", r.get("genre_id"), r.get("name"))); + } + + cloudOracleSysUser.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + + return new CheckResult(true, "Sent to Oracle."); + } + }; + } + + private void assertAuthorsTableContents() { + List> authorEvents = new ArrayList<>(); + authorEvents.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorEvents.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorEvents.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorEvents.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(authorEvents); + } + + private void assertBooksTableContents() { + List> bookEvents = new ArrayList<>(); + bookEvents.add(Map.of("id", 1, "title", "Pride and Prejudice")); + bookEvents.add(Map.of("id", 2, "title", "Oliver Twist")); + bookEvents.add(Map.of("id", 3, "title", "War and Peace")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(bookEvents); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java new file mode 100644 index 0000000000..8e0082e2fd --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -0,0 +1,315 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Arrays; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.JDBCSource; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * An integration test for {@link DataStreamToSpanner} Flex template which tests use-cases where a + * session file is required. + */ +@Category({TemplateIntegrationTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerSessionIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Category"; + private static final String TABLE2 = "Books"; + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json"; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerSessionIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"Category\""); + } catch (Exception e) {} + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"Books\""); + } catch (Exception e) {} + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + // Pre-insert books + oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the Rings', 1)"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and Prejudice', 2)"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers Guide to the Galaxy', 3)"); + + // Pre-insert categories + oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc', CURRENT_TIMESTAMP)"); + + flushOracleLogs(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + JDBCSource jdbcSource = + OracleSource.builder( + System.getProperty("hostIp"), + System.getProperty("cloudProxyUsername"), + System.getProperty("cloudProxyPassword"), + 1521, + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + new HashMap<>() { + { + put( + oracleResourceManager.getUsername().toUpperCase(), + Arrays.asList("Category", "Books")); + } + }) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSessionIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerSessionIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager, datastreamResourceManager, oracleResourceManager); + } + + private void flushOracleLogs() { + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/FREE", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement()) { + stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); + } catch (SQLException e) { + throw new RuntimeException("Failed to flush Oracle logs", e); + } + } + + @Test + public void migrationTestWithRenameAndDrops() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertCategoryTableBackfillContents(); + + oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate("UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); + oracleResourceManager.runSQLUpdate("DELETE FROM \"Category\" WHERE \"category_id\"=1"); + + flushOracleLogs(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertCategoryTableCdcContents(); + } + + @Test + public void migrationTestWithSyntheticPKAndExtraColumn() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertBooksBackfillContents(); + } + + private void assertCategoryTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 1); + row1.put("full_name", "xyz"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 2); + row2.put("full_name", "abc"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableCdcContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 2); + row1.put("full_name", "abc1"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 3); + row2.put("full_name", "def"); + + Map row3 = new HashMap<>(); + row3.put("category_id", 4); + row3.put("full_name", "ghi"); + + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + row.put("author_id", 2); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhikers Guide to the Galaxy"); + row.put("author_id", 3); + events.add(row); + + ImmutableList synthIds = spannerResourceManager.runQuery("select synth_id from Books"); + + Assert.assertEquals(3, synthIds.size()); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, title, author_id from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java new file mode 100644 index 0000000000..4f5da9e4f2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -0,0 +1,206 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Dialect; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerTimezoneIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerTimezoneIT.class); + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql"; + + private static final String TABLE1 = "DateData"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager oracleResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerTimezoneIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + oracleResourceManager = setUpOracleResourceManager(); + CloudOracleResourceManager.Builder sysBuilder = CloudOracleResourceManager.builder(testName); + sysBuilder.setPassword("TestPassword123"); + sysBuilder.setHost("" + System.getProperty("hostIp") + ""); + sysBuilder.setPort(1521); + sysBuilder.setUsername("system"); + sysBuilder.setDatabaseName("XEPDB1"); + cloudOracleSysUser = (CloudOracleResourceManager) sysBuilder.build(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"DateData\""); + } catch (Exception e) { + // ignore + } + + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + // Explicit Log flush + flushOracleRedoLogs(cloudOracleSysUser); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("DateData"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "TimezoneIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDataStreamToSpannerTimezoneIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager, oracleResourceManager, cloudOracleSysUser, datastreamResourceManager); + } + + @Test + public void testTimezoneHandling() { + ConditionCheck insertData = new ConditionCheck() { + @Override + protected String getDescription() { + return "Insert data into Oracle"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02 20:00:00.000000')"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03 06:00:00.000000')"); + + flushOracleRedoLogs(cloudOracleSysUser); + return new CheckResult(true, "Data inserted successfully"); + } catch (Exception e) { + LOG.error("Failed to insert data into Oracle", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertData, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertUsersBackfillContents(); + } + + private void assertUsersBackfillContents() { + List> expectedRows = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("timestamp_column", "2024-02-02T00:00:00Z"); + row.put("datetime_column", "2024-02-02T10:00:00Z"); + expectedRows.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("timestamp_column", "2024-02-02T10:00:00Z"); + row.put("datetime_column", "2024-02-02T20:00:00Z"); + expectedRows.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, timestamp_column, datetime_column from DateData")) + .hasRecordsUnorderedCaseInsensitiveColumns(expectedRows); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java new file mode 100644 index 0000000000..20206f917b --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java @@ -0,0 +1,378 @@ +/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.cloudsql.CloudSqlResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT + extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + + private static final int NUM_COLUMNS = 16; + private static final List COLUMNS = new ArrayList<>(); + private static CloudSqlResourceManager cloudSqlResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + private static DatastreamResourceManager datastreamResourceManager; + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add("DataStreamToSpanner_" + i + "_" + RandomStringUtils.randomAlphanumeric(5)); + } + for (int i = 1; i <= NUM_COLUMNS; i++) { + COLUMNS.add("COL_" + i); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + cloudSqlResourceManager = setUpOracleResourceManager(); + String sessionContent = generateBaseSchema(); + sessionContent = sessionContent + .replaceAll("SRC_DATABASE", cloudSqlResourceManager.getDatabaseName()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName))); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerForMax16KeyTablesPerDatabase() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + PipelineOperator.Config.builder().setJobId(jobInfo.jobId()).setProject(PROJECT).setRegion(REGION).setTimeoutAfter(Duration.ofMinutes(45)).setCheckAfter(Duration.ofSeconds(5)).build(), conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(sessionTemplate); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + final int stringLength = 20; + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "STRING"); + colType.put("Len", stringLength); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + colType.get("Type")); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + public static Map createSessionTemplate() { + + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + Map sessionTemplate = createSessionTemplate(NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + sessionTemplate.put("DatabaseType", "oracle"); + return sessionTemplate; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + + for (int i = 0; i < NUM_COLUMNS; i++) { + sb.append("\"").append(COLUMNS.get(i)).append("\"").append(" VARCHAR2(20) NOT NULL"); + + if (i != NUM_COLUMNS - 1) { + sb.append(", "); + } + } + + sb.append(", PRIMARY KEY (\"").append(String.join("\", \"", COLUMNS)).append("\"))"); + + return sb.toString(); + } + + /** Creates Spanner tables dynamically with 16 columns as a composite primary key. */ + private void createSpannerTables() { + TABLE_NAMES.forEach( + tableName -> { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE ").append(tableName).append(" ("); + + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)).append(" STRING(20) NOT NULL "); + if (i != NUM_COLUMNS) { + sb.append(","); + } + } + sb.append(")"); + sb.append("PRIMARY KEY ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append(COLUMNS.get(i - 1)); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(")"); + + spannerResourceManager.executeDdlStatement(sb.toString()); + }); + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected @NotNull CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + + for (String tableName : TABLE_NAMES) { + List> rows = new ArrayList<>(); + + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + for (int j = 1; j <= NUM_COLUMNS; j++) { + values.put(COLUMNS.get(j - 1), RandomStringUtils.randomAlphabetic(10)); + } + rows.add(values); + } + + cdcEvents.put(tableName, rows); + success &= cloudSqlResourceManager.write(tableName, rows); + + try { + String dynamicHost = cloudSqlResourceManager.getHost(); + int dynamicPort = cloudSqlResourceManager.getPort(); + + // Set system property because the builder internally relies on it! + System.setProperty("cloudOracleHost", dynamicHost); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder("oracle_sys"); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setPassword("TestPassword123"); + sysBuilder.setHost(dynamicHost); + sysBuilder.setPort(dynamicPort); + sysBuilder.setDatabaseName("XE"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudOracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + flushOracleRedoLogs(cloudOracleSysUser); + cloudOracleSysUser.cleanupAll(); + } catch (Throwable e) { + org.slf4j.LoggerFactory.getLogger(OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class).error("FAILED TO EXECUTE SYSDBA SWITCH LOGFILE", e); + e.printStackTrace(); + } + + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java new file mode 100644 index 0000000000..549161ee45 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java @@ -0,0 +1,357 @@ +/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT + extends DataStreamToSpannerITBase { + + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + private static final Integer NUM_COLUMNS = 2; + + private static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudSqlResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static String oracleSchema; + + private static final List COLUMNS = new ArrayList<>(); + private static HashSet + testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add("DataStreamToSpanner_" + i + "_" + RandomStringUtils.randomAlphanumeric(5)); + } + for (int i = 1; i < NUM_COLUMNS; i++) { + COLUMNS.add("col_" + i); + } + COLUMNS.add( + NUM_COLUMNS - 1, "col_" + (NUM_COLUMNS - 1) + RandomStringUtils.randomAlphanumeric(25)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + cloudSqlResourceManager = setUpOracleResourceManager(); + oracleSchema = ("TEST_U" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5)).toUpperCase(); + cloudSqlResourceManager.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s1A", oracleSchema, oracleSchema)); + cloudSqlResourceManager.runSQLUpdate(String.format("GRANT DBA TO %s", oracleSchema)); + String sessionContent = generateBaseSchema(); + sessionContent = sessionContent.replaceAll("SRC_DATABASE", oracleSchema); + sessionContent = sessionContent.replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "DataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + org.apache.beam.it.gcp.datastream.OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleSchema, TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + cloudSqlResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName)); + cloudSqlResourceManager.runSQLUpdate("ALTER TABLE \"" + oracleSchema + "\".\"" + tableName + "\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamMySqlToSpannerForMaxTableNameWithMaxColumnNames() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + + Map>> cdcEvents = new HashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(10)), conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + sessionTemplate.put("DatabaseType", "oracle"); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + return gson.toJson(sessionTemplate); + } + + public static Map createSessionTemplate() { + List colIds = new ArrayList<>(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + colIds.add("c" + ci); + } + return createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + } + + /** Creates column definitions based on column IDs. */ + private static List> createColumnDefinitions(List colIds) { + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + colType.put("Type", "NUMERIC"); + colType.put("Len", 0); + colType.put("IsArray", false); + colType.put("Name", COLUMNS.get(j - 1)); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: " + COLUMNS.get(j - 1) + " MEDIUMTEXT"); + colTypeConfigs.add(colType); + } + return colTypeConfigs; + } + + /** Creates a list of primary key definitions. */ + private static List> createPrimaryKeys(List colIds) { + List> primaryKeys = new ArrayList<>(); + + for (int j = 0; j < colIds.size(); j++) { + Map primaryKey = new LinkedHashMap<>(); + primaryKey.put("ColId", colIds.get(j)); + primaryKey.put("Desc", false); + primaryKey.put("Order", j + 1); + primaryKeys.add(primaryKey); + } + + return primaryKeys; + } + + private String getJDBCSchema(String tableName) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE \"").append(oracleSchema).append("\".\"").append(tableName).append("\" ("); + for (int i = 1; i <= NUM_COLUMNS; i++) { + sb.append("\"").append(COLUMNS.get(i - 1)).append("\" NUMERIC NOT NULL"); + if (i != NUM_COLUMNS) { + sb.append(", "); + } + } + sb.append(", PRIMARY KEY (\"").append(COLUMNS.get(0)).append("\"))"); + return sb.toString(); + } + + private void createSpannerTables() { + for (String tableName : TABLE_NAMES) { + List columns = new ArrayList<>(); + columns.add(COLUMNS.get(0) + " NUMERIC NOT NULL"); + + for (int i = 2; i <= NUM_COLUMNS; i++) { + columns.add(COLUMNS.get(i - 1) + " NUMERIC"); + } + + String ddlStatement = + String.format( + "CREATE TABLE %s (%s) PRIMARY KEY (%s)", + tableName, String.join(", ", columns), COLUMNS.get(0)); + + spannerResourceManager.executeDdlStatement(ddlStatement); + } + } + + /** + * Helper function for constructing a ConditionCheck whose check() method checks the rows in the + * destination Spanner database for specific rows. + * + * @return A ConditionCheck containing the check operation. + */ + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected @NotNull String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected @NotNull CheckResult check() { + // First, check that correct number of rows were deleted. + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + + // Next, make sure in-place mutations were applied. + try { + checkSpannerTables(spannerResourceManager, TABLE_NAMES, cdcEvents, COLUMNS); + return new CheckResult(true, "Spanner tables contain expected rows."); + } catch (AssertionError error) { + return new CheckResult(false, "Spanner tables do not contain expected rows."); + } + } + }; + } + + /** + * Helper function for constructing a ConditionCheck whose check() method constructs the initial + * rows of data in the JDBC database according to the common schema for the IT's in this class. + * + * @return A ConditionCheck containing the JDBC write operation. + */ + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + for (String tableName : TABLE_NAMES) { + + List> rows = new ArrayList<>(); + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new HashMap<>(); + StringBuilder cols = new StringBuilder(); + StringBuilder vals = new StringBuilder(); + for (int ci = 1; ci <= NUM_COLUMNS; ci++) { + values.put(COLUMNS.get(ci - 1), ci); + cols.append("\"").append(COLUMNS.get(ci - 1)).append("\""); + vals.append(ci); + if (ci != NUM_COLUMNS) { + cols.append(", "); + vals.append(", "); + } + } + rows.add(values); + String sql = "INSERT INTO \"" + oracleSchema + "\".\"" + tableName + "\" (" + cols.toString() + ") VALUES (" + vals.toString() + ")"; + try { cloudSqlResourceManager.runSQLUpdate(sql); cloudSqlResourceManager.runSQLUpdate("COMMIT"); } catch (Exception e) { success = false; } + } + cdcEvents.put(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + flushOracleRedoLogs(cloudSqlResourceManager); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java new file mode 100644 index 0000000000..fc84a1fef3 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -0,0 +1,575 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleDatastreamToSpannerDataTypesIT.class); + + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String TABLE_PREFIX = "T_SPRG_"; + private static final List UNSUPPORTED_TYPE_TABLES = List.of( + "T_SPRG_R27_INTERVAL_Y", + "T_SPRG_R28_INTERVAL_D", + "T_SPRG_R30_LONG_RAW", + "T_SPRG_R34_BFILE", + "T_SPRG_R35_LONG", + "T_SPRG_R36_ROWID", + "T_SPRG_R37_UROWID", + "T_SPRG_R38_BOOLEAN", + "T_SPRG_R39_JSON", + "T_SPRG_R40_XMLTYPE", + "T_SPRG_R41_SDO_GEOMET", + "T_SPRG_R42_SDO_TOPO_G", + "T_SPRG_R43_SDO_GEORAS", + "T_SPRG_R44_VECTOR", + "T_SPRG_R45_ANYDATA", + "T_SPRG_R46_ANYTYPE", + "T_SPRG_R47_ANYDATASET", + "T_SPRG_R48_URITYPE", + "T_SPRG_R49_DBURITYPE", + "T_SPRG_R50_XDBURITYPE", + "T_SPRG_R51_HTTPURITYP", + "T_SPRG_R52_EXPRESSION", + "T_SPRG_R53_ORDAUDIO", + "T_SPRG_R54_ORDIMAGE", + "T_SPRG_R55_ORDVIDEO", + "T_SPRG_R56_ORDDOC", + "T_SPRG_R57_SI_STILLIM", + "T_SPRG_R58_SI_COLOR", + "T_SPRG_R59_SI_AVERAGE", + "T_SPRG_R60_SI_COLORHI", + "T_SPRG_R61_SI_POSITIO", + "T_SPRG_R62_SI_TEXTURE", + "T_SPRG_R63_SI_FEATURE", + "T_SPRG_R64_MLSLABEL", + "T_SPRG_R65_REF", + "T_SPRG_R66_REF_CURSOR", + "T_SPRG_R67_VARRAY", + "T_SPRG_R68_NESTED_TAB", + "T_SPRG_R69_ASSOCIATIV", + "T_SPRG_R70_OBJECT_TYP" + ); + + private static boolean initialized = false; + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + + private static HashSet testInstances = new HashSet<>(); + private static final String[] ORACLE_DDL_COMMANDS = new String[] { + "CREATE TABLE T_SPRG_R0_VARCHAR2 (ID NUMBER PRIMARY KEY, COL_0 VARCHAR2(255))", + "ALTER TABLE T_SPRG_R0_VARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R0_VARCHAR2 (ID, COL_0) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R1_VARCHAR (ID NUMBER PRIMARY KEY, COL_1 VARCHAR(255))", + "ALTER TABLE T_SPRG_R1_VARCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R1_VARCHAR (ID, COL_1) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R2_CHAR (ID NUMBER PRIMARY KEY, COL_2 CHAR(255))", + "ALTER TABLE T_SPRG_R2_CHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R2_CHAR (ID, COL_2) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R3_CHARACTER (ID NUMBER PRIMARY KEY, COL_3 CHARACTER(255))", + "ALTER TABLE T_SPRG_R3_CHARACTER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R3_CHARACTER (ID, COL_3) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID NUMBER PRIMARY KEY, COL_4 NVARCHAR2(255))", + "ALTER TABLE T_SPRG_R4_NVARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R4_NVARCHAR2 (ID, COL_4) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R5_NCHAR (ID NUMBER PRIMARY KEY, COL_5 NCHAR(255))", + "ALTER TABLE T_SPRG_R5_NCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R5_NCHAR (ID, COL_5) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID NUMBER PRIMARY KEY, COL_6 NCHAR VARYING(255))", + "ALTER TABLE T_SPRG_R6_NCHAR_VARY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R6_NCHAR_VARY (ID, COL_6) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R7_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_7 NATIONAL CHARACTER(255))", + "ALTER TABLE T_SPRG_R7_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R7_NATIONAL_C (ID, COL_7) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R8_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_8 NATIONAL CHAR(255))", + "ALTER TABLE T_SPRG_R8_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R8_NATIONAL_C (ID, COL_8) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER VARYING(255))", + "ALTER TABLE T_SPRG_R9_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R9_NATIONAL_C (ID, COL_9) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR VARYING(255))", + "ALTER TABLE T_SPRG_R10_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R10_NATIONAL_C (ID, COL_10) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R11_NUMBER (ID NUMBER PRIMARY KEY, COL_11 NUMBER)", + "ALTER TABLE T_SPRG_R11_NUMBER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R11_NUMBER (ID, COL_11) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R12_NUMERIC (ID NUMBER PRIMARY KEY, COL_12 NUMERIC)", + "ALTER TABLE T_SPRG_R12_NUMERIC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R12_NUMERIC (ID, COL_12) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R13_DECIMAL (ID NUMBER PRIMARY KEY, COL_13 DECIMAL)", + "ALTER TABLE T_SPRG_R13_DECIMAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R13_DECIMAL (ID, COL_13) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R14_DEC (ID NUMBER PRIMARY KEY, COL_14 DEC)", + "ALTER TABLE T_SPRG_R14_DEC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R14_DEC (ID, COL_14) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R15_FLOAT (ID NUMBER PRIMARY KEY, COL_15 FLOAT)", + "ALTER TABLE T_SPRG_R15_FLOAT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R15_FLOAT (ID, COL_15) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID NUMBER PRIMARY KEY, COL_16 DOUBLE PRECISION)", + "ALTER TABLE T_SPRG_R16_DOUBLE_PRE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R16_DOUBLE_PRE (ID, COL_16) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R17_REAL (ID NUMBER PRIMARY KEY, COL_17 REAL)", + "ALTER TABLE T_SPRG_R17_REAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R17_REAL (ID, COL_17) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R18_BINARY_FLO (ID NUMBER PRIMARY KEY, COL_18 BINARY_FLOAT)", + "ALTER TABLE T_SPRG_R18_BINARY_FLO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R18_BINARY_FLO (ID, COL_18) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R19_BINARY_DOU (ID NUMBER PRIMARY KEY, COL_19 BINARY_DOUBLE)", + "ALTER TABLE T_SPRG_R19_BINARY_DOU ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R19_BINARY_DOU (ID, COL_19) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R20_INTEGER (ID NUMBER PRIMARY KEY, COL_20 INTEGER)", + "ALTER TABLE T_SPRG_R20_INTEGER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R20_INTEGER (ID, COL_20) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R21_INT (ID NUMBER PRIMARY KEY, COL_21 INT)", + "ALTER TABLE T_SPRG_R21_INT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R21_INT (ID, COL_21) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R22_SMALLINT (ID NUMBER PRIMARY KEY, COL_22 SMALLINT)", + "ALTER TABLE T_SPRG_R22_SMALLINT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R22_SMALLINT (ID, COL_22) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R23_DATE (ID NUMBER PRIMARY KEY, COL_23 DATE)", + "ALTER TABLE T_SPRG_R23_DATE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R23_DATE (ID, COL_23) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R24_TIMESTAMP (ID NUMBER PRIMARY KEY, COL_24 TIMESTAMP)", + "ALTER TABLE T_SPRG_R24_TIMESTAMP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R24_TIMESTAMP (ID, COL_24) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME ZONE)", + "ALTER TABLE T_SPRG_R25_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R25_TIMESTAMP_ (ID, COL_25) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL TIME ZONE)", + "ALTER TABLE T_SPRG_R26_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R26_TIMESTAMP_ (ID, COL_26) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R27_INTERVAL_Y ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R27_INTERVAL_Y (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R28_INTERVAL_D (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R28_INTERVAL_D ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R28_INTERVAL_D (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R29_RAW (ID NUMBER PRIMARY KEY, COL_29 RAW(255))", + "ALTER TABLE T_SPRG_R29_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R29_RAW (ID, COL_29) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R30_LONG_RAW (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R30_LONG_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R30_LONG_RAW (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R31_BLOB (ID NUMBER PRIMARY KEY, COL_31 BLOB)", + "ALTER TABLE T_SPRG_R31_BLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R31_BLOB (ID, COL_31) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R32_CLOB (ID NUMBER PRIMARY KEY, COL_32 CLOB)", + "ALTER TABLE T_SPRG_R32_CLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R32_CLOB (ID, COL_32) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R33_NCLOB (ID NUMBER PRIMARY KEY, COL_33 NCLOB)", + "ALTER TABLE T_SPRG_R33_NCLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R33_NCLOB (ID, COL_33) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R34_BFILE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R34_BFILE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R34_BFILE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R35_LONG (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R35_LONG ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R35_LONG (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R36_ROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R36_ROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R36_ROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R37_UROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R37_UROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R37_UROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R38_BOOLEAN (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R38_BOOLEAN ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R38_BOOLEAN (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R39_JSON (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R39_JSON ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R39_JSON (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R40_XMLTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R40_XMLTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R40_XMLTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R41_SDO_GEOMET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R41_SDO_GEOMET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R42_SDO_TOPO_G ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R42_SDO_TOPO_G (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R43_SDO_GEORAS ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R43_SDO_GEORAS (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R44_VECTOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R44_VECTOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R44_VECTOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R45_ANYDATA (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R45_ANYDATA ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R45_ANYDATA (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R46_ANYTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R46_ANYTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R46_ANYTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R47_ANYDATASET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R47_ANYDATASET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R47_ANYDATASET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R48_URITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R48_URITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R48_URITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R49_DBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R49_DBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R49_DBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R50_XDBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R50_XDBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R50_XDBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R51_HTTPURITYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R51_HTTPURITYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R51_HTTPURITYP (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R52_EXPRESSION (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R52_EXPRESSION ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R52_EXPRESSION (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R53_ORDAUDIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R53_ORDAUDIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R53_ORDAUDIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R54_ORDIMAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R54_ORDIMAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R54_ORDIMAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R55_ORDVIDEO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R55_ORDVIDEO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R55_ORDVIDEO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R56_ORDDOC (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R56_ORDDOC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R56_ORDDOC (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R57_SI_STILLIM (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R57_SI_STILLIM ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R57_SI_STILLIM (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R58_SI_COLOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R58_SI_COLOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R58_SI_COLOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R59_SI_AVERAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R59_SI_AVERAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R60_SI_COLORHI (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R60_SI_COLORHI ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R60_SI_COLORHI (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R61_SI_POSITIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R61_SI_POSITIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R61_SI_POSITIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R62_SI_TEXTURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R62_SI_TEXTURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R63_SI_FEATURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R63_SI_FEATURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R63_SI_FEATURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R64_MLSLABEL (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R64_MLSLABEL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R64_MLSLABEL (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R65_REF (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R65_REF ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R65_REF (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R66_REF_CURSOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R66_REF_CURSOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R66_REF_CURSOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R67_VARRAY (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R67_VARRAY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R67_VARRAY (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R68_NESTED_TAB (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R68_NESTED_TAB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R68_NESTED_TAB (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R69_ASSOCIATIV ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R69_ASSOCIATIV (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R70_OBJECT_TYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R70_OBJECT_TYP (ID, DUMMY) VALUES (1, 'test')" + }; + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerDataTypesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + // Build isolated RM + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + LOG.info("Executing Oracle DDL commands efficiently using a single JDBC connection..."); + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), oracleResourceManager.getUsername(), oracleResourceManager.getPassword()); + java.sql.Statement stmt = conn.createStatement()) { + for (String cmd : ORACLE_DDL_COMMANDS) { + stmt.execute(cmd); + } + } catch (Exception e) { + throw new RuntimeException("Failed to efficiently upload Oracle DDL via single connection", e); + } + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + LOG.info("Cleaning up resources..."); + for (OracleDatastreamToSpannerDataTypesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void testOracleDataTypes() throws Exception { + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + Map>> expectedData = getExpectedData(); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), getAllowedTables(expectedData))) + .build(); + + LOG.info("Launching Dataflow job..."); + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + "oracle-data-types", + null, + null, + "oracle-datastream-to-spanner-data-types", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>(), + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck condition = buildConditionCheck(spannerResourceManager, expectedData); + LOG.info("Waiting for pipeline to process data..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES)), + condition); + assertThatResult(result).meetsConditions(); + + validateResult(spannerResourceManager, expectedData); + } + + private void validateResult( + SpannerResourceManager resourceManager, Map>> expectedData) { + for (Map.Entry>> entry : expectedData.entrySet()) { + String type = entry.getKey(); + String tableName = TABLE_PREFIX + type; + LOG.info("Asserting type: {}", type); + + List rows = + resourceManager.readTableRecords(tableName, entry.getValue().get(0).keySet()); + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(entry.getValue()); + } + } + + private List> createRows(String colName, Object... values) { + List vals = Arrays.asList(values); + List> rows = new ArrayList<>(); + for (int i = 0; i < vals.size(); i++) { + Map row = new HashMap<>(); + row.put("ID", i + 1L); + row.put(colName, vals.get(i)); + rows.add(row); + } + return rows; + } + + private List getAllowedTables(Map>> expectedData) { + List tableNames = new ArrayList<>(expectedData.size() + UNSUPPORTED_TYPE_TABLES.size()); + for (String tablePrefix : expectedData.keySet()) { + tableNames.add(TABLE_PREFIX + tablePrefix); + } + tableNames.addAll(UNSUPPORTED_TYPE_TABLES); + return tableNames; + } + + private ConditionCheck buildConditionCheck( + SpannerResourceManager resourceManager, Map>> expectedData) { + ConditionCheck combinedCondition = null; + for (Map.Entry>> entry : expectedData.entrySet()) { + String tableName = TABLE_PREFIX + entry.getKey(); + int numRows = entry.getValue().size(); + ConditionCheck c = + SpannerRowsCheck.builder(resourceManager, tableName).setMinRows(numRows).build(); + if (combinedCondition == null) { + combinedCondition = c; + } else { + combinedCondition = combinedCondition.and(c); + } + } + return combinedCondition; + } + + private Map>> getExpectedData() { + Map>> expectedData = new HashMap<>(); + expectedData.put("R0_VARCHAR2", createRows("COL_0", "NULL")); + expectedData.put("R1_VARCHAR", createRows("COL_1", "NULL")); + expectedData.put("R2_CHAR", createRows("COL_2", "NULL")); + expectedData.put("R3_CHARACTER", createRows("COL_3", "NULL")); + expectedData.put("R4_NVARCHAR2", createRows("COL_4", "NULL")); + expectedData.put("R5_NCHAR", createRows("COL_5", "NULL")); + expectedData.put("R6_NCHAR_VARY", createRows("COL_6", "NULL")); + expectedData.put("R7_NATIONAL_C", createRows("COL_7", "NULL")); + expectedData.put("R8_NATIONAL_C", createRows("COL_8", "NULL")); + expectedData.put("R9_NATIONAL_C", createRows("COL_9", "NULL")); + expectedData.put("R10_NATIONAL_C", createRows("COL_10", "NULL")); + expectedData.put("R11_NUMBER", createRows("COL_11", "NULL")); + expectedData.put("R12_NUMERIC", createRows("COL_12", "NULL")); + expectedData.put("R13_DECIMAL", createRows("COL_13", "NULL")); + expectedData.put("R14_DEC", createRows("COL_14", "NULL")); + expectedData.put("R15_FLOAT", createRows("COL_15", "NULL")); + expectedData.put("R16_DOUBLE_PRE", createRows("COL_16", "NULL")); + expectedData.put("R17_REAL", createRows("COL_17", "NULL")); + expectedData.put("R18_BINARY_FLO", createRows("COL_18", "NULL")); + expectedData.put("R19_BINARY_DOU", createRows("COL_19", "NULL")); + expectedData.put("R20_INTEGER", createRows("COL_20", "NULL")); + expectedData.put("R21_INT", createRows("COL_21", "NULL")); + expectedData.put("R22_SMALLINT", createRows("COL_22", "NULL")); + expectedData.put("R23_DATE", createRows("COL_23", "NULL")); + expectedData.put("R24_TIMESTAMP", createRows("COL_24", "NULL")); + expectedData.put("R25_TIMESTAMP_", createRows("COL_25", "NULL")); + expectedData.put("R26_TIMESTAMP_", createRows("COL_26", "NULL")); + expectedData.put("R29_RAW", createRows("COL_29", "NULL")); + expectedData.put("R31_BLOB", createRows("COL_31", "NULL")); + expectedData.put("R32_CLOB", createRows("COL_32", "NULL")); + expectedData.put("R33_NCLOB", createRows("COL_33", "NULL")); + + return expectedData; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java new file mode 100644 index 0000000000..9f44934ec7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -0,0 +1,352 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for sharded data migration using a single Dataflow job for Oracle Source targeting Spanner. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerSingleDFShardedMigrationIT extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerSingleDFShardedMigrationIT.class); + + private static final String TABLE = "Users"; + + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static final String ORACLE_SCHEMA_FILE_RESOURCE = + "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql"; + + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + public static DatastreamResourceManager datastreamResourceManager; + private static String streamNameA; + + @Before + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerSingleDFShardedMigrationIT.class) { + testInstances.add(this); + if (spannerResourceManager == null) { + spannerResourceManager = setUpSpannerResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + } + if (pubsubResourceManager == null) { + pubsubResourceManager = setUpPubSubResourceManager(); + } + if (gcsResourceManager == null) { + gcsResourceManager = setUpSpannerITGcsResourceManager(); + } + + if (jobInfo == null) { + + String oracleUser = System.getProperty("cloudProxyUsername", "system"); + String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + + jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} + + executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); + + jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + + datastreamResourceManager = + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = + org.apache.beam.it.gcp.datastream.OracleSource.builder( + System.getProperty("hostIp"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleUser, + java.util.List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), "oracle-shard-cdc/cdc/", org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_"+ org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); + + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + String shardConfig = + generateSourceConfig( + streamNameA, + oracleUser, + "L1" + ); + + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + String sessionFileContent = + com.google.common.io.Resources.toString(com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleDatastreamToSpannerSingleDFShardedMigrationIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager); + } + + @Test + public void multiShardMigration() throws Exception { + + // Check if pipeline is running + assertThatPipeline(jobInfo).isRunning(); + + insertDataInOracle(); + + // Setup condition + ConditionCheck rowsConditionCheck = + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + // Sleep for cutover time to wait till all CDCs propagate. + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + + // Assert specific rows + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + + LOG.info("Wait, executing Hard-boot raw JDBC switch logfile..."); + // Force log file archive - needed so Datastream can see changes which are read from archived log files. + // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + System.getProperty("hostIp") + ":1521 (User: system, Pass: TestPassword123) calling ALTER SYSTEM SWITCH LOGFILE. + try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XEPDB1", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + LOG.warn("Error while executing ALTER SYSTEM SWITCH LOGFILE. Using framework fallback...", e); + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "Tester1"); + row.put("age", 20); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Tester3"); + row.put("age", 103); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 13); + row.put("name", "Tester13"); + row.put("age", 113); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 4); + row.put("name", "Tester4"); + row.put("age", 104); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 5); + row.put("name", "Tester5"); + row.put("age", 105); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 6); + row.put("name", "Tester6"); + row.put("age", 106); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 7); + row.put("name", "Tester7"); + row.put("age", 107); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 8); + row.put("name", "Tester8"); + row.put("age", 108); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 9); + row.put("name", "Tester9"); + row.put("age", 109); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 10); + row.put("name", "Tester10"); + row.put("age", 110); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 11); + row.put("name", "Tester11"); + row.put("age", 111); + row.put("migration_shard_id", "L1"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 12); + row.put("name", "Tester12"); + row.put("age", 112); + row.put("migration_shard_id", "L1"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java new file mode 100644 index 0000000000..2aee8ce1a1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -0,0 +1,252 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static java.util.Map.entry; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Arrays; +import java.nio.charset.StandardCharsets; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerTableAndIndexLimitsIT extends DataStreamToSpannerITBase { + + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerTableAndIndexLimitsIT.class); + + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE = + "oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json"; + + private static final String LARGE_KEY_TABLE = "LargeKey"; + private static final String LARGE_CELL_TABLE = "LargeCell"; + private static final List TABLES = + List.of(LARGE_KEY_TABLE, LARGE_CELL_TABLE); + + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerTableAndIndexLimitsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + LOG.info("Setting up Oracle resource manager..."); + oracleResourceManager = setUpOracleResourceManager(); + LOG.info("Oracle resource manager created with URI: {}", oracleResourceManager.getUri()); + + try { oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeKey\" CASCADE CONSTRAINTS"); } catch (Exception e) {} + try { oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeCell\" CASCADE CONSTRAINTS"); } catch (Exception e) {} + + LOG.info("Setting up Spanner resource manager..."); + spannerResourceManager = setUpSpannerResourceManager(); + LOG.info( + "Spanner resource manager created with instance ID: {}", + spannerResourceManager.getInstanceId()); + LOG.info("Setting up GCS resource manager..."); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + LOG.info("GCS resource manager created with bucket: {}", gcsResourceManager.getBucket()); + LOG.info("Setting up Pub/Sub resource manager..."); + pubsubResourceManager = setUpPubSubResourceManager(); + LOG.info("Pub/Sub resource manager created."); + LOG.info("Setting up Datastream resource manager..."); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + LOG.info("Datastream resource manager created"); + + LOG.info("Executing Oracle DDL script..."); + executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); + + LOG.info("Adding supplemental logging to tables..."); + try { oracleResourceManager.runSQLUpdate("ALTER TABLE \"LargeKey\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} + try { oracleResourceManager.runSQLUpdate("ALTER TABLE \"LargeCell\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} + + LOG.info("Creating Spanner DDL..."); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // Pre-insert testing data for LargeCell + try (Connection conn = DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + PreparedStatement pstmt = conn.prepareStatement("INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\", \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { + + byte[] bytes = new byte[4000]; + Arrays.fill(bytes, (byte)'b'); + pstmt.setBytes(1, bytes); + pstmt.setString(2, new String(bytes, StandardCharsets.UTF_8)); + pstmt.executeUpdate(); + } catch (Exception e) { + LOG.error("Failed to insert LargeCell locally: ", e); + throw new RuntimeException("Failed to insert LargeCell", e); + } + + // Force a log switch and insert to LargeCell. + try (Connection conn = DriverManager.getConnection( + "jdbc:oracle:thin:@" + oracleResourceManager.getHost() + ":" + oracleResourceManager.getPort() + "/XEPDB1", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement(); ) { + flushOracleRedoLogs(null); + } catch (Exception e) { + LOG.warn("Failed to switch log file natively: ", e); + } + + LOG.info("Generating session file content..."); + String sessionFileContent = + generateSessionFile( + 1, + oracleResourceManager.getDatabaseName(), + spannerResourceManager.getDatabaseId(), + TABLES, + SESSION_FILE); + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), TABLES)) + .build(); + + LOG.info("Launching Dataflow job..."); + jobInfo = + launchDataflowJob( + "oracle-table-and-index-limits", + null, + null, + "datastream-to-spanner-table-and-index-limits", + spannerResourceManager, + pubsubResourceManager, + Map.of("dlqMaxRetryCount", "1", "inputFileFormat", "avro"), + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + oracleSource); + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + LOG.info("Cleaning up resources..."); + for (OracleDatastreamToSpannerTableAndIndexLimitsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void testKeySize() { + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data for key size test..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, LARGE_KEY_TABLE) + .setMinRows(1) + .build()); + assertThatResult(result).meetsConditions(); + List rows = null; + try { + rows = + spannerResourceManager.readTableRecords( + LARGE_KEY_TABLE, + List.of("pk_col1", "pk_col2")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from {}", LARGE_KEY_TABLE, e); + throw e; + } + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns( + List.of( + Map.ofEntries( + // `com.google.cloud.spanner.Value.StringImpl::valueToString`, which truncates + // strings to 33 characters + entry("pk_col1", "A".repeat(33) + "..."), + entry("pk_col2", "B".repeat(33) + "...")))); + } + + @Test + public void testCellSize() { + assertThatPipeline(jobInfo).isRunning(); + + LOG.info("Waiting for pipeline to process data for cell size test..."); + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(10)), + SpannerRowsCheck.builder(spannerResourceManager, LARGE_CELL_TABLE) + .setMinRows(1) + .build()); + assertThatResult(result).meetsConditions(); + List rows = null; + try { + rows = spannerResourceManager.readTableRecords(LARGE_CELL_TABLE, List.of("id")); + } catch (Exception e) { + LOG.error("Exception while reading spanner rows from {}", LARGE_CELL_TABLE, e); + throw e; + } + SpannerAsserts.assertThatStructs(rows) + .hasRecordsUnorderedCaseInsensitiveColumns(List.of(Map.ofEntries(entry("id", 1)))); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java new file mode 100644 index 0000000000..f783d6179d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java @@ -0,0 +1,332 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.beam.it.jdbc.JDBCResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT + extends DataStreamToSpannerITBase { + private static final int STRING_LENGTH = 200; + private static final Integer NUM_EVENTS = 1; + private static final Integer NUM_TABLES = 1; + + private static final String ROW_ID = "ROW_ID"; + private static final String NAME = "NAME"; + private static final String AGE = "AGE"; + private static final String MEMBER = "MEMBER"; + private static final String ENTRY_ADDED = "ENTRY_ADDED"; + private static final String LARGE_BLOB_ADDED = "LARGE_BLOB"; + + private static final List COLUMNS = + List.of(ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB_ADDED); + + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static GcsResourceManager gcsResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static HashSet testInstances = + new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + private static final List TABLE_NAMES = new ArrayList<>(); + + static { + for (int i = 1; i <= NUM_TABLES; i++) { + TABLE_NAMES.add("DATASTREAM_WIDETB_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + } + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + oracleResourceManager = setUpOracleResourceManager(); + String sessionContent = generateBaseSchema(); + sessionContent = sessionContent + .replaceAll("SRC_DATABASE", oracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + for (int i = 1; i <= NUM_TABLES; i++) { + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + } + setupSchema(); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleDataStreamToSpannerWideRowFor9MibTablePerDatabaseIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>(), + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionContent, + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) + .build()); + } + } + } + + @After + public void cleanUp() throws IOException { + for (OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + datastreamResourceManager, + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager); + } + + private void setupSchema() { + TABLE_NAMES.forEach( + tableName -> { + oracleResourceManager.createTable(tableName, createJdbcSchema()); + oracleResourceManager.runSQLUpdate( + "ALTER TABLE " + tableName + " ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + }); + createSpannerTables(); + } + + @Test + public void testDataStreamOracleToSpannerFor9MBTablesPerDatabase() throws IOException { + assertThatPipeline(jobInfo).isRunning(); + Map>> cdcEvents = new LinkedHashMap<>(); + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeJdbcData(cdcEvents), + SpannerRowsCheck.builder(spannerResourceManager, TABLE_NAMES.get(0)) + .setMinRows(NUM_EVENTS) + .build(), + checkDestinationRows(cdcEvents))) + .build(); + + // Job needs to be cancelled as draining will time out + PipelineOperator.Result result = + pipelineOperator() + .waitForConditionAndCancel( + createConfig(jobInfo, Duration.ofMinutes(20)), conditionCheck); + + // Assert + assertThatResult(result).meetsConditions(); + } + + private String generateBaseSchema() throws IOException { + Map sessionTemplate = createSessionTemplate(); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(sessionTemplate); + } + + private Map createSessionTemplate() { + List colIds = Arrays.asList("c1", "c2", "c3", "c4", "c5", "c6"); + List> colTypeConfigs = new ArrayList<>(); + for (int j = 1; j <= colIds.size(); j++) { + Map colType = new LinkedHashMap<>(); + if (j == 6) { + colType.put("Type", "BLOB"); + colType.put("Len", 0); + } else if (j % 2 == 0) { + colType.put("Type", "VARCHAR2"); + colType.put("Len", STRING_LENGTH); + } else { + colType.put("Type", "NUMBER"); + colType.put("Len", 0); + } + colType.put("IsArray", false); + colType.put("Name", "column_" + j); + colType.put("NotNull", (j == 1)); + colType.put("Comment", "From: column_" + j + " " + colType.get("Type")); + colTypeConfigs.add(colType); + } + List> primaryKeys = + List.of(Map.of("ColId", colIds.get(0), "Desc", false, "Order", 1)); + Map sessionTemplate = createSessionTemplate(NUM_TABLES, colTypeConfigs, primaryKeys); + sessionTemplate.put("DatabaseType", "oracle"); + return sessionTemplate; + } + + private JDBCResourceManager.JDBCSchema createJdbcSchema() { + HashMap columns = new HashMap<>(); + columns.put(ROW_ID, "NUMBER NOT NULL"); + columns.put(NAME, "VARCHAR2(200)"); + columns.put(AGE, "NUMBER"); + columns.put(MEMBER, "VARCHAR2(200)"); + columns.put(ENTRY_ADDED, "VARCHAR2(200)"); + columns.put(LARGE_BLOB_ADDED, "BLOB"); + return new JDBCResourceManager.JDBCSchema(columns, ROW_ID); + } + + private void createSpannerTables() { + TABLE_NAMES.forEach( + tableName -> + spannerResourceManager.executeDdlStatement( + "CREATE TABLE IF NOT EXISTS " + + tableName + + " (" + + ROW_ID + + " INT64 NOT NULL, " + + NAME + + " STRING(1024), " + + AGE + + " INT64, " + + MEMBER + + " STRING(1024), " + + ENTRY_ADDED + + " STRING(1024), " + + LARGE_BLOB_ADDED + + " BYTES(9437184)" // 9 MiB + + ") PRIMARY KEY (" + + ROW_ID + + ")")); + } + + private ConditionCheck checkDestinationRows(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Check Spanner rows."; + } + + @Override + protected CheckResult check() { + for (String tableName : TABLE_NAMES) { + long totalRows = spannerResourceManager.getRowCount(tableName); + long maxRows = cdcEvents.get(tableName).size(); + if (totalRows > maxRows) { + return new CheckResult( + false, String.format("Expected up to %d rows but found %d", maxRows, totalRows)); + } + } + return new CheckResult(true, "Spanner tables contain expected rows."); + } + }; + } + + private ConditionCheck writeJdbcData(Map>> cdcEvents) { + return new ConditionCheck() { + @Override + protected String getDescription() { + return "Send initial JDBC events."; + } + + @Override + protected CheckResult check() { + boolean success = true; + List messages = new ArrayList<>(); + Random random = new Random(); + byte[] largeData = new byte[9 * 1024 * 1024]; + random.nextBytes(largeData); + + for (String tableName : TABLE_NAMES) { + List> rows = new ArrayList<>(); + + for (int i = 0; i < NUM_EVENTS; i++) { + Map values = new LinkedHashMap<>(); + values.put(COLUMNS.get(0), i); + values.put(COLUMNS.get(1), RandomStringUtils.randomAlphabetic(10)); + values.put(COLUMNS.get(2), random.nextInt(100)); + values.put(COLUMNS.get(3), random.nextBoolean() ? "Y" : "N"); + values.put(COLUMNS.get(4), Instant.now().toString()); + byte[] rowData = largeData.clone(); + values.put(COLUMNS.get(5), rowData); + rows.add(values); + } + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), oracleResourceManager.getUsername(), oracleResourceManager.getPassword()); + java.sql.PreparedStatement pstmt = conn.prepareStatement( + "INSERT INTO " + tableName + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?, ?, ?, ?)")) { + for (Map values : rows) { + pstmt.setLong(1, ((Number) values.get(COLUMNS.get(0))).longValue()); + pstmt.setString(2, (String) values.get(COLUMNS.get(1))); + pstmt.setLong(3, ((Number) values.get(COLUMNS.get(2))).longValue()); + pstmt.setString(4, (String) values.get(COLUMNS.get(3))); + pstmt.setString(5, (String) values.get(COLUMNS.get(4))); + pstmt.setBytes(6, (byte[]) values.get(COLUMNS.get(5))); + pstmt.addBatch(); + } + pstmt.executeBatch(); + } catch (Exception e) { + success = false; + messages.add("Failed to write to " + tableName + ": " + e.getMessage()); + } + + rows.forEach( + values -> + values.put( + COLUMNS.get(5), + Base64.getEncoder().encodeToString((byte[]) values.get(COLUMNS.get(5))))); + + cdcEvents.put(tableName, rows); + messages.add(String.format("%d rows to %s", rows.size(), tableName)); + } + + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + } + }; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java new file mode 100644 index 0000000000..0c16372671 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -0,0 +1,671 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck.CheckResult; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for datatype conversions for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseDDLIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseDDLIT.class); + + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static final String TABLE1 = "AllDatatypeColumns"; + private static final String TABLE2 = "AllDatatypeColumns2"; + private static final String TABLE3 = "DatatypeColumnsWithSizes"; + private static final String TABLE4 = "DatatypeColumnsReducedSizes"; + private static final String TABLE5 = "Users"; + private static final String TABLE6 = "Books"; + private static final String TABLE7 = "Authors"; + private static final String TABLE8 = "Singers"; + + private static final String TRANSFORMATION_TABLE = "AllDatatypeTransformation"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; + + + private void flushOracleLogs() { + try { + Class.forName("oracle.jdbc.OracleDriver"); + try (Connection conn = DriverManager.getConnection("jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + } catch (Exception e) { + throw new RuntimeException("Failed to switch logfile", e); + } + } + + @Before + + public void setUp() throws IOException, InterruptedException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseDDLIT.class) { + testInstances.add(this); + if (jobInfo == null) { + oracleResourceManager = setUpOracleResourceManager(); + + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + createAndUploadJarToGcs("OracleSeparateShadowTableDatabaseDDLIT", gcsResourceManager); + CustomTransformation customTransformation = + CustomTransformation.builder( + "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") + .build(); + + String[] tables = {"AllDatatypeColumns", "AllDatatypeColumns2", "DatatypeColumnsWithSizes", "DatatypeColumnsReducedSizes", "Users", "Authors", "AllDatatypeTransformation", "Singers", "Books"}; + for (String table : tables) { + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); + } catch (Exception e) { + } +executeSqlScript(oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql"); + flushOracleLogs(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseDDLIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "avro"); + put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + } + }, + customTransformation, // customTransformation + null, // shardingContextFileResourceName + gcsResourceManager, + datastreamResourceManager, + null, // sessionResourceContent + org.apache.beam.it.gcp.datastream.OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables(java.util.Map.of(oracleResourceManager.getUsername().toUpperCase(), java.util.Arrays.asList(TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + .build()); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseDDLIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, shadowSpannerResourceManager, gcsResourceManager); + } + + @Test + public void migrationTestWithAllDatatypeConversionMapping() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Run SQL inserts"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + executeSqlScript(oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql"); + executed = true; // Repositioned ABOVE flushOracleLogs + flushOracleLogs(); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Success"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(15)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertAllDatatypeColumnsTableBackfillContents(); + } + + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + // Only asserting primary key + simple int column to verify end-to-end sync without complex base64 logic + row.put("int_column", "50000"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, int_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("date_column", "2024-02-08T00:00:00Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("decimal_column", "456.12"); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + // text, char, tinytext, mediumtext, longtext are BYTE columns + row.put("text_column", "/u/9n58P"); + row.put("char_column", "v58P"); + row.put("tinytext_column", "7+/+7/2fnw8="); + row.put("mediumtext_column", "/+3v79/v2vrx"); + row.put("longtext_column", "/+/v3+/a+vE="); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f3100000000000000000..."); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column" + + ", tinyblob_column, blob_column, mediumblob_column" + + ", longblob_column, enum_column, bool_column, other_bool_column" + + ", varbinary_column, bit_column, decimal_column, text_column, binary_column" + + ", char_column, tinytext_column, mediumtext_column, longtext_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("decimal_column", 123.45); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "15:30:45"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" + + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 21); + row.put("text_column", "Some text 2 append"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 201); + row.put("bigint_column", 987655); + row.put("float_column", 24.45); + row.put("double_column", 235.567); + row.put("decimal_column", 23457.78); + row.put("datetime_column", "2022-12-31T23:59:58Z"); + row.put("timestamp_column", "2022-12-31T23:59:58Z"); + row.put("time_column", "00:59:59"); + row.put("year_column", "2023"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 31); + row.put("text_column", "Some text 3 append"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 301); + row.put("bigint_column", 112234); + row.put("float_column", 35.56); + row.put("double_column", 346.678); + row.put("decimal_column", 34568.89); + row.put("datetime_column", "2023-12-31T23:59:59Z"); + row.put("timestamp_column", "2023-12-31T23:59:59Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example4"); + row.put("tinyint_column", 41); + row.put("text_column", "Some text 4 append"); + row.put("date_column", "2021-11-12"); + row.put("int_column", 401); + row.put("bigint_column", 223345); + row.put("float_column", 46.67); + row.put("double_column", 457.789); + row.put("decimal_column", 45679.90); + row.put("datetime_column", "2021-11-11T11:11:10Z"); + row.put("timestamp_column", "2021-11-11T11:11:10Z"); + row.put("time_column", "12:11:11"); + row.put("year_column", "2022"); + row.put("blob_column", "V29ybWQ="); + row.put("enum_column", "1"); + row.put("bool_column", true); + row.put("binary_column", "AQIDBAUGBwgJCgsMDQ4PEBESExQ="); + row.put("bit_column", "DQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, binary_column, bit_column FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeTransformationTableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "example2"); + row.put("tinyint_column", 25); + row.put("text_column", "Updated text 2"); + row.put("date_column", "2023-01-01"); + row.put("int_column", 250); + row.put("bigint_column", 56789); + row.put("float_column", 25.45); + row.put("double_column", 345.678); + row.put("decimal_column", 23456.79); + row.put("datetime_column", "2023-01-01T12:00:00Z"); + row.put("timestamp_column", "2023-01-01T12:00:00Z"); + row.put("time_column", "12:00:00"); + row.put("year_column", "2023"); + row.put("blob_column", "EjRWeJCrze8="); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "EjRWeJCrze8SNFZ4kKvN7xI0Vng="); + row.put("bit_column", "fw=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "example3"); + row.put("tinyint_column", 35); + row.put("text_column", "Updated text 3"); + row.put("date_column", "2024-01-02"); + row.put("int_column", 350); + row.put("bigint_column", 88000); + row.put("float_column", 35.67); + row.put("double_column", 456.789); + row.put("decimal_column", 34567.90); + row.put("datetime_column", "2024-01-02T00:00:00Z"); + row.put("timestamp_column", "2024-01-02T00:00:00Z"); + row.put("time_column", "01:00:00"); + row.put("year_column", "2025"); + row.put("blob_column", "q83vEjRWeJA="); + row.put("enum_column", "1"); + row.put("bool_column", false); + row.put("binary_column", "q83vEjRWeJCrze8SNFZ4kKvN7xI="); + row.put("bit_column", "AA=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("SELECT * FROM AllDatatypeTransformation")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("decimal_column", 456.12); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "08:15:30"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" + + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDataTypeSizeConversionBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("float_column", 45.67); + row.put("decimal_column", 456.12); + row.put("char_column", "char_1"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "Zg=="); + events.add(row); + + row = new HashMap<>(); + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column" + + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsWithSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOException { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + + row.put("varchar_column", "value2"); + row.put("float_column", 12.34); + row.put("decimal_column", 123.45); + row.put("char_column", "char_2"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "GQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, float_column, decimal_column, char_column, bool_column" + + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsReducedSizes")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertUsersBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("user_id", 1); + row.put("first_name", "Lorem"); + row.put("last_name", "Epsum"); + row.put("full_name", "Lorem Epsum"); + row.put("age", 20); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 2); + row.put("first_name", "Jane"); + row.put("last_name", "Doe"); + row.put("full_name", "Jane Doe"); + row.put("age", 21); + events.add(row); + + row = new HashMap<>(); + row.put("user_id", 9); + row.put("first_name", "James"); + row.put("last_name", "Dove"); + row.put("full_name", "James Dove"); + row.put("age", 22); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select user_id, first_name, last_name, full_name, age from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAuthorsBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("name", "J.R.R. Tolkien"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("name", "Jane Austen"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("name", "Douglas Adams"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, name from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhiker's Guide to the Galaxy"); + events.add(row); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java new file mode 100644 index 0000000000..a91e16610b --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -0,0 +1,420 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.Date; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseEventsIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Users"; + private static final String TABLE2 = "Movie"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseEventsIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + // SYSTEM AUTHORIZATIONS + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Movie", "Users", "Authors", "Articles", "Books"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseEventsIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + + // Supplement logging requirement + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseEventsIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); + } + + @Test + public void migrationTestWithUpdatesAndDeletes() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeUsersInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build(), + writeUsersNextData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertUsersTableContents(); + } + + @Test + public void migrationTestWithInsertsOnly() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeMovieInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertMovieTableContents(); + } + + @Test + public void interleavedAndFKAndIndexTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeArticlesInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Articles") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTable(); + assertBooksTable(); + assertArticlesTable(); + } + + private ConditionCheck writeUsersInitialData() { + return new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Write initial Users data"; } + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent initial Users data"); + } + }; + } + + private ConditionCheck writeUsersNextData() { + return new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Write next Users data"; } + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent next Users data"); + } + }; + } + + private ConditionCheck writeMovieInitialData() { + return new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Write Movie data"; } + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1', TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2', TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent Movie data"); + } + }; + } + + private ConditionCheck writeArticlesInitialData() { + return new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Write Articles data"; } + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (1, 'a1')"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (2, 'a2')"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (3, 'a3')"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (4, 'a4')"); + + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); + + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (1, 'Book005', 3)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (2, 'Book002', 3)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (3, 'Book004', 4)"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (4, 'Book005', 2)"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + executed = true; + } + return new CheckResult(true, "Sent Articles data"); + } + }; + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester Kumar", "age", 30, "subscribed", false, "plan", "A", "startDate", Date.parseDate("2023-01-01"))); + events.add(Map.of("id", 3, "name", "Tester Gupta", "age", 50, "subscribed", false, "plan", "Z", "startDate", Date.parseDate("2023-06-07"))); + events.add(Map.of("id", 4, "name", "Tester", "age", 38, "subscribed", true, "plan", "D", "startDate", Date.parseDate("2023-09-10"))); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertMovieTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "movie1", "startTime", Timestamp.parseTimestamp("2023-01-01T12:12:12Z"))); + events.add(Map.of("id", 2, "name", "movie2", "startTime", Timestamp.parseTimestamp("2023-11-25T17:10:12Z"))); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, name, startTime from Movie where id in (1, 2)")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + + ImmutableList numericVals = + spannerResourceManager.runQuery("select actor from Movie order by id"); + Assert.assertEquals(12345.09876, numericVals.get(0).getBigDecimal(0).doubleValue(), 0.00000001); + Assert.assertEquals(931.5123, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.00000001); + } + + private void assertAuthorsTable() { + List> events = new ArrayList<>(); + events.add(Map.of("author_id", 1, "name", "a1")); + events.add(Map.of("author_id", 2, "name", "a2")); + events.add(Map.of("author_id", 3, "name", "a3")); + events.add(Map.of("author_id", 4, "name", "a4")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksTable() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "title", "Book005", "author_id", 3)); + events.add(Map.of("id", 2, "title", "Book002", "author_id", 3)); + events.add(Map.of("id", 3, "title", "Book004", "author_id", 4)); + events.add(Map.of("id", 4, "title", "Book005", "author_id", 2)); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertArticlesTable() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Article001", "published_date", Date.parseDate("2024-01-01"), "author_id", 1)); + events.add(Map.of("id", 2, "name", "Article002", "published_date", Date.parseDate("2024-01-01"), "author_id", 1)); + events.add(Map.of("id", 3, "name", "Article004", "published_date", Date.parseDate("2024-01-01"), "author_id", 4)); + events.add(Map.of("id", 4, "name", "Article005", "published_date", Date.parseDate("2024-01-01"), "author_id", 3)); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java new file mode 100644 index 0000000000..ffc78ea637 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -0,0 +1,273 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.com.google.common.io.Resources; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseFileOverridesIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseFileOverridesIT.class); + + private static final String ORACLE_SCHEMA = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json"; + private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager shadowSpannerResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = new HashSet<>(); + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseFileOverridesIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + gcsResourceManager.uploadArtifact( + GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); + + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseFileOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() throws Exception { + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + overridesMap.put( + "schemaOverridesFilePath", + getGcsPath(GCS_PATH_PREFIX + "/override.json", gcsResourceManager)); + + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck sendDataCondition = new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } + } + }; + + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + sendDataCondition, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java new file mode 100644 index 0000000000..e98e65bf67 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -0,0 +1,277 @@ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseMixedIT extends DataStreamToSpannerITBase { + private static final String SESSION_FILE_RESOURCE = "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json"; + + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseMixedIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql"); + + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudSqlResourceManager.getUsername().toUpperCase(), + List.of("Authors", "Books", "Genre"))) + .build(); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSeparateShadowTableDatabaseMixedIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + put("inputFileFormat", "avro"); + put("datastreamSourceType", "oracle"); + } + }, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseMixedIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); + } + + @Test + public void mixedMigrationTest() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialData(), + SpannerRowsCheck.builder(spannerResourceManager, "Authors").setMinRows(4).setMaxRows(4).build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books").setMinRows(3).setMaxRows(3).build(), + SpannerRowsCheck.builder(spannerResourceManager, "Genre").setMinRows(1).setMaxRows(1).build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + assertAuthorsTableContents(); + assertBooksTableContents(); + } + + private ConditionCheck writeInitialData() { + return new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { + return "Write JDBC data"; + } + + @Override + protected CheckResult check() { + if (!executed) { + + List> authorRows = new ArrayList<>(); + authorRows.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + + List> bookRows = new ArrayList<>(); + bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); + bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); + bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); + + List> genreRows = new ArrayList<>(); + genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); + + for (Map r : authorRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", r.get("author_id"), r.get("full_name"))); + } + for (Map r : bookRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", r.get("id"), r.get("title"), r.get("author_id"))); + } + for (Map r : genreRows) { + cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", r.get("genre_id"), r.get("name"))); + } + + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + executed = true; + } + return new CheckResult(true, "Sent to Oracle."); + } + }; + } + + private void assertAuthorsTableContents() { + List> authorEvents = new ArrayList<>(); + authorEvents.add(Map.of("author_id", 4, "full_name", "Stephen King")); + authorEvents.add(Map.of("author_id", 1, "full_name", "Jane Austen")); + authorEvents.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); + authorEvents.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Authors")) + .hasRecordsUnorderedCaseInsensitiveColumns(authorEvents); + } + + private void assertBooksTableContents() { + List> bookEvents = new ArrayList<>(); + bookEvents.add(Map.of("id", 1, "title", "Pride and Prejudice")); + bookEvents.add(Map.of("id", 2, "title", "Oliver Twist")); + bookEvents.add(Map.of("id", 3, "title", "War and Peace")); + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, title from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(bookEvents); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java new file mode 100644 index 0000000000..865bd2a39d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -0,0 +1,439 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.com.google.common.io.Resources; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabasePKFocusedIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabasePKFocusedIT.class); + + private static final String ORACLE_SCHEMA = "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String GCS_PATH_PREFIX = "OracleSeparateShadowTableDatabasePKFocusedIT"; + + // Oracle defaults to uppercase unquoted identifiers + private static final String ORACLE_TABLE_MY_TABLE = "MY_TABLE"; + private static final String ORACLE_TABLE_ALLTYPES = "ALLTYPES"; + private static final String SPANNER_TABLE_MY_TABLE = "MY_TABLE"; + private static final String SPANNER_TABLE_ALLTYPES = "ALLTYPES"; + + private static CloudOracleResourceManager oracleSysUser; + private static CloudOracleResourceManager oracleResourceManager; + private static SpannerResourceManager shadowSpannerResourceManager; + private static SpannerResourceManager spannerResourceManager; + private static GcsResourceManager gcsResourceManager; + private static PubsubResourceManager pubsubResourceManager; + private static DatastreamResourceManager datastreamResourceManager; + private static boolean initialized = false; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + private void setUpOracleUser(String user, String password) { + oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + } + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabasePKFocusedIT.class) { + testInstances.add(this); + if (!initialized) { + LOG.info("Setting up Oracle sys resource manager..."); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setPort(1521); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setDatabaseName("XE"); + } + oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + + LOG.info("Provisioning isolated user: " + oracleUser); + setUpOracleUser(oracleUser, oraclePassword); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + if (System.getProperty("hostIp") != null) { + builder.setPassword(oraclePassword); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setUsername(oracleUser); + builder.setDatabaseName("XEPDB1"); + } + oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + spannerResourceManager = setUpSpannerResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity("datastream-connect-2") + .build(); + + executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE_MY_TABLE, ORACLE_TABLE_ALLTYPES))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + + assertThatPipeline(jobInfo).isRunning(); + initialized = true; + } + } + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabasePKFocusedIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + oracleResourceManager, + spannerResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + pubsubResourceManager, + datastreamResourceManager); + } + + private void flushOracleLogs() { + try (java.sql.Connection conn = java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XE", "system", System.getProperty("cloudProxyPassword")); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + LOG.warn("Log switch failed. Continuing...", e); + } + } + + @Test + public void migrationTestSimpleTable() throws Exception { + ConditionCheck insertRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "Insert 3 records into MY_TABLE"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (1, 10)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (2, 20)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (3, 30)"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "Update and Delete in MY_TABLE"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET VAL = 10 WHERE ID = 2"); + oracleResourceManager.runSQLUpdate("DELETE FROM MY_TABLE WHERE ID = 1"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "PK Update in MY_TABLE"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET ID = 10 WHERE ID = 3"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data PK updated"); + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_MY_TABLE) + .setMinRows(3) + .setMaxRows(3) + .build(), + updateDeleteRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_MY_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build(), + pkUpdateRecords, + new ConditionCheck() { + @Override + public String getDescription() { return "Check if MY_TABLE PK updated to 10"; } + @Override + protected CheckResult check() { + try { + com.google.common.collect.ImmutableList rows = spannerResourceManager.runQuery("SELECT ID FROM MY_TABLE WHERE ID = 10"); + if (!rows.isEmpty()) { + return new CheckResult(true, "PK updated to 10"); + } + } catch (Exception e) {} + return new CheckResult(false, "Not yet"); + } + })) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + try { Thread.sleep(CUTOVER_MILLIS); } catch (Exception e) {} + + List> events = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("ID", 2L); + row1.put("VAL", 10L); + Map row2 = new HashMap<>(); + row2.put("ID", 10L); + row2.put("VAL", 30L); + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select ID, VAL from MY_TABLE")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + @Test + public void migrationTestAllDataType() throws Exception { + ConditionCheck insertRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "Insert records into ALLTYPES"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + // 1 + oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 1, 3.14, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 10)"); + // 2 + oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 20)"); + // 3 + oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 30)"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "Update and Delete in ALLTYPES"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("DELETE FROM ALLTYPES WHERE INT64_FIELD = 1"); + oracleResourceManager.runSQLUpdate("UPDATE ALLTYPES SET VAL = 10 WHERE INT64_FIELD = 2"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = new ConditionCheck() { + boolean executed = false; + @Override + public String getDescription() { return "Update PK in ALLTYPES"; } + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE ALLTYPES SET INT64_FIELD = 10 WHERE INT64_FIELD = 3"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { return new CheckResult(false, e.getMessage()); } + } + return new CheckResult(true, "Data PK updated"); + } + }; + + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + insertRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_ALLTYPES) + .setMinRows(3) + .setMaxRows(3) + .build(), + updateDeleteRecords, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE_ALLTYPES) + .setMinRows(2) + .setMaxRows(2) + .build(), + pkUpdateRecords, + new ConditionCheck() { + @Override + public String getDescription() { return "Check if ALLTYPES PK updated to 10"; } + @Override + protected CheckResult check() { + try { + com.google.common.collect.ImmutableList rows = spannerResourceManager.runQuery("SELECT INT64_FIELD FROM ALLTYPES WHERE INT64_FIELD = 10"); + if (!rows.isEmpty()) { + return new CheckResult(true, "PK updated to 10"); + } + } catch (Exception e) {} + return new CheckResult(false, "Not yet"); + } + })) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + try { Thread.sleep(CUTOVER_MILLIS); } catch (Exception e) {} + + List> events = new ArrayList<>(); + Map r1 = new HashMap<>(); + r1.put("INT64_FIELD", 2L); + r1.put("VAL", 10L); + Map r2 = new HashMap<>(); + r2.put("INT64_FIELD", 10L); + r2.put("VAL", 30L); + events.add(r1); + events.add(r2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select INT64_FIELD, VAL from ALLTYPES")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java new file mode 100644 index 0000000000..01efebed2b --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -0,0 +1,411 @@ +/* GENERATED BY: Test Automation Skill */ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * An integration test using separate shadow table database for {@link DataStreamToSpanner} Flex + * template which tests use-cases where a session file is required. + */ +@Category({TemplateIntegrationTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseSessionIT extends DataStreamToSpannerITBase { + + private static final String TABLE1 = "Category"; + private static final String TABLE2 = "Books"; + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudOracleResourceManager; + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static String oracleUser; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql"; + private static final String ORACLE_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json"; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseSessionIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + cloudOracleResourceManager = setUpOracleResourceManager(); + try { cloudOracleResourceManager.runSQLUpdate(String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE1)); } catch (Exception e) {} + try { cloudOracleResourceManager.runSQLUpdate(String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE2)); } catch (Exception e) {} + executeSqlScript(cloudOracleResourceManager, ORACLE_DDL_RESOURCE); + cloudOracleResourceManager.runSQLUpdate(String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE1)); + cloudOracleResourceManager.runSQLUpdate(String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE2)); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + OracleSource jdbcSource = + OracleSource.builder( + cloudOracleResourceManager.getHost(), + cloudOracleResourceManager.getUsername(), + cloudOracleResourceManager.getPassword(), + cloudOracleResourceManager.getPort(), + cloudOracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of( + cloudOracleResourceManager.getUsername().toUpperCase(), + List.of(TABLE1, TABLE2))) + .build(); + + Map jobParams = new HashMap<>(); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "OracleSeparateShadowTableDatabaseSessionIT", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + public static void setUpOracleUser(CloudOracleResourceManager sysUser, String user, String password) { + sysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + sysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + + // Add c##datastream grants based on requirements + sysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL")); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseSessionIT instance : testInstances) { + try { + instance.tearDownBase(); + } catch (Exception e) { + // Ignore UnsupportedOperationException from Thread.stop() on Java 21+ + } + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + cloudOracleSysUser, + cloudOracleResourceManager, + datastreamResourceManager, + gcsResourceManager); + } + + @Test + public void migrationTestWithRenameAndDrops() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeInitialCategoryData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertCategoryTableBackfillContents(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeCdcCategoryData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + assertThatResult(result).meetsConditions(); + + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (InterruptedException e) { + } + assertCategoryTableCdcContents(); + } + + @Test + public void migrationTestWithSyntheticPKAndExtraColumn() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + writeBooksData(), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertBooksBackfillContents(); + } + + private ConditionCheck writeInitialCategoryData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { return "Write initial Category rows"; } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent category initial"); + } + try { + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (1, 'xyz')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (2, 'abc')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category initial"); + } catch(Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private ConditionCheck writeCdcCategoryData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { return "Write cdc Category rows"; } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent category cdc"); + } + try { + cloudOracleResourceManager.runSQLUpdate(String.format("DELETE FROM \"%s\" WHERE \"category_id\" = 1", TABLE1)); + cloudOracleResourceManager.runSQLUpdate(String.format("UPDATE \"%s\" SET \"full_name\" = 'abc1' WHERE \"category_id\" = 2", TABLE1)); + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (3, 'def')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (4, 'ghi')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category cdc"); + } catch(Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private ConditionCheck writeBooksData() { + return new ConditionCheck() { + private boolean executed = false; + + @Override + protected String getDescription() { return "Write Books rows"; } + + @Override + protected CheckResult check() { + if (executed) { + return new CheckResult(true, "Sent books"); + } + try { + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of the Rings', 1)", TABLE2)); + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and Prejudice', 2)", TABLE2)); + cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The Hitchhikers Guide to the Galaxy', 3)", TABLE2)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent books"); + } catch(Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + }; + } + + private void assertCategoryTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 1); + row1.put("full_name", "xyz"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 2); + row2.put("full_name", "abc"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableCdcContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 2); + row1.put("full_name", "abc1"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 3); + row2.put("full_name", "def"); + + Map row3 = new HashMap<>(); + row3.put("category_id", 4); + row3.put("full_name", "ghi"); + + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertBooksBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("id", 1); + row.put("title", "The Lord of the Rings"); + row.put("author_id", 1); + events.add(row); + + row = new HashMap<>(); + row.put("id", 2); + row.put("title", "Pride and Prejudice"); + row.put("author_id", 2); + events.add(row); + + row = new HashMap<>(); + row.put("id", 3); + row.put("title", "The Hitchhikers Guide to the Galaxy"); + row.put("author_id", 3); + events.add(row); + + ImmutableList synthIds = spannerResourceManager.runQuery("select synth_id from Books"); + + Assert.assertEquals(3, synthIds.size()); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select id, title, author_id from Books")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java new file mode 100644 index 0000000000..3301de1711 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -0,0 +1,301 @@ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class); + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql"); + try { jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} + + OracleSource jdbcSource = OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = "oracle-shard-cdc_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), uniqueCdcPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( + "test_stream_"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); + if (!fileLocal.exists()) { fileLocal = new java.io.File("v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); } + String sessionFileContent = new String(java.nio.file.Files.readAllBytes(fileLocal.toPath()), java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Insert Data into Oracle"; } + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build() + ) + ).build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20, "migration_shard_id", "L1")); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103, "migration_shard_id", "L1")); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113, "migration_shard_id", "L1")); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104, "migration_shard_id", "L1")); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105, "migration_shard_id", "L1")); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106, "migration_shard_id", "L1")); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107, "migration_shard_id", "L1")); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108, "migration_shard_id", "L1")); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109, "migration_shard_id", "L1")); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110, "migration_shard_id", "L1")); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111, "migration_shard_id", "L1")); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112, "migration_shard_id", "L1")); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java new file mode 100644 index 0000000000..a489684de8 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -0,0 +1,299 @@ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class); + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setSystemIdentifier("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); + String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); + setUpOracleUser(oracleUser, oraclePassword); + + jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql"); + + OracleSource jdbcSource = OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = "oracle-shard-cdc_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), uniqueCdcPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( + "test_stream_"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); + if (!fileLocal.exists()) { fileLocal = new java.io.File("v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); } + String sessionFileContent = new String(java.nio.file.Files.readAllBytes(fileLocal.toPath()), java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "shard1", + spannerResourceManager, + null, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + null); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + org.apache.beam.it.conditions.ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Insert Data into Oracle"; } + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build() + ) + ).build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20)); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103)); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113)); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104)); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105)); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106)); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107)); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108)); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109)); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110)); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111)); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112)); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java new file mode 100644 index 0000000000..2900e3262c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -0,0 +1,298 @@ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class); + + + private static final String TABLE = "Users"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager jdbcResourceManagerShardA; + private static String streamNameA; + + @Before + public void setUp() throws Exception { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setDatabaseName("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudProxyUsername", "system")) + .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} + + try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} + executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql"); + + OracleSource jdbcSource = OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); + + com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", gcsResourceManager.getBucket(), "oracle-shard-cdc/cdc/", DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( + "test-stream-"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.startStream(stream); + streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); + + gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, "system", "L1")); + + Map jobParams = new HashMap<>(); + jobParams.put("inputFileFormat", "avro"); + jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put("datastreamSourceType", "oracle"); + jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + if (System.getProperty("jdbcDriverJars") != null) { + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); + } + + String sessionFileContent = + com.google.common.io.Resources.toString(com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = sessionFileContent.replace("it_test", "system").replace("shard_1", "L1"); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName() + "shard1", + null, + null, + "OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT_shard1", + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager, + datastreamResourceManager, + sessionFileContent, + jdbcSource); + + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + } + + @Test + public void multiShardMigration() throws Exception { + assertThatPipeline(jobInfo).isRunning(); + + ConditionCheck rowsConditionCheck = + org.apache.beam.it.conditions.ChainedConditionCheck.builder( + List.of( + new ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { return "Insert Data into Oracle"; } + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, TABLE) + .setMinRows(12) + .setMaxRows(12) + .build() + ) + ).build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); + + assertThatResult(result).meetsConditions(); + + assertUsersTableContents(); + } + + private void insertDataInOracle() { + LOG.info("Inserting rows into Users table in Oracle"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); + + try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } catch (Exception e) { + flushOracleRedoLogs(jdbcResourceManagerShardA); + } + } + + private void assertUsersTableContents() { + List> events = new ArrayList<>(); + events.add(Map.of("id", 1, "name", "Tester1", "age", 20, "migration_shard_id", "L1")); + events.add(Map.of("id", 3, "name", "Tester3", "age", 103, "migration_shard_id", "L1")); + events.add(Map.of("id", 13, "name", "Tester13", "age", 113, "migration_shard_id", "L1")); + events.add(Map.of("id", 4, "name", "Tester4", "age", 104, "migration_shard_id", "L1")); + events.add(Map.of("id", 5, "name", "Tester5", "age", 105, "migration_shard_id", "L1")); + events.add(Map.of("id", 6, "name", "Tester6", "age", 106, "migration_shard_id", "L1")); + events.add(Map.of("id", 7, "name", "Tester7", "age", 107, "migration_shard_id", "L1")); + events.add(Map.of("id", 8, "name", "Tester8", "age", 108, "migration_shard_id", "L1")); + events.add(Map.of("id", 9, "name", "Tester9", "age", 109, "migration_shard_id", "L1")); + events.add(Map.of("id", 10, "name", "Tester10", "age", 110, "migration_shard_id", "L1")); + events.add(Map.of("id", 11, "name", "Tester11", "age", 111, "migration_shard_id", "L1")); + events.add(Map.of("id", 12, "name", "Tester12", "age", 112, "migration_shard_id", "L1")); + + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private String generateSourceConfig( + String streamA, String dbA, String shardA) { + return "{\n" + + " \"shardConfigs\": [\n" + + " {\n" + + " \"logicalShardId\": \"" + shardA + "\",\n" + + " \"dbName\": \"" + dbA + "\",\n" + + " \"streamId\": \"" + streamA + "\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java new file mode 100644 index 0000000000..fec02abe26 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -0,0 +1,239 @@ +package com.google.cloud.teleport.v2.templates.oracle; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; +import org.apache.beam.it.gcp.datastream.OracleSource; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class OracleSeparateShadowTableDatabaseStringOverridesIT extends DataStreamToSpannerITBase { + + private static final String ORACLE_TABLE = "person1"; + private static final String SPANNER_TABLE = "human1"; + + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + + private static PipelineLauncher.LaunchInfo jobInfo; + private static HashSet testInstances = new HashSet<>(); + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + public static SpannerResourceManager shadowSpannerResourceManager; + public static GcsResourceManager gcsResourceManager; + public static CloudOracleResourceManager cloudOracleSysUser; + public static CloudOracleResourceManager cloudSqlResourceManager; + public static DatastreamResourceManager datastreamResourceManager; + + @Before + public void setUp() throws IOException { + skipBaseCleanup = true; + synchronized (OracleSeparateShadowTableDatabaseStringOverridesIT.class) { + testInstances.add(this); + if (jobInfo == null) { + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + .setCredentialsProvider(credentialsProvider) + .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .build(); + + spannerResourceManager = setUpSpannerResourceManager(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + gcsResourceManager = setUpSpannerITGcsResourceManager(); + + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.setUsername("sys as sysdba"); + builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setHost(System.getProperty("hostIp")); + builder.setPort(1521); + builder.setDatabaseName("XE"); + cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + + cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudProxyUsername", "system")) + .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setDatabaseName("XE") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + + + try { cloudSqlResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch (Exception e) {} + executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql"); + + OracleSource jdbcSource = OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); + + Map overridesMap = new HashMap<>(); + overridesMap.put("inputFileFormat", "avro"); + overridesMap.put("datastreamSourceType", "oracle"); + overridesMap.put("tableOverrides", "[{person1, human1}]"); + overridesMap.put("columnOverrides", "[{person1.first_name1, person1.name1}]"); + overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + "OracleSeparateShadowTableDatabaseStringOverridesIT", + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + jdbcSource); + } + } + } + + private void setUpOracleUser(String user, String password) { + cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + + } + + @AfterClass + public static void cleanUp() throws IOException { + for (OracleSeparateShadowTableDatabaseStringOverridesIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources( + spannerResourceManager, pubsubResourceManager, shadowSpannerResourceManager, gcsResourceManager, datastreamResourceManager, cloudOracleSysUser, cloudSqlResourceManager); + } + + @Test + public void migrationTestWithRenameTableAndColumns() { + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + new org.apache.beam.it.conditions.ConditionCheck() { + boolean executed = false; + @Override + protected String getDescription() { + return "Insert records into Oracle"; + } + @Override + protected CheckResult check() { + if (!executed) { + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); + cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + } + return new CheckResult(true, "Inserted successfully"); + } + }, + SpannerRowsCheck.builder(spannerResourceManager, SPANNER_TABLE) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition( + createConfig(jobInfo, Duration.ofMinutes(45)), + conditionCheck); + + assertThatResult(result).meetsConditions(); + assertHumanTableContents(); + } + + private void assertHumanTableContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("name1", "John"); + row1.put("last_name1", "Doe"); + + Map row2 = new HashMap<>(); + row2.put("name1", "Alice"); + row2.put("last_name1", "Johnson"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select name1, last_name1 from human1")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..6b6168912a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,131 @@ +CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsWithSizes ( + varchar_column STRING(30) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(50), + bool_column BOOL, + binary_column BYTES(30), + varbinary_column BYTES(30), + bit_column BYTES(10), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsReducedSizes ( + varchar_column STRING(10) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(20), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS Users ( + user_id INT64 NOT NULL, + first_name STRING(50), + last_name STRING(50), + age INT64, + full_name STRING(100) AS (ARRAY_TO_STRING([first_name, last_name], " ")) STORED, +) PRIMARY KEY (user_id); + +CREATE TABLE IF NOT EXISTS Authors ( + id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (id); + +CREATE TABLE IF NOT EXISTS AllDatatypeTransformation ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + blob_column BYTES(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY (varchar_column); + +CREATE SEQUENCE sequence_id OPTIONS (sequence_kind='bit_reversed_positive', skip_range_min = 0, skip_range_max = 3); + +CREATE TABLE IF NOT EXISTS Singers ( + singer_id INT64 NOT NULL DEFAULT (GET_NEXT_SEQUENCE_VALUE(SEQUENCE sequence_id)) , + first_name STRING(1024), + last_name STRING(1024), +) PRIMARY KEY (singer_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE), + title STRING(200), +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql new file mode 100644 index 0000000000..fc1f4a088c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql @@ -0,0 +1,4 @@ +INSERT INTO "AllDatatypeColumns" +("varchar_column", "tinyint_column", "text_column", "date_column", "smallint_column", "mediumint_column", "int_column", "bigint_column", "float_column", "double_column", "decimal_column", "datetime_column", "timestamp_column", "time_column", "year_column", "char_column", "tinyblob_column", "tinytext_column", "blob_column", "mediumblob_column", "mediumtext_column", "longblob_column", "longtext_column", "enum_column", "bool_column", "other_bool_column", "binary_column", "varbinary_column", "bit_column") +VALUES +('value1', 10, 'text_content', TO_DATE('2024-02-08', 'YYYY-MM-DD'), 50, 1000, 50000, 987654321, 45.67, 123.789, 456.12, TO_TIMESTAMP('2024-02-08 08:15:30', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP_TZ('2024-02-08 08:15:30 UTC', 'YYYY-MM-DD HH24:MI:SS TZR'), '08:15:30', 2022, 'char_1 ', HEXTORAW('74696e79626c6f625f646174615f31'), 'tinytext', HEXTORAW('626c6f625f646174615f31'), HEXTORAW('6d656469756d626c6f625f646174615f31'), 'mediumtext', HEXTORAW('6c6f6e67626c6f625f646174615f31'), 'longtext_content', '2', 0, 1, HEXTORAW('62696e6172795f31000000000000000000000000'), HEXTORAW('76617262696e6172795f646174615f31'), HEXTORAW('0102')); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql new file mode 100644 index 0000000000..830bc4ba29 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql @@ -0,0 +1,156 @@ + +CREATE TABLE "AllDatatypeColumns" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "other_bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "AllDatatypeColumns2" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsWithSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsReducedSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Users" ( + "user_id" NUMBER NOT NULL, + "first_name" VARCHAR2(50), + "last_name" VARCHAR2(50), + "age" NUMBER, + PRIMARY KEY ("user_id") +); + +CREATE TABLE "Authors" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +CREATE TABLE "AllDatatypeTransformation" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "blob_column" BLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Singers" ( + "singer_id" NUMBER NOT NULL , + "first_name" VARCHAR2(1024), + "last_name" VARCHAR2(1024), + PRIMARY KEY ("singer_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL , + "title" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +ALTER TABLE "AllDatatypeColumns" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeColumns2" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsWithSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsReducedSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeTransformation" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Singers" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..3ac9187f46 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,22 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, + LegacyRegion STRING(50), +) PRIMARY KEY (CustomerId); + +ALTER TABLE Customers ADD CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + LegacyOrderSystem STRING(50) NOT NULL, +) PRIMARY KEY (CustomerId, LegacyOrderSystem, OrderId); + +ALTER TABLE Orders ADD CONSTRAINT FK_CustomerOrder FOREIGN KEY (CustomerId) REFERENCES Customers(CustomerId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar2_col STRING(MAX), +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..9122439183 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,21 @@ +CREATE TABLE "Customers" ( + "CustomerId" INTEGER NOT NULL, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER NOT NULL, + "LoyaltyTier" VARCHAR2(50), + PRIMARY KEY ("CustomerId") +); + +CREATE TABLE "Orders" ( + "CustomerId" INTEGER NOT NULL, + "OrderId" INTEGER NOT NULL, + "OrderValue" NUMBER, + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" INTEGER, + "varchar2_col" VARCHAR2(1000), + PRIMARY KEY ("id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..8000ac7d0d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..0a6151cc54 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE `Customers` ( + `CustomerId` INT64 NOT NULL , + `CustomerName` STRING(255), + `CreditLimit` NUMERIC NOT NULL , + `LegacyRegion` STRING(50), + `migration_shard_id` STRING(50), + CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000) +) PRIMARY KEY (`migration_shard_id`, `CustomerId`); + +CREATE TABLE `Orders` ( + `CustomerId` INT64 NOT NULL , + `OrderId` INT64 NOT NULL , + `OrderValue` NUMERIC, + `migration_shard_id` STRING(50), + `LegacyOrderSystem` STRING(50) NOT NULL +) PRIMARY KEY (`migration_shard_id`, `CustomerId`, `LegacyOrderSystem`, `OrderId`), +INTERLEAVE IN PARENT `Customers` ON DELETE NO ACTION; + +CREATE TABLE `AllDataTypes` ( + `id` INT64 NOT NULL , + `varchar_col` STRING(1000), + `tinyint_col` INT64, + `tinyint_unsigned_col` INT64, + `text_col` STRING(MAX), + `date_col` TIMESTAMP, + `smallint_col` INT64, + `smallint_unsigned_col` INT64, + `mediumint_col` INT64, + `mediumint_unsigned_col` INT64, + `bigint_col` INT64, + `bigint_unsigned_col` INT64, + `float_col` FLOAT32, + `double_col` FLOAT64, + `decimal_col` NUMERIC, + `datetime_col` TIMESTAMP, + `time_col` STRING(50), + `year_col` STRING(4), + `char_col` STRING(255), + `tinyblob_col` BYTES(255), + `tinytext_col` STRING(MAX), + `blob_col` BYTES(65535), + `mediumblob_col` BYTES(10485760), + `mediumtext_col` STRING(MAX), + `test_json_col` STRING(MAX), + `longblob_col` BYTES(10485760), + `longtext_col` STRING(MAX), + `enum_col` STRING(MAX), + `bool_col` INT64, + `binary_col` BYTES(255), + `varbinary_col` BYTES(1000), + `bit_col` BYTES(MAX), + `bit8_col` BYTES(MAX), + `bit1_col` INT64, + `boolean_col` INT64, + `int_col` INT64, + `integer_unsigned_col` INT64, + `timestamp_col` TIMESTAMP, + `set_col` STRING(MAX), + `migration_shard_id` STRING(50) +) PRIMARY KEY (`migration_shard_id`, `id`); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..fb08ca5296 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/oracle-schema.sql @@ -0,0 +1,57 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER NOT NULL PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER(10, 2) NOT NULL, + "LoyaltyTier" VARCHAR2(50) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER(10, 2), + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER DEFAULT NULL, + "tinyint_unsigned_col" NUMBER DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER DEFAULT NULL, + "smallint_unsigned_col" NUMBER DEFAULT NULL, + "mediumint_col" NUMBER DEFAULT NULL, + "mediumint_unsigned_col" NUMBER DEFAULT NULL, + "bigint_col" NUMBER DEFAULT NULL, + "bigint_unsigned_col" NUMBER DEFAULT NULL, + "float_col" BINARY_FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER(38,9) DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" VARCHAR2(4) DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" RAW(255) DEFAULT NULL, + "tinytext_col" CLOB DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" VARCHAR2(4000) DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(255) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" RAW(1) DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER DEFAULT NULL, + "integer_unsigned_col" NUMBER DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(255) DEFAULT NULL +); + diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json new file mode 100644 index 0000000000..8000ac7d0d --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json new file mode 100644 index 0000000000..e366c8ed21 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleRetryDLQIT/session.json @@ -0,0 +1,4272 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "XE", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "AllDataTypes", + "ColIds": [ + "c4", + "c5", + "c6", + "c7", + "c8", + "c9", + "c10", + "c11", + "c12", + "c13", + "c14", + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c51" + ], + "ShardIdColumn": "c51", + "ColDefs": { + "c10": { + "Name": "smallint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_col smallint(5)", + "Id": "c10", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c11": { + "Name": "smallint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_unsigned_col smallint(5)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c12": { + "Name": "mediumint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_col mediumint(7)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c13": { + "Name": "mediumint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_unsigned_col mediumint(7)", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c14": { + "Name": "bigint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_col bigint(19)", + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c15": { + "Name": "bigint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_unsigned_col bigint unsigned(20)", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c16": { + "Name": "float_col", + "T": { + "Name": "FLOAT32", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_col float(12)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c17": { + "Name": "double_col", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_col double(22)", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c18": { + "Name": "decimal_col", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_col decimal(65,30)", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c19": { + "Name": "datetime_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_col datetime", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c20": { + "Name": "time_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_col time", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c21": { + "Name": "year_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_col year", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c22": { + "Name": "char_col", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: char_col char(255)", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c23": { + "Name": "tinyblob_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyblob_col tinyblob(255)", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c24": { + "Name": "tinytext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinytext_col tinytext(255)", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c25": { + "Name": "blob_col", + "T": { + "Name": "BYTES", + "Len": 65535, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_col blob(65535)", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c26": { + "Name": "mediumblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumblob_col mediumblob(16777215)", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c27": { + "Name": "mediumtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumtext_col mediumtext(16777215)", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c28": { + "Name": "test_json_col", + "T": { + "Name": "JSON", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: test_json_col json", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c29": { + "Name": "longblob_col", + "T": { + "Name": "BYTES", + "Len": 10485760, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longblob_col longblob(4294967295)", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c30": { + "Name": "longtext_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longtext_col longtext(4294967295)", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c31": { + "Name": "enum_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_col enum(1)", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c32": { + "Name": "bool_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_col tinyint(1)", + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c33": { + "Name": "binary_col", + "T": { + "Name": "BYTES", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_col binary(255)", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c34": { + "Name": "varbinary_col", + "T": { + "Name": "BYTES", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varbinary_col varbinary(1000)", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c35": { + "Name": "bit_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_col bit(64)", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c36": { + "Name": "bit8_col", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit8_col bit(8)", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c37": { + "Name": "bit1_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit1_col bit(1)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c38": { + "Name": "boolean_col", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: boolean_col tinyint(1)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c39": { + "Name": "int_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_col int(10)", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c4": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c40": { + "Name": "integer_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: integer_unsigned_col int(10)", + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c41": { + "Name": "timestamp_col", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_col timestamp", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c42": { + "Name": "set_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: set_col set[]", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c5": { + "Name": "varchar_col", + "T": { + "Name": "STRING", + "Len": 1000, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varchar_col varchar(1000)", + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c51": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c6": { + "Name": "tinyint_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_col tinyint(3)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c7": { + "Name": "tinyint_unsigned_col", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_unsigned_col tinyint", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c8": { + "Name": "text_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_col text(65535)", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c9": { + "Name": "date_col", + "T": { + "Name": "DATE", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_col date", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c4", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c51", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table AllDataTypes", + "Id": "t1" + }, + "t2": { + "Name": "Customers", + "ColIds": [ + "c47", + "c48", + "c49", + "c50", + "c52" + ], + "ShardIdColumn": "c52", + "ColDefs": { + "c47": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c48": { + "Name": "CustomerName", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: CustomerName varchar(255)", + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c49": { + "Name": "CreditLimit", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CreditLimit decimal(10,2)", + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c50": { + "Name": "LegacyRegion", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: LoyaltyTier varchar(50)", + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c52": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c52", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": [ + { + "Id": "1", + "Name": "CHK_CreditLimit", + "Expr": "(CreditLimit > 1000)", + "ExprId": "e976" + } + ], + "Comment": "Spanner schema for source table Customers", + "Id": "t2" + }, + "t3": { + "Name": "Orders", + "ColIds": [ + "c43", + "c44", + "c45", + "c53", + "c55" + ], + "ShardIdColumn": "c53", + "ColDefs": { + "c43": { + "Name": "CustomerId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: CustomerId int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c44": { + "Name": "OrderId", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: OrderId int(10)", + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c45": { + "Name": "OrderValue", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: OrderValue decimal(10,2)", + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c53": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + }, + "c55": { + "Name": "LegacyOrderSystem", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": true, + "Comment": "", + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c44", + "Desc": false, + "Order": 4 + }, + { + "ColId": "c53", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c55", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "t2", + "OnDelete": "NO ACTION", + "InterleaveType": "IN PARENT" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table Orders", + "Id": "t3" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "AllDataTypes", + "Schema": "XE", + "ColIds": [ + "c4", + "c5", + "c6", + "c7", + "c8", + "c9", + "c10", + "c11", + "c12", + "c13", + "c14", + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42" + ], + "ColDefs": { + "c10": { + "Name": "smallint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c10", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c11": { + "Name": "smallint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c12": { + "Name": "mediumint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c13": { + "Name": "mediumint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c14": { + "Name": "bigint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c15": { + "Name": "bigint_unsigned_col", + "Type": { + "Name": "bigint unsigned", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c16": { + "Name": "float_col", + "Type": { + "Name": "BINARY_FLOAT", + "Mods": [ + 12 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c17": { + "Name": "double_col", + "Type": { + "Name": "BINARY_DOUBLE", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c18": { + "Name": "decimal_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c19": { + "Name": "datetime_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c20": { + "Name": "time_col", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c21": { + "Name": "year_col", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c22": { + "Name": "char_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c23": { + "Name": "tinyblob_col", + "Type": { + "Name": "RAW", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c24": { + "Name": "tinytext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c25": { + "Name": "blob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c26": { + "Name": "mediumblob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c27": { + "Name": "mediumtext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c28": { + "Name": "test_json_col", + "Type": { + "Name": "VARCHAR2", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c29": { + "Name": "longblob_col", + "Type": { + "Name": "BLOB", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c30": { + "Name": "longtext_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c31": { + "Name": "enum_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c32": { + "Name": "bool_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c33": { + "Name": "binary_col", + "Type": { + "Name": "RAW", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c34": { + "Name": "varbinary_col", + "Type": { + "Name": "RAW", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c35": { + "Name": "bit_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c36": { + "Name": "bit8_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c37": { + "Name": "bit1_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c38": { + "Name": "boolean_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c39": { + "Name": "int_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c4": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c40": { + "Name": "integer_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c41": { + "Name": "timestamp_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c42": { + "Name": "set_col", + "Type": { + "Name": "VARCHAR2", + "Mods": null, + "ArrayBounds": [ + -1 + ] + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c5": { + "Name": "varchar_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 1000 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c6": { + "Name": "tinyint_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c7": { + "Name": "tinyint_unsigned_col", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c8": { + "Name": "text_col", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c9": { + "Name": "date_col", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c4", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t1" + }, + "t2": { + "Name": "Customers", + "Schema": "XE", + "ColIds": [ + "c47", + "c48", + "c49", + "c50" + ], + "ColDefs": { + "c47": { + "Name": "CustomerId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c48": { + "Name": "CustomerName", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c49": { + "Name": "CreditLimit", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c50": { + "Name": "LoyaltyTier", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t2" + }, + "t3": { + "Name": "Orders", + "Schema": "XE", + "ColIds": [ + "c43", + "c44", + "c45", + "c46" + ], + "ColDefs": { + "c43": { + "Name": "CustomerId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c44": { + "Name": "OrderId", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c45": { + "Name": "OrderValue", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + }, + "c46": { + "Name": "OrderSource", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 50 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "GeneratedColumn": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + }, + "Type": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c44", + "Desc": false, + "Order": 2 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t3" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c10": [ + 14 + ], + "c11": [ + 14 + ], + "c12": [ + 14 + ], + "c13": [ + 14 + ], + "c15": [ + 52 + ], + "c19": [ + 13 + ], + "c20": [ + 15 + ], + "c21": [ + 15 + ], + "c39": [ + 14 + ], + "c4": [ + 14 + ], + "c40": [ + 14 + ], + "c42": [ + 31 + ], + "c51": [ + 29 + ], + "c6": [ + 14 + ], + "c7": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t2": { + "ColumnLevelIssues": { + "c47": [ + 14 + ], + "c52": [ + 5 + ] + }, + "TableLevelIssues": null + }, + "t3": { + "ColumnLevelIssues": { + "c43": [ + 14 + ], + "c44": [ + 14 + ], + "c53": [ + 5 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "ToSpanner": { + "AllDataTypes": { + "Name": "AllDataTypes", + "Cols": { + "bigint_col": "bigint_col", + "bigint_unsigned_col": "bigint_unsigned_col", + "binary_col": "binary_col", + "bit1_col": "bit1_col", + "bit8_col": "bit8_col", + "bit_col": "bit_col", + "blob_col": "blob_col", + "bool_col": "bool_col", + "boolean_col": "boolean_col", + "char_col": "char_col", + "date_col": "date_col", + "datetime_col": "datetime_col", + "decimal_col": "decimal_col", + "double_col": "double_col", + "enum_col": "enum_col", + "float_col": "float_col", + "id": "id", + "int_col": "int_col", + "integer_unsigned_col": "integer_unsigned_col", + "longblob_col": "longblob_col", + "longtext_col": "longtext_col", + "mediumblob_col": "mediumblob_col", + "mediumint_col": "mediumint_col", + "mediumint_unsigned_col": "mediumint_unsigned_col", + "mediumtext_col": "mediumtext_col", + "set_col": "set_col", + "smallint_col": "smallint_col", + "smallint_unsigned_col": "smallint_unsigned_col", + "test_json_col": "test_json_col", + "text_col": "text_col", + "time_col": "time_col", + "timestamp_col": "timestamp_col", + "tinyblob_col": "tinyblob_col", + "tinyint_col": "tinyint_col", + "tinyint_unsigned_col": "tinyint_unsigned_col", + "tinytext_col": "tinytext_col", + "varbinary_col": "varbinary_col", + "varchar_col": "varchar_col", + "year_col": "year_col" + } + }, + "Customers": { + "Name": "Customers", + "Cols": { + "CreditLimit": "CreditLimit", + "CustomerId": "CustomerId", + "CustomerName": "CustomerName", + "LoyaltyTier": "LegacyRegion" + } + }, + "Orders": { + "Name": "Orders", + "Cols": { + "CustomerId": "CustomerId", + "OrderId": "OrderId", + "OrderSource": "OrderSource", + "OrderValue": "OrderValue" + } + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r54", + "Name": "r54", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": null + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "SpProjectId": "span-cloud-ck-testing-external", + "SpInstanceId": "ea-functional-tests", + "Source": "mysql", + "DatabaseOptions": { + "DbName": "", + "DefaultTimezone": "" + }, + "DefaultIdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3e8b59c4ff --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..c87cdd07ab --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerOracleStringOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500) DEFAULT NULL, + "last_name1" VARCHAR2(500) DEFAULT NULL, + "first_name2" VARCHAR2(500) DEFAULT NULL, + "last_name2" VARCHAR2(500) DEFAULT NULL, + "first_name3" VARCHAR2(500) DEFAULT NULL, + "last_name3" VARCHAR2(500) DEFAULT NULL, + "ID" NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..55f2e40bc7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,59 @@ +CREATE TABLE Customers ( + CustomerId INT64 NOT NULL, + CustomerName STRING(255), + CreditLimit NUMERIC, + LegacyRegion STRING(50), +) PRIMARY KEY (CustomerId); + +ALTER TABLE Customers ADD CONSTRAINT CHK_CreditLimit CHECK (CreditLimit > 1000); + +CREATE TABLE Orders ( + CustomerId INT64 NOT NULL, + OrderId INT64 NOT NULL, + OrderValue NUMERIC, + LegacyOrderSystem STRING(50) NOT NULL, +) PRIMARY KEY (CustomerId, LegacyOrderSystem, OrderId); + +ALTER TABLE Orders ADD CONSTRAINT FK_CustomerOrder FOREIGN KEY (CustomerId) REFERENCES Customers(CustomerId); + +CREATE TABLE AllDataTypes ( + id INT64 NOT NULL, + varchar_col STRING(1000), + tinyint_col INT64, + tinyint_unsigned_col INT64, + text_col STRING(MAX), + date_col DATE, + smallint_col INT64, + smallint_unsigned_col INT64, + mediumint_col INT64, + mediumint_unsigned_col INT64, + bigint_col INT64, + bigint_unsigned_col NUMERIC, + float_col FLOAT64, + double_col FLOAT64, + decimal_col NUMERIC, + datetime_col TIMESTAMP, + time_col STRING(MAX), + year_col STRING(MAX), + char_col STRING(255), + tinyblob_col BYTES(MAX), + tinytext_col STRING(MAX), + blob_col BYTES(MAX), + mediumblob_col BYTES(MAX), + mediumtext_col STRING(MAX), + test_json_col JSON, + longblob_col BYTES(MAX), + longtext_col STRING(MAX), + enum_col STRING(MAX), + bool_col BOOL, + binary_col BYTES(MAX), + varbinary_col BYTES(MAX), + bit_col BYTES(MAX), + bit8_col INT64, + bit1_col BOOL, + boolean_col BOOL, + int_col INT64, + integer_unsigned_col INT64, + timestamp_col TIMESTAMP, + set_col STRING(MAX), +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql new file mode 100644 index 0000000000..fc5bcef3d1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/oracle-schema.sql @@ -0,0 +1,56 @@ +CREATE TABLE "Customers" ( + "CustomerId" NUMBER PRIMARY KEY, + "CustomerName" VARCHAR2(255), + "CreditLimit" NUMBER NOT NULL, + "LoyaltyTier" VARCHAR2(50) +); + +CREATE TABLE "Orders" ( + "CustomerId" NUMBER NOT NULL, + "OrderId" NUMBER NOT NULL, + "OrderValue" NUMBER, + "OrderSource" VARCHAR2(50) NOT NULL, + PRIMARY KEY ("CustomerId", "OrderId") +); + +CREATE TABLE "AllDataTypes" ( + "id" NUMBER PRIMARY KEY, + "varchar_col" VARCHAR2(1000) DEFAULT NULL, + "tinyint_col" NUMBER DEFAULT NULL, + "tinyint_unsigned_col" NUMBER DEFAULT NULL, + "text_col" CLOB DEFAULT NULL, + "date_col" DATE DEFAULT NULL, + "smallint_col" NUMBER DEFAULT NULL, + "smallint_unsigned_col" NUMBER DEFAULT NULL, + "mediumint_col" NUMBER DEFAULT NULL, + "mediumint_unsigned_col" NUMBER DEFAULT NULL, + "bigint_col" NUMBER DEFAULT NULL, + "bigint_unsigned_col" NUMBER DEFAULT NULL, + "float_col" BINARY_FLOAT DEFAULT NULL, + "double_col" BINARY_DOUBLE DEFAULT NULL, + "decimal_col" NUMBER DEFAULT NULL, + "datetime_col" TIMESTAMP DEFAULT NULL, + "time_col" VARCHAR2(50) DEFAULT NULL, + "year_col" NUMBER DEFAULT NULL, + "char_col" CHAR(255) DEFAULT NULL, + "tinyblob_col" RAW(255) DEFAULT NULL, + "tinytext_col" VARCHAR2(255) DEFAULT NULL, + "blob_col" BLOB DEFAULT NULL, + "mediumblob_col" BLOB DEFAULT NULL, + "mediumtext_col" CLOB DEFAULT NULL, + "test_json_col" CLOB DEFAULT NULL, + "longblob_col" BLOB DEFAULT NULL, + "longtext_col" CLOB DEFAULT NULL, + "enum_col" VARCHAR2(50) DEFAULT NULL, + "bool_col" NUMBER(1) DEFAULT NULL, + "binary_col" RAW(255) DEFAULT NULL, + "varbinary_col" RAW(1000) DEFAULT NULL, + "bit_col" RAW(8) DEFAULT NULL, + "bit8_col" NUMBER(3) DEFAULT NULL, + "bit1_col" NUMBER(1) DEFAULT NULL, + "boolean_col" NUMBER(1) DEFAULT NULL, + "int_col" NUMBER DEFAULT NULL, + "integer_unsigned_col" NUMBER DEFAULT NULL, + "timestamp_col" TIMESTAMP DEFAULT NULL, + "set_col" VARCHAR2(255) DEFAULT NULL +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json new file mode 100644 index 0000000000..2c964b41cd --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json @@ -0,0 +1,8 @@ +{ + "renamedTables": {}, + "renamedColumns": { + "Customers": { + "LegacyRegion": "LoyaltyTier" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..0b324f3480 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,140 @@ +CREATE TABLE `true` ( + id INT64 NOT NULL, + `ALL` STRING(MAX), + `ANALYSE` STRING(MAX), + `ANALYZE` STRING(MAX), + `AND` STRING(MAX), + `ANY` STRING(MAX), + `ARRAY` STRING(MAX), + `AS` STRING(MAX), + `ASC` STRING(MAX), + `ASSERT_ROWS_MODIFIED` STRING(MAX), + `ASYMMETRIC` STRING(MAX), + `AT` STRING(MAX), + `BETWEEN` STRING(MAX), + `BOTH` STRING(MAX), + `BY` STRING(MAX), + `CASE` STRING(MAX), + `CAST` STRING(MAX), + `CHECK` STRING(MAX), + `COLLATE` STRING(MAX), + `COLUMN` STRING(MAX), + `CONSTRAINT` STRING(MAX), + `CONTAINS` STRING(MAX), + `CREATE` STRING(MAX), + `CROSS` STRING(MAX), + `CUBE` STRING(MAX), + `CURRENT` STRING(MAX), + `CURRENT_CATALOG` STRING(MAX), + `CURRENT_DATE` STRING(MAX), + `CURRENT_ROLE` STRING(MAX), + `CURRENT_TIME` STRING(MAX), + `CURRENT_TIMESTAMP` STRING(MAX), + `CURRENT_USER` STRING(MAX), + `DEFAULT` STRING(MAX), + `DEFERRABLE` STRING(MAX), + `DEFINE` STRING(MAX), + `DESC` STRING(MAX), + `DISTINCT` STRING(MAX), + `DO` STRING(MAX), + `ELSE` STRING(MAX), + `END` STRING(MAX), + `END_P` STRING(MAX), + `ENUM` STRING(MAX), + `ESCAPE` STRING(MAX), + `EXCEPT` STRING(MAX), + `EXCLUDE` STRING(MAX), + `EXISTS` STRING(MAX), + `EXTRACT` STRING(MAX), + `FALSE` STRING(MAX), + `FALSE_P` STRING(MAX), + `FETCH` STRING(MAX), + `FOLLOWING` STRING(MAX), + `FOR` STRING(MAX), + `FOREIGN` STRING(MAX), + `FROM` STRING(MAX), + `FULL` STRING(MAX), + `GRANT` STRING(MAX), + `GRAPH_TABLE` STRING(MAX), + `GROUP` STRING(MAX), + `GROUPING` STRING(MAX), + `GROUPS` STRING(MAX), + `GROUP_P` STRING(MAX), + `HASH` STRING(MAX), + `HAVING` STRING(MAX), + `IF` STRING(MAX), + `IGNORE` STRING(MAX), + `IN` STRING(MAX), + `INITIALLY` STRING(MAX), + `INNER` STRING(MAX), + `INTERSECT` STRING(MAX), + `INTERVAL` STRING(MAX), + `INTO` STRING(MAX), + `IN_P` STRING(MAX), + `IS` STRING(MAX), + `JOIN` STRING(MAX), + `LATERAL` STRING(MAX), + `LATERAL_P` STRING(MAX), + `LEADING` STRING(MAX), + `LEFT` STRING(MAX), + `LIKE` STRING(MAX), + `LIMIT` STRING(MAX), + `LOCALTIME` STRING(MAX), + `LOCALTIMESTAMP` STRING(MAX), + `LOOKUP` STRING(MAX), + `MERGE` STRING(MAX), + `NATURAL` STRING(MAX), + `NEW` STRING(MAX), + `NO` STRING(MAX), + `NOT` STRING(MAX), + `NULL` STRING(MAX), + `NULLS` STRING(MAX), + `NULL_P` STRING(MAX), + `OF` STRING(MAX), + `OFFSET` STRING(MAX), + `ON` STRING(MAX), + `ONLY` STRING(MAX), + `OR` STRING(MAX), + `ORDER` STRING(MAX), + `OUTER` STRING(MAX), + `OVER` STRING(MAX), + `PARTITION` STRING(MAX), + `PLACING` STRING(MAX), + `PRECEDING` STRING(MAX), + `PRIMARY` STRING(MAX), + `PROTO` STRING(MAX), + `RANGE` STRING(MAX), + `RECURSIVE` STRING(MAX), + `REFERENCES` STRING(MAX), + `RESPECT` STRING(MAX), + `RETURNING` STRING(MAX), + `RIGHT` STRING(MAX), + `ROLLUP` STRING(MAX), + `ROWS` STRING(MAX), + `SELECT` STRING(MAX), + `SESSION_USER` STRING(MAX), + `SET` STRING(MAX), + `SOME` STRING(MAX), + `STRUCT` STRING(MAX), + `SYMMETRIC` STRING(MAX), + `TABLE` STRING(MAX), + `TABLESAMPLE` STRING(MAX), + `THEN` STRING(MAX), + `TO` STRING(MAX), + `TRAILING` STRING(MAX), + `TREAT` STRING(MAX), + `TRUE` STRING(MAX), + `TRUE_P` STRING(MAX), + `UNBOUNDED` STRING(MAX), + `UNION` STRING(MAX), + `UNIQUE` STRING(MAX), + `UNNEST` STRING(MAX), + `USER` STRING(MAX), + `USING` STRING(MAX), + `VARIADIC` STRING(MAX), + `WHEN` STRING(MAX), + `WHERE` STRING(MAX), + `WINDOW` STRING(MAX), + `WITH` STRING(MAX), + `WITHIN` STRING(MAX) +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql new file mode 100644 index 0000000000..11889d70cb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/oracle-schema.sql @@ -0,0 +1,142 @@ +CREATE TABLE "true" ( + "id" NUMBER(38, 0) PRIMARY KEY, + "ALL" VARCHAR2(4000), + "ANALYSE" VARCHAR2(4000), + "ANALYZE" VARCHAR2(4000), + "AND" VARCHAR2(4000), + "ANY" VARCHAR2(4000), + "ARRAY" VARCHAR2(4000), + "AS" VARCHAR2(4000), + "ASC" VARCHAR2(4000), + "ASSERT_ROWS_MODIFIED" VARCHAR2(4000), + "ASYMMETRIC" VARCHAR2(4000), + "AT" VARCHAR2(4000), + "BETWEEN" VARCHAR2(4000), + "BOTH" VARCHAR2(4000), + "BY" VARCHAR2(4000), + "CASE" VARCHAR2(4000), + "CAST" VARCHAR2(4000), + "CHECK" VARCHAR2(4000), + "COLLATE" VARCHAR2(4000), + "COLUMN" VARCHAR2(4000), + "CONSTRAINT" VARCHAR2(4000), + "CONTAINS" VARCHAR2(4000), + "CREATE" VARCHAR2(4000), + "CROSS" VARCHAR2(4000), + "CUBE" VARCHAR2(4000), + "CURRENT" VARCHAR2(4000), + "CURRENT_CATALOG" VARCHAR2(4000), + "CURRENT_DATE" VARCHAR2(4000), + "CURRENT_ROLE" VARCHAR2(4000), + "CURRENT_TIME" VARCHAR2(4000), + "CURRENT_TIMESTAMP" VARCHAR2(4000), + "CURRENT_USER" VARCHAR2(4000), + "DEFAULT" VARCHAR2(4000), + "DEFERRABLE" VARCHAR2(4000), + "DEFINE" VARCHAR2(4000), + "DESC" VARCHAR2(4000), + "DISTINCT" VARCHAR2(4000), + "DO" VARCHAR2(4000), + "ELSE" VARCHAR2(4000), + "END" VARCHAR2(4000), + "END_P" VARCHAR2(4000), + "ENUM" VARCHAR2(4000), + "ESCAPE" VARCHAR2(4000), + "EXCEPT" VARCHAR2(4000), + "EXCLUDE" VARCHAR2(4000), + "EXISTS" VARCHAR2(4000), + "EXTRACT" VARCHAR2(4000), + "FALSE" VARCHAR2(4000), + "FALSE_P" VARCHAR2(4000), + "FETCH" VARCHAR2(4000), + "FOLLOWING" VARCHAR2(4000), + "FOR" VARCHAR2(4000), + "FOREIGN" VARCHAR2(4000), + "FROM" VARCHAR2(4000), + "FULL" VARCHAR2(4000), + "GRANT" VARCHAR2(4000), + "GRAPH_TABLE" VARCHAR2(4000), + "GROUP" VARCHAR2(4000), + "GROUPING" VARCHAR2(4000), + "GROUPS" VARCHAR2(4000), + "GROUP_P" VARCHAR2(4000), + "HASH" VARCHAR2(4000), + "HAVING" VARCHAR2(4000), + "IF" VARCHAR2(4000), + "IGNORE" VARCHAR2(4000), + "IN" VARCHAR2(4000), + "INITIALLY" VARCHAR2(4000), + "INNER" VARCHAR2(4000), + "INTERSECT" VARCHAR2(4000), + "INTERVAL" VARCHAR2(4000), + "INTO" VARCHAR2(4000), + "IN_P" VARCHAR2(4000), + "IS" VARCHAR2(4000), + "JOIN" VARCHAR2(4000), + "LATERAL" VARCHAR2(4000), + "LATERAL_P" VARCHAR2(4000), + "LEADING" VARCHAR2(4000), + "LEFT" VARCHAR2(4000), + "LIKE" VARCHAR2(4000), + "LIMIT" VARCHAR2(4000), + "LOCALTIME" VARCHAR2(4000), + "LOCALTIMESTAMP" VARCHAR2(4000), + "LOOKUP" VARCHAR2(4000), + "MERGE" VARCHAR2(4000), + "NATURAL" VARCHAR2(4000), + "NEW" VARCHAR2(4000), + "NO" VARCHAR2(4000), + "NOT" VARCHAR2(4000), + "NULL" VARCHAR2(4000), + "NULLS" VARCHAR2(4000), + "NULL_P" VARCHAR2(4000), + "OF" VARCHAR2(4000), + "OFFSET" VARCHAR2(4000), + "ON" VARCHAR2(4000), + "ONLY" VARCHAR2(4000), + "OR" VARCHAR2(4000), + "ORDER" VARCHAR2(4000), + "OUTER" VARCHAR2(4000), + "OVER" VARCHAR2(4000), + "PARTITION" VARCHAR2(4000), + "PLACING" VARCHAR2(4000), + "PRECEDING" VARCHAR2(4000), + "PRIMARY" VARCHAR2(4000), + "PROTO" VARCHAR2(4000), + "RANGE" VARCHAR2(4000), + "RECURSIVE" VARCHAR2(4000), + "REFERENCES" VARCHAR2(4000), + "RESPECT" VARCHAR2(4000), + "RETURNING" VARCHAR2(4000), + "RIGHT" VARCHAR2(4000), + "ROLLUP" VARCHAR2(4000), + "ROWS" VARCHAR2(4000), + "SELECT" VARCHAR2(4000), + "SESSION_USER" VARCHAR2(4000), + "SET" VARCHAR2(4000), + "SOME" VARCHAR2(4000), + "STRUCT" VARCHAR2(4000), + "SYMMETRIC" VARCHAR2(4000), + "TABLE" VARCHAR2(4000), + "TABLESAMPLE" VARCHAR2(4000), + "THEN" VARCHAR2(4000), + "TO" VARCHAR2(4000), + "TRAILING" VARCHAR2(4000), + "TREAT" VARCHAR2(4000), + "TRUE" VARCHAR2(4000), + "TRUE_P" VARCHAR2(4000), + "UNBOUNDED" VARCHAR2(4000), + "UNION" VARCHAR2(4000), + "UNIQUE" VARCHAR2(4000), + "UNNEST" VARCHAR2(4000), + "USER" VARCHAR2(4000), + "USING" VARCHAR2(4000), + "VARIADIC" VARCHAR2(4000), + "WHEN" VARCHAR2(4000), + "WHERE" VARCHAR2(4000), + "WINDOW" VARCHAR2(4000), + "WITH" VARCHAR2(4000), + "WITHIN" VARCHAR2(4000) +); +INSERT INTO "true" ("id", "ALL", "AND", "AS") VALUES (1, 'all', 'and', 'as'); +COMMIT; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json new file mode 100644 index 0000000000..229d36db8a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/DatastreamToSpannerReservedKeywordsOracleIT/session.json @@ -0,0 +1,323 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "SP_DATABASE", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t1": { + "Name": "true", + "ColIds": ["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19", "c20", "c21", "c22", "c23", "c24", "c25", "c26", "c27", "c28", "c29", "c30", "c31", "c32", "c33", "c34", "c35", "c36", "c37", "c38", "c39", "c40", "c41", "c42", "c43", "c44", "c45", "c46", "c47", "c48", "c49", "c50", "c51", "c52", "c53", "c54", "c55", "c56", "c57", "c58", "c59", "c60", "c61", "c62", "c63", "c64", "c65", "c66", "c67", "c68", "c69", "c70", "c71", "c72", "c73", "c74", "c75", "c76", "c77", "c78", "c79", "c80", "c81", "c82", "c83", "c84", "c85", "c86", "c87", "c88", "c89", "c90", "c91", "c92", "c93", "c94", "c95", "c96", "c97", "c98", "c99", "c100", "c101", "c102", "c103", "c104", "c105", "c106", "c107", "c108", "c109", "c110", "c111", "c112", "c113", "c114", "c115", "c116", "c117", "c118", "c119", "c120", "c121", "c122", "c123", "c124", "c125", "c126", "c127", "c128", "c129", "c130", "c131", "c132", "c133", "c134", "c135", "c136", "c137", "c138"], + "ShardIdColumn": "", + "ColDefs": { + "c1": {"Name": "id", "T": {"Name": "INT64"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "ALL", "T": {"Name": "STRING"}, "Id": "c2"}, + "c3": {"Name": "ANALYSE", "T": {"Name": "STRING"}, "Id": "c3"}, + "c4": {"Name": "ANALYZE", "T": {"Name": "STRING"}, "Id": "c4"}, + "c5": {"Name": "AND", "T": {"Name": "STRING"}, "Id": "c5"}, + "c6": {"Name": "ANY", "T": {"Name": "STRING"}, "Id": "c6"}, + "c7": {"Name": "ARRAY", "T": {"Name": "STRING"}, "Id": "c7"}, + "c8": {"Name": "AS", "T": {"Name": "STRING"}, "Id": "c8"}, + "c9": {"Name": "ASC", "T": {"Name": "STRING"}, "Id": "c9"}, + "c10": {"Name": "ASSERT_ROWS_MODIFIED", "T": {"Name": "STRING"}, "Id": "c10"}, + "c11": {"Name": "ASYMMETRIC", "T": {"Name": "STRING"}, "Id": "c11"}, + "c12": {"Name": "AT", "T": {"Name": "STRING"}, "Id": "c12"}, + "c13": {"Name": "BETWEEN", "T": {"Name": "STRING"}, "Id": "c13"}, + "c14": {"Name": "BOTH", "T": {"Name": "STRING"}, "Id": "c14"}, + "c15": {"Name": "BY", "T": {"Name": "STRING"}, "Id": "c15"}, + "c16": {"Name": "CASE", "T": {"Name": "STRING"}, "Id": "c16"}, + "c17": {"Name": "CAST", "T": {"Name": "STRING"}, "Id": "c17"}, + "c18": {"Name": "CHECK", "T": {"Name": "STRING"}, "Id": "c18"}, + "c19": {"Name": "COLLATE", "T": {"Name": "STRING"}, "Id": "c19"}, + "c20": {"Name": "COLUMN", "T": {"Name": "STRING"}, "Id": "c20"}, + "c21": {"Name": "CONSTRAINT", "T": {"Name": "STRING"}, "Id": "c21"}, + "c22": {"Name": "CONTAINS", "T": {"Name": "STRING"}, "Id": "c22"}, + "c23": {"Name": "CREATE", "T": {"Name": "STRING"}, "Id": "c23"}, + "c24": {"Name": "CROSS", "T": {"Name": "STRING"}, "Id": "c24"}, + "c25": {"Name": "CUBE", "T": {"Name": "STRING"}, "Id": "c25"}, + "c26": {"Name": "CURRENT", "T": {"Name": "STRING"}, "Id": "c26"}, + "c27": {"Name": "CURRENT_CATALOG", "T": {"Name": "STRING"}, "Id": "c27"}, + "c28": {"Name": "CURRENT_DATE", "T": {"Name": "STRING"}, "Id": "c28"}, + "c29": {"Name": "CURRENT_ROLE", "T": {"Name": "STRING"}, "Id": "c29"}, + "c30": {"Name": "CURRENT_TIME", "T": {"Name": "STRING"}, "Id": "c30"}, + "c31": {"Name": "CURRENT_TIMESTAMP", "T": {"Name": "STRING"}, "Id": "c31"}, + "c32": {"Name": "CURRENT_USER", "T": {"Name": "STRING"}, "Id": "c32"}, + "c33": {"Name": "DEFAULT", "T": {"Name": "STRING"}, "Id": "c33"}, + "c34": {"Name": "DEFERRABLE", "T": {"Name": "STRING"}, "Id": "c34"}, + "c35": {"Name": "DEFINE", "T": {"Name": "STRING"}, "Id": "c35"}, + "c36": {"Name": "DESC", "T": {"Name": "STRING"}, "Id": "c36"}, + "c37": {"Name": "DISTINCT", "T": {"Name": "STRING"}, "Id": "c37"}, + "c38": {"Name": "DO", "T": {"Name": "STRING"}, "Id": "c38"}, + "c39": {"Name": "ELSE", "T": {"Name": "STRING"}, "Id": "c39"}, + "c40": {"Name": "END", "T": {"Name": "STRING"}, "Id": "c40"}, + "c41": {"Name": "END_P", "T": {"Name": "STRING"}, "Id": "c41"}, + "c42": {"Name": "ENUM", "T": {"Name": "STRING"}, "Id": "c42"}, + "c43": {"Name": "ESCAPE", "T": {"Name": "STRING"}, "Id": "c43"}, + "c44": {"Name": "EXCEPT", "T": {"Name": "STRING"}, "Id": "c44"}, + "c45": {"Name": "EXCLUDE", "T": {"Name": "STRING"}, "Id": "c45"}, + "c46": {"Name": "EXISTS", "T": {"Name": "STRING"}, "Id": "c46"}, + "c47": {"Name": "EXTRACT", "T": {"Name": "STRING"}, "Id": "c47"}, + "c48": {"Name": "FALSE", "T": {"Name": "STRING"}, "Id": "c48"}, + "c49": {"Name": "FALSE_P", "T": {"Name": "STRING"}, "Id": "c49"}, + "c50": {"Name": "FETCH", "T": {"Name": "STRING"}, "Id": "c50"}, + "c51": {"Name": "FOLLOWING", "T": {"Name": "STRING"}, "Id": "c51"}, + "c52": {"Name": "FOR", "T": {"Name": "STRING"}, "Id": "c52"}, + "c53": {"Name": "FOREIGN", "T": {"Name": "STRING"}, "Id": "c53"}, + "c54": {"Name": "FROM", "T": {"Name": "STRING"}, "Id": "c54"}, + "c55": {"Name": "FULL", "T": {"Name": "STRING"}, "Id": "c55"}, + "c56": {"Name": "GRANT", "T": {"Name": "STRING"}, "Id": "c56"}, + "c57": {"Name": "GRAPH_TABLE", "T": {"Name": "STRING"}, "Id": "c57"}, + "c58": {"Name": "GROUP", "T": {"Name": "STRING"}, "Id": "c58"}, + "c59": {"Name": "GROUPING", "T": {"Name": "STRING"}, "Id": "c59"}, + "c60": {"Name": "GROUPS", "T": {"Name": "STRING"}, "Id": "c60"}, + "c61": {"Name": "GROUP_P", "T": {"Name": "STRING"}, "Id": "c61"}, + "c62": {"Name": "HASH", "T": {"Name": "STRING"}, "Id": "c62"}, + "c63": {"Name": "HAVING", "T": {"Name": "STRING"}, "Id": "c63"}, + "c64": {"Name": "IF", "T": {"Name": "STRING"}, "Id": "c64"}, + "c65": {"Name": "IGNORE", "T": {"Name": "STRING"}, "Id": "c65"}, + "c66": {"Name": "IN", "T": {"Name": "STRING"}, "Id": "c66"}, + "c67": {"Name": "INITIALLY", "T": {"Name": "STRING"}, "Id": "c67"}, + "c68": {"Name": "INNER", "T": {"Name": "STRING"}, "Id": "c68"}, + "c69": {"Name": "INTERSECT", "T": {"Name": "STRING"}, "Id": "c69"}, + "c70": {"Name": "INTERVAL", "T": {"Name": "STRING"}, "Id": "c70"}, + "c71": {"Name": "INTO", "T": {"Name": "STRING"}, "Id": "c71"}, + "c72": {"Name": "IN_P", "T": {"Name": "STRING"}, "Id": "c72"}, + "c73": {"Name": "IS", "T": {"Name": "STRING"}, "Id": "c73"}, + "c74": {"Name": "JOIN", "T": {"Name": "STRING"}, "Id": "c74"}, + "c75": {"Name": "LATERAL", "T": {"Name": "STRING"}, "Id": "c75"}, + "c76": {"Name": "LATERAL_P", "T": {"Name": "STRING"}, "Id": "c76"}, + "c77": {"Name": "LEADING", "T": {"Name": "STRING"}, "Id": "c77"}, + "c78": {"Name": "LEFT", "T": {"Name": "STRING"}, "Id": "c78"}, + "c79": {"Name": "LIKE", "T": {"Name": "STRING"}, "Id": "c79"}, + "c80": {"Name": "LIMIT", "T": {"Name": "STRING"}, "Id": "c80"}, + "c81": {"Name": "LOCALTIME", "T": {"Name": "STRING"}, "Id": "c81"}, + "c82": {"Name": "LOCALTIMESTAMP", "T": {"Name": "STRING"}, "Id": "c82"}, + "c83": {"Name": "LOOKUP", "T": {"Name": "STRING"}, "Id": "c83"}, + "c84": {"Name": "MERGE", "T": {"Name": "STRING"}, "Id": "c84"}, + "c85": {"Name": "NATURAL", "T": {"Name": "STRING"}, "Id": "c85"}, + "c86": {"Name": "NEW", "T": {"Name": "STRING"}, "Id": "c86"}, + "c87": {"Name": "NO", "T": {"Name": "STRING"}, "Id": "c87"}, + "c88": {"Name": "NOT", "T": {"Name": "STRING"}, "Id": "c88"}, + "c89": {"Name": "NULL", "T": {"Name": "STRING"}, "Id": "c89"}, + "c90": {"Name": "NULLS", "T": {"Name": "STRING"}, "Id": "c90"}, + "c91": {"Name": "NULL_P", "T": {"Name": "STRING"}, "Id": "c91"}, + "c92": {"Name": "OF", "T": {"Name": "STRING"}, "Id": "c92"}, + "c93": {"Name": "OFFSET", "T": {"Name": "STRING"}, "Id": "c93"}, + "c94": {"Name": "ON", "T": {"Name": "STRING"}, "Id": "c94"}, + "c95": {"Name": "ONLY", "T": {"Name": "STRING"}, "Id": "c95"}, + "c96": {"Name": "OR", "T": {"Name": "STRING"}, "Id": "c96"}, + "c97": {"Name": "ORDER", "T": {"Name": "STRING"}, "Id": "c97"}, + "c98": {"Name": "OUTER", "T": {"Name": "STRING"}, "Id": "c98"}, + "c99": {"Name": "OVER", "T": {"Name": "STRING"}, "Id": "c99"}, + "c100": {"Name": "PARTITION", "T": {"Name": "STRING"}, "Id": "c100"}, + "c101": {"Name": "PLACING", "T": {"Name": "STRING"}, "Id": "c101"}, + "c102": {"Name": "PRECEDING", "T": {"Name": "STRING"}, "Id": "c102"}, + "c103": {"Name": "PRIMARY", "T": {"Name": "STRING"}, "Id": "c103"}, + "c104": {"Name": "PROTO", "T": {"Name": "STRING"}, "Id": "c104"}, + "c105": {"Name": "RANGE", "T": {"Name": "STRING"}, "Id": "c105"}, + "c106": {"Name": "RECURSIVE", "T": {"Name": "STRING"}, "Id": "c106"}, + "c107": {"Name": "REFERENCES", "T": {"Name": "STRING"}, "Id": "c107"}, + "c108": {"Name": "RESPECT", "T": {"Name": "STRING"}, "Id": "c108"}, + "c109": {"Name": "RETURNING", "T": {"Name": "STRING"}, "Id": "c109"}, + "c110": {"Name": "RIGHT", "T": {"Name": "STRING"}, "Id": "c110"}, + "c111": {"Name": "ROLLUP", "T": {"Name": "STRING"}, "Id": "c111"}, + "c112": {"Name": "ROWS", "T": {"Name": "STRING"}, "Id": "c112"}, + "c113": {"Name": "SELECT", "T": {"Name": "STRING"}, "Id": "c113"}, + "c114": {"Name": "SESSION_USER", "T": {"Name": "STRING"}, "Id": "c114"}, + "c115": {"Name": "SET", "T": {"Name": "STRING"}, "Id": "c115"}, + "c116": {"Name": "SOME", "T": {"Name": "STRING"}, "Id": "c116"}, + "c117": {"Name": "STRUCT", "T": {"Name": "STRING"}, "Id": "c117"}, + "c118": {"Name": "SYMMETRIC", "T": {"Name": "STRING"}, "Id": "c118"}, + "c119": {"Name": "TABLE", "T": {"Name": "STRING"}, "Id": "c119"}, + "c120": {"Name": "TABLESAMPLE", "T": {"Name": "STRING"}, "Id": "c120"}, + "c121": {"Name": "THEN", "T": {"Name": "STRING"}, "Id": "c121"}, + "c122": {"Name": "TO", "T": {"Name": "STRING"}, "Id": "c122"}, + "c123": {"Name": "TRAILING", "T": {"Name": "STRING"}, "Id": "c123"}, + "c124": {"Name": "TREAT", "T": {"Name": "STRING"}, "Id": "c124"}, + "c125": {"Name": "TRUE", "T": {"Name": "STRING"}, "Id": "c125"}, + "c126": {"Name": "TRUE_P", "T": {"Name": "STRING"}, "Id": "c126"}, + "c127": {"Name": "UNBOUNDED", "T": {"Name": "STRING"}, "Id": "c127"}, + "c128": {"Name": "UNION", "T": {"Name": "STRING"}, "Id": "c128"}, + "c129": {"Name": "UNIQUE", "T": {"Name": "STRING"}, "Id": "c129"}, + "c130": {"Name": "UNNEST", "T": {"Name": "STRING"}, "Id": "c130"}, + "c131": {"Name": "USER", "T": {"Name": "STRING"}, "Id": "c131"}, + "c132": {"Name": "USING", "T": {"Name": "STRING"}, "Id": "c132"}, + "c133": {"Name": "VARIADIC", "T": {"Name": "STRING"}, "Id": "c133"}, + "c134": {"Name": "WHEN", "T": {"Name": "STRING"}, "Id": "c134"}, + "c135": {"Name": "WHERE", "T": {"Name": "STRING"}, "Id": "c135"}, + "c136": {"Name": "WINDOW", "T": {"Name": "STRING"}, "Id": "c136"}, + "c137": {"Name": "WITH", "T": {"Name": "STRING"}, "Id": "c137"}, + "c138": {"Name": "WITHIN", "T": {"Name": "STRING"}, "Id": "c138"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "true", + "Schema": "SRC_DATABASE", + "ColIds": ["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19", "c20", "c21", "c22", "c23", "c24", "c25", "c26", "c27", "c28", "c29", "c30", "c31", "c32", "c33", "c34", "c35", "c36", "c37", "c38", "c39", "c40", "c41", "c42", "c43", "c44", "c45", "c46", "c47", "c48", "c49", "c50", "c51", "c52", "c53", "c54", "c55", "c56", "c57", "c58", "c59", "c60", "c61", "c62", "c63", "c64", "c65", "c66", "c67", "c68", "c69", "c70", "c71", "c72", "c73", "c74", "c75", "c76", "c77", "c78", "c79", "c80", "c81", "c82", "c83", "c84", "c85", "c86", "c87", "c88", "c89", "c90", "c91", "c92", "c93", "c94", "c95", "c96", "c97", "c98", "c99", "c100", "c101", "c102", "c103", "c104", "c105", "c106", "c107", "c108", "c109", "c110", "c111", "c112", "c113", "c114", "c115", "c116", "c117", "c118", "c119", "c120", "c121", "c122", "c123", "c124", "c125", "c126", "c127", "c128", "c129", "c130", "c131", "c132", "c133", "c134", "c135", "c136", "c137", "c138"], + "ColDefs": { + "c1": {"Name": "id", "T": {"Name": "INT64"}, "NotNull": true, "Id": "c1"}, + "c2": {"Name": "ALL", "T": {"Name": "STRING"}, "Id": "c2"}, + "c3": {"Name": "ANALYSE", "T": {"Name": "STRING"}, "Id": "c3"}, + "c4": {"Name": "ANALYZE", "T": {"Name": "STRING"}, "Id": "c4"}, + "c5": {"Name": "AND", "T": {"Name": "STRING"}, "Id": "c5"}, + "c6": {"Name": "ANY", "T": {"Name": "STRING"}, "Id": "c6"}, + "c7": {"Name": "ARRAY", "T": {"Name": "STRING"}, "Id": "c7"}, + "c8": {"Name": "AS", "T": {"Name": "STRING"}, "Id": "c8"}, + "c9": {"Name": "ASC", "T": {"Name": "STRING"}, "Id": "c9"}, + "c10": {"Name": "ASSERT_ROWS_MODIFIED", "T": {"Name": "STRING"}, "Id": "c10"}, + "c11": {"Name": "ASYMMETRIC", "T": {"Name": "STRING"}, "Id": "c11"}, + "c12": {"Name": "AT", "T": {"Name": "STRING"}, "Id": "c12"}, + "c13": {"Name": "BETWEEN", "T": {"Name": "STRING"}, "Id": "c13"}, + "c14": {"Name": "BOTH", "T": {"Name": "STRING"}, "Id": "c14"}, + "c15": {"Name": "BY", "T": {"Name": "STRING"}, "Id": "c15"}, + "c16": {"Name": "CASE", "T": {"Name": "STRING"}, "Id": "c16"}, + "c17": {"Name": "CAST", "T": {"Name": "STRING"}, "Id": "c17"}, + "c18": {"Name": "CHECK", "T": {"Name": "STRING"}, "Id": "c18"}, + "c19": {"Name": "COLLATE", "T": {"Name": "STRING"}, "Id": "c19"}, + "c20": {"Name": "COLUMN", "T": {"Name": "STRING"}, "Id": "c20"}, + "c21": {"Name": "CONSTRAINT", "T": {"Name": "STRING"}, "Id": "c21"}, + "c22": {"Name": "CONTAINS", "T": {"Name": "STRING"}, "Id": "c22"}, + "c23": {"Name": "CREATE", "T": {"Name": "STRING"}, "Id": "c23"}, + "c24": {"Name": "CROSS", "T": {"Name": "STRING"}, "Id": "c24"}, + "c25": {"Name": "CUBE", "T": {"Name": "STRING"}, "Id": "c25"}, + "c26": {"Name": "CURRENT", "T": {"Name": "STRING"}, "Id": "c26"}, + "c27": {"Name": "CURRENT_CATALOG", "T": {"Name": "STRING"}, "Id": "c27"}, + "c28": {"Name": "CURRENT_DATE", "T": {"Name": "STRING"}, "Id": "c28"}, + "c29": {"Name": "CURRENT_ROLE", "T": {"Name": "STRING"}, "Id": "c29"}, + "c30": {"Name": "CURRENT_TIME", "T": {"Name": "STRING"}, "Id": "c30"}, + "c31": {"Name": "CURRENT_TIMESTAMP", "T": {"Name": "STRING"}, "Id": "c31"}, + "c32": {"Name": "CURRENT_USER", "T": {"Name": "STRING"}, "Id": "c32"}, + "c33": {"Name": "DEFAULT", "T": {"Name": "STRING"}, "Id": "c33"}, + "c34": {"Name": "DEFERRABLE", "T": {"Name": "STRING"}, "Id": "c34"}, + "c35": {"Name": "DEFINE", "T": {"Name": "STRING"}, "Id": "c35"}, + "c36": {"Name": "DESC", "T": {"Name": "STRING"}, "Id": "c36"}, + "c37": {"Name": "DISTINCT", "T": {"Name": "STRING"}, "Id": "c37"}, + "c38": {"Name": "DO", "T": {"Name": "STRING"}, "Id": "c38"}, + "c39": {"Name": "ELSE", "T": {"Name": "STRING"}, "Id": "c39"}, + "c40": {"Name": "END", "T": {"Name": "STRING"}, "Id": "c40"}, + "c41": {"Name": "END_P", "T": {"Name": "STRING"}, "Id": "c41"}, + "c42": {"Name": "ENUM", "T": {"Name": "STRING"}, "Id": "c42"}, + "c43": {"Name": "ESCAPE", "T": {"Name": "STRING"}, "Id": "c43"}, + "c44": {"Name": "EXCEPT", "T": {"Name": "STRING"}, "Id": "c44"}, + "c45": {"Name": "EXCLUDE", "T": {"Name": "STRING"}, "Id": "c45"}, + "c46": {"Name": "EXISTS", "T": {"Name": "STRING"}, "Id": "c46"}, + "c47": {"Name": "EXTRACT", "T": {"Name": "STRING"}, "Id": "c47"}, + "c48": {"Name": "FALSE", "T": {"Name": "STRING"}, "Id": "c48"}, + "c49": {"Name": "FALSE_P", "T": {"Name": "STRING"}, "Id": "c49"}, + "c50": {"Name": "FETCH", "T": {"Name": "STRING"}, "Id": "c50"}, + "c51": {"Name": "FOLLOWING", "T": {"Name": "STRING"}, "Id": "c51"}, + "c52": {"Name": "FOR", "T": {"Name": "STRING"}, "Id": "c52"}, + "c53": {"Name": "FOREIGN", "T": {"Name": "STRING"}, "Id": "c53"}, + "c54": {"Name": "FROM", "T": {"Name": "STRING"}, "Id": "c54"}, + "c55": {"Name": "FULL", "T": {"Name": "STRING"}, "Id": "c55"}, + "c56": {"Name": "GRANT", "T": {"Name": "STRING"}, "Id": "c56"}, + "c57": {"Name": "GRAPH_TABLE", "T": {"Name": "STRING"}, "Id": "c57"}, + "c58": {"Name": "GROUP", "T": {"Name": "STRING"}, "Id": "c58"}, + "c59": {"Name": "GROUPING", "T": {"Name": "STRING"}, "Id": "c59"}, + "c60": {"Name": "GROUPS", "T": {"Name": "STRING"}, "Id": "c60"}, + "c61": {"Name": "GROUP_P", "T": {"Name": "STRING"}, "Id": "c61"}, + "c62": {"Name": "HASH", "T": {"Name": "STRING"}, "Id": "c62"}, + "c63": {"Name": "HAVING", "T": {"Name": "STRING"}, "Id": "c63"}, + "c64": {"Name": "IF", "T": {"Name": "STRING"}, "Id": "c64"}, + "c65": {"Name": "IGNORE", "T": {"Name": "STRING"}, "Id": "c65"}, + "c66": {"Name": "IN", "T": {"Name": "STRING"}, "Id": "c66"}, + "c67": {"Name": "INITIALLY", "T": {"Name": "STRING"}, "Id": "c67"}, + "c68": {"Name": "INNER", "T": {"Name": "STRING"}, "Id": "c68"}, + "c69": {"Name": "INTERSECT", "T": {"Name": "STRING"}, "Id": "c69"}, + "c70": {"Name": "INTERVAL", "T": {"Name": "STRING"}, "Id": "c70"}, + "c71": {"Name": "INTO", "T": {"Name": "STRING"}, "Id": "c71"}, + "c72": {"Name": "IN_P", "T": {"Name": "STRING"}, "Id": "c72"}, + "c73": {"Name": "IS", "T": {"Name": "STRING"}, "Id": "c73"}, + "c74": {"Name": "JOIN", "T": {"Name": "STRING"}, "Id": "c74"}, + "c75": {"Name": "LATERAL", "T": {"Name": "STRING"}, "Id": "c75"}, + "c76": {"Name": "LATERAL_P", "T": {"Name": "STRING"}, "Id": "c76"}, + "c77": {"Name": "LEADING", "T": {"Name": "STRING"}, "Id": "c77"}, + "c78": {"Name": "LEFT", "T": {"Name": "STRING"}, "Id": "c78"}, + "c79": {"Name": "LIKE", "T": {"Name": "STRING"}, "Id": "c79"}, + "c80": {"Name": "LIMIT", "T": {"Name": "STRING"}, "Id": "c80"}, + "c81": {"Name": "LOCALTIME", "T": {"Name": "STRING"}, "Id": "c81"}, + "c82": {"Name": "LOCALTIMESTAMP", "T": {"Name": "STRING"}, "Id": "c82"}, + "c83": {"Name": "LOOKUP", "T": {"Name": "STRING"}, "Id": "c83"}, + "c84": {"Name": "MERGE", "T": {"Name": "STRING"}, "Id": "c84"}, + "c85": {"Name": "NATURAL", "T": {"Name": "STRING"}, "Id": "c85"}, + "c86": {"Name": "NEW", "T": {"Name": "STRING"}, "Id": "c86"}, + "c87": {"Name": "NO", "T": {"Name": "STRING"}, "Id": "c87"}, + "c88": {"Name": "NOT", "T": {"Name": "STRING"}, "Id": "c88"}, + "c89": {"Name": "NULL", "T": {"Name": "STRING"}, "Id": "c89"}, + "c90": {"Name": "NULLS", "T": {"Name": "STRING"}, "Id": "c90"}, + "c91": {"Name": "NULL_P", "T": {"Name": "STRING"}, "Id": "c91"}, + "c92": {"Name": "OF", "T": {"Name": "STRING"}, "Id": "c92"}, + "c93": {"Name": "OFFSET", "T": {"Name": "STRING"}, "Id": "c93"}, + "c94": {"Name": "ON", "T": {"Name": "STRING"}, "Id": "c94"}, + "c95": {"Name": "ONLY", "T": {"Name": "STRING"}, "Id": "c95"}, + "c96": {"Name": "OR", "T": {"Name": "STRING"}, "Id": "c96"}, + "c97": {"Name": "ORDER", "T": {"Name": "STRING"}, "Id": "c97"}, + "c98": {"Name": "OUTER", "T": {"Name": "STRING"}, "Id": "c98"}, + "c99": {"Name": "OVER", "T": {"Name": "STRING"}, "Id": "c99"}, + "c100": {"Name": "PARTITION", "T": {"Name": "STRING"}, "Id": "c100"}, + "c101": {"Name": "PLACING", "T": {"Name": "STRING"}, "Id": "c101"}, + "c102": {"Name": "PRECEDING", "T": {"Name": "STRING"}, "Id": "c102"}, + "c103": {"Name": "PRIMARY", "T": {"Name": "STRING"}, "Id": "c103"}, + "c104": {"Name": "PROTO", "T": {"Name": "STRING"}, "Id": "c104"}, + "c105": {"Name": "RANGE", "T": {"Name": "STRING"}, "Id": "c105"}, + "c106": {"Name": "RECURSIVE", "T": {"Name": "STRING"}, "Id": "c106"}, + "c107": {"Name": "REFERENCES", "T": {"Name": "STRING"}, "Id": "c107"}, + "c108": {"Name": "RESPECT", "T": {"Name": "STRING"}, "Id": "c108"}, + "c109": {"Name": "RETURNING", "T": {"Name": "STRING"}, "Id": "c109"}, + "c110": {"Name": "RIGHT", "T": {"Name": "STRING"}, "Id": "c110"}, + "c111": {"Name": "ROLLUP", "T": {"Name": "STRING"}, "Id": "c111"}, + "c112": {"Name": "ROWS", "T": {"Name": "STRING"}, "Id": "c112"}, + "c113": {"Name": "SELECT", "T": {"Name": "STRING"}, "Id": "c113"}, + "c114": {"Name": "SESSION_USER", "T": {"Name": "STRING"}, "Id": "c114"}, + "c115": {"Name": "SET", "T": {"Name": "STRING"}, "Id": "c115"}, + "c116": {"Name": "SOME", "T": {"Name": "STRING"}, "Id": "c116"}, + "c117": {"Name": "STRUCT", "T": {"Name": "STRING"}, "Id": "c117"}, + "c118": {"Name": "SYMMETRIC", "T": {"Name": "STRING"}, "Id": "c118"}, + "c119": {"Name": "TABLE", "T": {"Name": "STRING"}, "Id": "c119"}, + "c120": {"Name": "TABLESAMPLE", "T": {"Name": "STRING"}, "Id": "c120"}, + "c121": {"Name": "THEN", "T": {"Name": "STRING"}, "Id": "c121"}, + "c122": {"Name": "TO", "T": {"Name": "STRING"}, "Id": "c122"}, + "c123": {"Name": "TRAILING", "T": {"Name": "STRING"}, "Id": "c123"}, + "c124": {"Name": "TREAT", "T": {"Name": "STRING"}, "Id": "c124"}, + "c125": {"Name": "TRUE", "T": {"Name": "STRING"}, "Id": "c125"}, + "c126": {"Name": "TRUE_P", "T": {"Name": "STRING"}, "Id": "c126"}, + "c127": {"Name": "UNBOUNDED", "T": {"Name": "STRING"}, "Id": "c127"}, + "c128": {"Name": "UNION", "T": {"Name": "STRING"}, "Id": "c128"}, + "c129": {"Name": "UNIQUE", "T": {"Name": "STRING"}, "Id": "c129"}, + "c130": {"Name": "UNNEST", "T": {"Name": "STRING"}, "Id": "c130"}, + "c131": {"Name": "USER", "T": {"Name": "STRING"}, "Id": "c131"}, + "c132": {"Name": "USING", "T": {"Name": "STRING"}, "Id": "c132"}, + "c133": {"Name": "VARIADIC", "T": {"Name": "STRING"}, "Id": "c133"}, + "c134": {"Name": "WHEN", "T": {"Name": "STRING"}, "Id": "c134"}, + "c135": {"Name": "WHERE", "T": {"Name": "STRING"}, "Id": "c135"}, + "c136": {"Name": "WINDOW", "T": {"Name": "STRING"}, "Id": "c136"}, + "c137": {"Name": "WITH", "T": {"Name": "STRING"}, "Id": "c137"}, + "c138": {"Name": "WITHIN", "T": {"Name": "STRING"}, "Id": "c138"} + }, + "PrimaryKeys": [{"ColId": "c1", "Desc": false, "Order": 1}], + "Id": "t1" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-Articles.avro new file mode 100644 index 0000000000000000000000000000000000000000..01421913ae369ac9a7a5b0b9f603a78d42f76ae5 GIT binary patch literal 2484 zcmdUvO^ee&7{|L|sVjI8l?88-bF)qQw#nW+2zu}Z-P=;aWM;CRb&`oQ6QL}HqA0j$ z4}Jr|gXm}Q=EvBR_yQ{IMLamOO{dwmTPt|b913av&piM6J^z{8(cX%+4jyv0Az%+D z;NarEE&Ca^w``6fz?wTyx!%!M`_9^>F zuI7tBSFSke2qtryjCX7kR$c~-nlqz4v z1iMq>j0qwMOO10=P;i?h7xqB9t8%6zIp%NFbCxF}M3`cUCjl|?h%t&ms^)THG8yPo z1}1tl-sy=LBhL~*Ct3LF$; z-)UjwI{}O%r;VHL@#DGfhu2TueS&Wehw68|bmLt~`r-j~Ac9UrQWdrrD;a0{su1;d ztKIVapzQ~4yG|JdI#!~d3kTvVGfDKi_|yVtT_eLhn;Kfa+?sb{ScRI%pj ztaCSR-Me;o+p1WWb*ZRPi&(?qvqy(VPpg)-3@z_JrgiXbIa*3rqP1G0)%Y`7UXiWg z_kTBABX4uE_3>yqT1r=ehEYY#&lop8f%tQ zr6Dh!(v{=F=v#=35hK74rad^lpEJ&xLX%oXB`Qhi|II0qV@kB^4*g>vIW-{GZr21- z$m2_PYp6`A$0u~WuBH-FTa6_@1?;ealwf0-rn&WF-mxWVfDW$#W}1f-G^{3DwtPmo z7O?8OhJ;g>YF@r(lP#5iTB|7~%R!De+oP*dD`Lhb&RDIHLsV3po4#fa4wuBGP)svX zxXxX`jN4*z$q^JIGiP=r=lq@en)FP?Ddkkt#eg_@A_S*U7+R-%Z@CB zt4_e>>5OZ~(3NF{M#C0LX$BS7Xy3A_0l6}zE@-~BN{ve3jToADSsdlA&LJKW{{;KKu1fPvUnU7AWDrFZIzgCV(_Al$x>?WiWwTljA7=}|TDiK16E{GA`#1J=eY`0E@7?6-)0R-JjWS!6F_S&=0+?|o4sspMj zA;iFh*qB%=#4jjfLHr8Dgv7wk0(VV(F=-k+FwjVelKB1Zd+&L_yL-|0f^``_;cQu; zEu6ylOS?c0a}3r1$B6M5v_S?_{hK=BO~!VF`X?cn#saKu?3NFUf@tZuR7g&;O|aKC z&pC#1C`pQiglTS+rWs{Q-v*Rz5(Me`$br$@DWQ}Iuvb}8u!1A}Z`WkeHb&C81E=gG z#d3?y-L5LcFb$8%&7m@49`4}byqxhOqqu< zl&IFN!WyZNF7;~FUF4}_GrkXQ7k|lqmp0PX;VT>u3crqYH9x+BS%v3IaB$LA- zWpJW5@lGc)5uQw0Wt$a&08FNB3U06eVSQtfuI*Z3@Z zkAB?v@aEw8wJUEwUEgngS-KtdqGZMO`gWY)!1iz)*nN~lwvXMeemUy>?eO5^cXagb zsrtNB-a=Q?p?FMf6eBw#nHqrC3pr={_==A2`JNN>{h;6VJCq@)qZXY)*a&BmL+W%! z2UK*n1 zj0g}TNGwc@ObDs!0%C!MfxiGKf(bEHg1aWUxJ{FGU?D@4IPbgfeeb>dawS+EGf#pQ zMyEMkLJ`UEY^_@74OLPSz`J{;v3At5USnd6ordUC{rCcx^ zFIw%q{+%HZHUy3k7a&SBQa2-1;`0`vix>hj-(jHfc7zGR+-he|rk>koj1fs^4awAmfXx_MmTBCY$%7nsj4brj!#(=8D0!7M?9%H4X6tupJ zV3IrtfdGSt?KHfraHe3;cL4&T5;4?x)hYmdBqm`v0<5*a8G9pUN0YH>G;xN=f)%HP zNM2nB>zUvva6&m2n8up9F6ipkPcB>n@uJL`iezt2jo#&W%o`yhNT7Z|G(4b`AP`Gm zwmFl47NMYz*Y@soqz@c)B6I(AKO76Ip0`5O1St_#O^K3GVp}|QQE#T; zgO)%ltJ^Pgh@f;?d6LMnFjwl=JXcyIFTC6@GrnK4D_{oLei?dp(J$)P`s252 z-{8{^Yw~@nw<6C*Eq%(PML=G&J#WH)z=CZluuu&BJ0V;YHa)j{aM{<*!-PxO&JqX7^ zun_!FXlG;Ve_$tKDYiD&mfF||md;#~NiM&%urh&Ve|+D(H=pmDcL#NrytA+;#j3*F zG=?u{_I=$=slVY%iiIS8!A~Hz&pd2CP*f`Oro|{H%HP=9AAZa-W~GBrr6o(+{z1Wg zmlQ}-vzRIkaq5&V85c%h@VRI+41BX+fzt;uKHAsCG^mMrn@nPykg9l4t2G0~lV_@STw{o$x+Ij!>&3XC!)VbZ8 zpYtD&U$4L#L8x8{0AmtVs^wNtqY(TO2-~QgWFSopvK5!AP zayBB+G7jM0g*8joB5X}s93#ekD{X}^(BG+3nHQJ~^-Gmt>I-XfVXbj7j)|7mONHbl zT(Z{F<~qmFuSgPLAz=_1rD;Z)(x)xTmI#8hoH#IgJ0O%2VXZfvC|Jx9{w3GY5W}EyK&}gwgn8vYu664v7?)b3$tQpfHpn^DwM61T&!gd3lT<(# zI{;JWp${c&Kdg1&b;3;r?Q@sVqu3nPXx}s@-O7Wat0v+kfV|e*J(?a(O;l_uRnlZ+ z!3twam9P2)7hA*;6GRdg8t0~<&h0R{Xc>k}DrY*94gN;m;dmk{KBidW!+;pM#~8&h zRC75pnF!V>gNJ%E-qwj63fm{q^!%BAB$U=-+VXJ~##CC98cGw1Bk?pvje9!24lZ>- zk$^kCNW`8%oOqqM{H->k3fi&oyY!Th%OaHRb`9c#*4-8l;!G``Hi~M1i|eVg4_!R) zKHRxJv$r(8!kClI<;EupRnN!%c+SZ=o;#M!jw3fW01{)V zx*=s|MEiE~VE4u8)R}{Sfqv?Me&VMmEQUQ4^>EldI`Ck`T-&c;K4C^=#|or~Oksg3`2@wNwC aN(3&?&lRthOSXwuD>YwJyJx7i?z2Bsr)3cU literal 0 HcmV?d00001 diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..60816e9bc9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,40 @@ +CREATE TABLE Movie ( + id INT64 NOT NULL, + name STRING(200), + actor NUMERIC, + startTime TIMESTAMP, +) PRIMARY KEY(id); + +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + subscribed BOOL, + plan STRING(1), + startDate TIMESTAMP, +) PRIMARY KEY (id); + +CREATE TABLE Authors ( + author_id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (author_id); + +CREATE TABLE Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, +) PRIMARY KEY (id); + +CREATE INDEX author_id_6 ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id); + +CREATE TABLE Articles ( + id INT64 NOT NULL, + name STRING(200), + published_date TIMESTAMP, + author_id INT64 NOT NULL, +) PRIMARY KEY (author_id, id), +INTERLEAVE IN PARENT Authors; + +CREATE INDEX author_id ON Articles (author_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql new file mode 100644 index 0000000000..dcdbc6bf55 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerEventsIT/oracle-schema.sql @@ -0,0 +1,44 @@ +CREATE TABLE "Movie" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "actor" NUMBER, + "startTime" TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE "Users" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "age" INTEGER, + "subscribed" NUMBER(1), + "plan" CHAR(1), + "startDate" DATE, + PRIMARY KEY ("id") +); + +CREATE TABLE "Authors" ( + "author_id" INTEGER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Articles" ( + "id" INTEGER NOT NULL, + "name" VARCHAR2(200), + "published_date" DATE, + "author_id" INTEGER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Articles" add FOREIGN KEY ("author_id") references "Authors"("author_id"); +CREATE INDEX "author_id" ON "Articles" ("author_id"); + +CREATE TABLE "Books" ( + "id" INTEGER NOT NULL, + "title" VARCHAR2(200), + "author_id" INTEGER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" add FOREIGN KEY ("author_id") references "Authors"("author_id"); +CREATE INDEX "author_id_6" ON "Books" ("author_id"); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..71ee384cbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..fd61478626 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500), + "last_name1" VARCHAR2(500), + "first_name2" VARCHAR2(500), + "last_name2" VARCHAR2(500), + "first_name3" VARCHAR2(500), + "last_name3" VARCHAR2(500), + "ID" NUMBER NOT NULL PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json new file mode 100644 index 0000000000..c0b5fc0009 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerFileOverridesIT/override.json @@ -0,0 +1,10 @@ +{ + "renamedTables":{ + "person1":"human1" + }, + "renamedColumns":{ + "person1":{ + "first_name1":"name1" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..62f8aaa748 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS Authors ( + author_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY (author_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), +) PRIMARY KEY (synth_id); + +CREATE INDEX author_id ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id) ON DELETE NO ACTION; + +CREATE TABLE IF NOT EXISTS Genre ( + name STRING(200), + genre_id INT64 NOT NULL, +) PRIMARY KEY (genre_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql new file mode 100644 index 0000000000..c2bb9f0fca --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE "Authors" ( + "author_id" NUMBER(10,0) NOT NULL, + "name" VARCHAR2(25), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER(10,0) NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER(10,0), + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD CONSTRAINT "Books_ibfk_1" FOREIGN KEY ("author_id") REFERENCES "Authors" ("author_id"); + +CREATE TABLE "Genre" ( + "name" VARCHAR2(200), + "genre_id" NUMBER(10,0) NOT NULL, + PRIMARY KEY ("genre_id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json new file mode 100644 index 0000000000..b7ff136a40 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json @@ -0,0 +1,506 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "e2e-it", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t34": { + "Name": "Authors", + "ColIds": [ + "c37", + "c38" + ], + "ShardIdColumn": "", + "ColDefs": { + "c37": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: author_id int(10)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Authors", + "Id": "t34" + }, + "t35": { + "Name": "Books", + "ColIds": [ + "c41", + "c42", + "c43", + "c47" + ], + "ShardIdColumn": "", + "ColDefs": { + "c41": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c47": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "Id": "f40", + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION" + } + ], + "Indexes": [ + { + "Name": "author_id", + "TableId": "t35", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t35" + } + }, + "SyntheticPKeys": { + "t35": { + "ColId": "c47", + "Sequence": 0 + } + }, + "SrcSchema": { + "t34": { + "Name": "Authors", + "Schema": "e2e-it", + "ColIds": [ + "c37", + "c38", + "c39" + ], + "ColDefs": { + "c37": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c39": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t34" + }, + "t35": { + "Name": "Books", + "Schema": "e2e-it", + "ColIds": [ + "c41", + "c42", + "c43" + ], + "ColDefs": { + "c41": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION", + "Id": "f40" + } + ], + "Indexes": [ + { + "Name": "author_id", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "Id": "t35" + }, + "t36": { + "Name": "Genre", + "Schema": "e2e-it", + "ColIds": [ + "c44", + "c45" + ], + "ColDefs": { + "c44": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c45": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c45", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t36" + } + }, + "SchemaIssues": { + "t34": { + "ColumnLevelIssues": { + "c37": [ + 14 + ], + "c38": [] + }, + "TableLevelIssues": null + }, + "t35": { + "ColumnLevelIssues": { + "c41": [ + 14 + ], + "c42": [], + "c43": [ + 14, + 24 + ], + "c47": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t36": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..d57b1f3c9e --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS Category ( + category_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY(category_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), + extraCol1 INT64, +) PRIMARY KEY(synth_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql new file mode 100644 index 0000000000..6de37949db --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-schema.sql @@ -0,0 +1,13 @@ +CREATE TABLE "Category" ( + "category_id" NUMBER PRIMARY KEY, + "name" VARCHAR2(25), + "last_update" TIMESTAMP +); +ALTER TABLE "Category" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +CREATE TABLE "Books" ( + "id" NUMBER, + "title" VARCHAR2(200), + "author_id" NUMBER +); +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json new file mode 100644 index 0000000000..43305ab260 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerSessionIT/oracle-session.json @@ -0,0 +1,456 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "mysql-schema.sql", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t15515": { + "Name": "Category", + "ColIds": [ + "c15516", + "c15517" + ], + "ShardIdColumn": "", + "ColDefs": { + "c15516": { + "Name": "category_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: category_id tinyint(4)", + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Category", + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "ColIds": [ + "c15520", + "c15521", + "c15522", + "c15523", + "c1" + ], + "ShardIdColumn": "", + "ColDefs": { + "c1": { + "Name": "extraCol1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c1", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15520": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(11)", + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15523": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c15523", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15523", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t15519" + } + }, + "SyntheticPKeys": { + "t15519": { + "ColId": "c15523", + "Sequence": 0 + } + }, + "SrcSchema": { + "t15515": { + "Name": "Category", + "Schema": "", + "ColIds": [ + "c15516", + "c15517", + "c15518" + ], + "ColDefs": { + "c15516": { + "Name": "category_id", + "Type": { + "Name": "tinyint", + "Mods": [ + 4 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15518": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15518", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "Schema": "", + "ColIds": [ + "c15520", + "c15521", + "c15522" + ], + "ColDefs": { + "c15520": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "Type": { + "Name": "varchar", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": null, + "Indexes": null, + "Id": "t15519" + }, + "t15520": { + "Name": "Genre", + "Schema": "", + "ColIds": [ + "c13", + "c14" + ], + "ColDefs": { + "c13": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + }, + "c14": { + "Name": "genre_id", + "Type": { + "Name": "decimal", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14" + } + }, + "PrimaryKeys": [ + { + "ColId": "c14", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15520" + } + }, + "SchemaIssues": { + "t15515": { + "ColumnLevelIssues": { + "c15516": [ + 14 + ], + "c15517": [] + }, + "TableLevelIssues": null + }, + "t15519": { + "ColumnLevelIssues": { + "c15520": [ + 14 + ], + "c15521": [], + "c15522": [ + 14 + ], + "c15523": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t15520": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..f677c0086c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,7 @@ +ALTER DATABASE db SET OPTIONS (default_time_zone = 'Australia/Brisbane'); + +CREATE TABLE IF NOT EXISTS DateData ( + id INT64 NOT NULL, + timestamp_column TIMESTAMP, + datetime_column TIMESTAMP, +) PRIMARY KEY(id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql new file mode 100644 index 0000000000..2a3ecbf1b2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDataStreamToSpannerTimezoneIT/oracle-schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE "DateData" ( + "id" NUMBER PRIMARY KEY, + "timestamp_column" TIMESTAMP, + "datetime_column" TIMESTAMP +); + +ALTER TABLE "DateData" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..8e5f5c2236 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,71 @@ +CREATE TABLE T_SPRG_R0_VARCHAR2 (ID INT64 NOT NULL, COL_0 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R1_VARCHAR (ID INT64 NOT NULL, COL_1 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R2_CHAR (ID INT64 NOT NULL, COL_2 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R3_CHARACTER (ID INT64 NOT NULL, COL_3 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID INT64 NOT NULL, COL_4 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R5_NCHAR (ID INT64 NOT NULL, COL_5 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID INT64 NOT NULL, COL_6 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R7_NATIONAL_C (ID INT64 NOT NULL, COL_7 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R8_NATIONAL_C (ID INT64 NOT NULL, COL_8 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R9_NATIONAL_C (ID INT64 NOT NULL, COL_9 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R10_NATIONAL_C (ID INT64 NOT NULL, COL_10 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R11_NUMBER (ID INT64 NOT NULL, COL_11 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R12_NUMERIC (ID INT64 NOT NULL, COL_12 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R13_DECIMAL (ID INT64 NOT NULL, COL_13 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R14_DEC (ID INT64 NOT NULL, COL_14 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R15_FLOAT (ID INT64 NOT NULL, COL_15 NUMERIC) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID INT64 NOT NULL, COL_16 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R17_REAL (ID INT64 NOT NULL, COL_17 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R18_BINARY_FLO (ID INT64 NOT NULL, COL_18 FLOAT32) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R19_BINARY_DOU (ID INT64 NOT NULL, COL_19 FLOAT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R20_INTEGER (ID INT64 NOT NULL, COL_20 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R21_INT (ID INT64 NOT NULL, COL_21 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R22_SMALLINT (ID INT64 NOT NULL, COL_22 INT64) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R23_DATE (ID INT64 NOT NULL, COL_23 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R24_TIMESTAMP (ID INT64 NOT NULL, COL_24 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID INT64 NOT NULL, COL_25 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID INT64 NOT NULL, COL_26 TIMESTAMP) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R28_INTERVAL_D (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R29_RAW (ID INT64 NOT NULL, COL_29 BYTES(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R30_LONG_RAW (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R31_BLOB (ID INT64 NOT NULL, COL_31 BYTES(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R32_CLOB (ID INT64 NOT NULL, COL_32 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R33_NCLOB (ID INT64 NOT NULL, COL_33 STRING(MAX)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R34_BFILE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R35_LONG (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R36_ROWID (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R37_UROWID (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R38_BOOLEAN (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R39_JSON (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R40_XMLTYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R44_VECTOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R45_ANYDATA (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R46_ANYTYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R47_ANYDATASET (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R48_URITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R49_DBURITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R50_XDBURITYPE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R51_HTTPURITYP (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R52_EXPRESSION (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R53_ORDAUDIO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R54_ORDIMAGE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R55_ORDVIDEO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R56_ORDDOC (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R57_SI_STILLIM (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R58_SI_COLOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R60_SI_COLORHI (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R61_SI_POSITIO (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R63_SI_FEATURE (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R64_MLSLABEL (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R65_REF (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R66_REF_CURSOR (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R67_VARRAY (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R68_NESTED_TAB (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); +CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID INT64 NOT NULL, DUMMY STRING(10)) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql new file mode 100644 index 0000000000..b6a89adea2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerDataTypesIT/oracle-schema.sql @@ -0,0 +1,142 @@ +CREATE TABLE T_RZFA_VARCHAR2_TABLE (ID NUMBER PRIMARY KEY, VARCHAR2_COL VARCHAR2(255)); +INSERT INTO T_RZFA_VARCHAR2_TABLE (ID, VARCHAR2_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_VARCHAR_TABLE (ID NUMBER PRIMARY KEY, VARCHAR_COL VARCHAR(255)); +INSERT INTO T_RZFA_VARCHAR_TABLE (ID, VARCHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CHAR_TABLE (ID NUMBER PRIMARY KEY, CHAR_COL CHAR(255)); +INSERT INTO T_RZFA_CHAR_TABLE (ID, CHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CHARACTER_TABLE (ID NUMBER PRIMARY KEY, CHARACTER_COL CHARACTER(255)); +INSERT INTO T_RZFA_CHARACTER_TABLE (ID, CHARACTER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NVARCHAR2_TABLE (ID NUMBER PRIMARY KEY, NVARCHAR2_COL NVARCHAR2(255)); +INSERT INTO T_RZFA_NVARCHAR2_TABLE (ID, NVARCHAR2_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCHAR_TABLE (ID NUMBER PRIMARY KEY, NCHAR_COL NCHAR(255)); +INSERT INTO T_RZFA_NCHAR_TABLE (ID, NCHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCHAR_VARYING_TABLE (ID NUMBER PRIMARY KEY, NCHAR_VARYING_COL NCHAR VARYING(255)); +INSERT INTO T_RZFA_NCHAR_VARYING_TABLE (ID, NCHAR_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHARACTER_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHARACTER_COL NATIONAL CHARACTER(255)); +INSERT INTO T_RZFA_NATIONAL_CHARACTER_TABLE (ID, NATIONAL_CHARACTER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHAR_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHAR_COL NATIONAL CHAR(255)); +INSERT INTO T_RZFA_NATIONAL_CHAR_TABLE (ID, NATIONAL_CHAR_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHARACTER_VARYING_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHARACTER_VARYING_COL NATIONAL CHARACTER VARYING(255)); +INSERT INTO T_RZFA_NATIONAL_CHARACTER_VARYING_TABLE (ID, NATIONAL_CHARACTER_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NATIONAL_CHAR_VARYING_TABLE (ID NUMBER PRIMARY KEY, NATIONAL_CHAR_VARYING_COL NATIONAL CHAR VARYING(255)); +INSERT INTO T_RZFA_NATIONAL_CHAR_VARYING_TABLE (ID, NATIONAL_CHAR_VARYING_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NUMBER_TABLE (ID NUMBER PRIMARY KEY, NUMBER_COL NUMBER); +INSERT INTO T_RZFA_NUMBER_TABLE (ID, NUMBER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NUMERIC_TABLE (ID NUMBER PRIMARY KEY, NUMERIC_COL NUMERIC); +INSERT INTO T_RZFA_NUMERIC_TABLE (ID, NUMERIC_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DECIMAL_TABLE (ID NUMBER PRIMARY KEY, DECIMAL_COL DECIMAL); +INSERT INTO T_RZFA_DECIMAL_TABLE (ID, DECIMAL_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DEC_TABLE (ID NUMBER PRIMARY KEY, DEC_COL DEC); +INSERT INTO T_RZFA_DEC_TABLE (ID, DEC_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_FLOAT_TABLE (ID NUMBER PRIMARY KEY, FLOAT_COL FLOAT); +INSERT INTO T_RZFA_FLOAT_TABLE (ID, FLOAT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DOUBLE_PRECISION_TABLE (ID NUMBER PRIMARY KEY, DOUBLE_PRECISION_COL DOUBLE PRECISION); +INSERT INTO T_RZFA_DOUBLE_PRECISION_TABLE (ID, DOUBLE_PRECISION_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_REAL_TABLE (ID NUMBER PRIMARY KEY, REAL_COL REAL); +INSERT INTO T_RZFA_REAL_TABLE (ID, REAL_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BINARY_FLOAT_TABLE (ID NUMBER PRIMARY KEY, BINARY_FLOAT_COL BINARY_FLOAT); +INSERT INTO T_RZFA_BINARY_FLOAT_TABLE (ID, BINARY_FLOAT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BINARY_DOUBLE_TABLE (ID NUMBER PRIMARY KEY, BINARY_DOUBLE_COL BINARY_DOUBLE); +INSERT INTO T_RZFA_BINARY_DOUBLE_TABLE (ID, BINARY_DOUBLE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INTEGER_TABLE (ID NUMBER PRIMARY KEY, INTEGER_COL INTEGER); +INSERT INTO T_RZFA_INTEGER_TABLE (ID, INTEGER_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INT_TABLE (ID NUMBER PRIMARY KEY, INT_COL INT); +INSERT INTO T_RZFA_INT_TABLE (ID, INT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_SMALLINT_TABLE (ID NUMBER PRIMARY KEY, SMALLINT_COL SMALLINT); +INSERT INTO T_RZFA_SMALLINT_TABLE (ID, SMALLINT_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_DATE_TABLE (ID NUMBER PRIMARY KEY, DATE_COL DATE); +INSERT INTO T_RZFA_DATE_TABLE (ID, DATE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_COL TIMESTAMP); +INSERT INTO T_RZFA_TIMESTAMP_TABLE (ID, TIMESTAMP_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_WITH_TIME_ZONE_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_WITH_TIME_ZONE_COL TIMESTAMP WITH TIME ZONE); +INSERT INTO T_RZFA_TIMESTAMP_WITH_TIME_ZONE_TABLE (ID, TIMESTAMP_WITH_TIME_ZONE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_TIMESTAMP_WITH_LOCAL_TIME_ZONE_TABLE (ID NUMBER PRIMARY KEY, TIMESTAMP_WITH_LOCAL_TIME_ZONE_COL TIMESTAMP WITH LOCAL TIME ZONE); +INSERT INTO T_RZFA_TIMESTAMP_WITH_LOCAL_TIME_ZONE_TABLE (ID, TIMESTAMP_WITH_LOCAL_TIME_ZONE_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_INTERVAL_YEAR_TO_MONTH_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_INTERVAL_YEAR_TO_MONTH_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_INTERVAL_DAY_TO_SECOND_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_INTERVAL_DAY_TO_SECOND_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_RAW_TABLE (ID NUMBER PRIMARY KEY, RAW_COL RAW(255)); +INSERT INTO T_RZFA_RAW_TABLE (ID, RAW_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_LONG_RAW_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_LONG_RAW_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_BLOB_TABLE (ID NUMBER PRIMARY KEY, BLOB_COL BLOB); +INSERT INTO T_RZFA_BLOB_TABLE (ID, BLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_CLOB_TABLE (ID NUMBER PRIMARY KEY, CLOB_COL CLOB); +INSERT INTO T_RZFA_CLOB_TABLE (ID, CLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_NCLOB_TABLE (ID NUMBER PRIMARY KEY, NCLOB_COL NCLOB); +INSERT INTO T_RZFA_NCLOB_TABLE (ID, NCLOB_COL) VALUES (1, NULL); +CREATE TABLE T_RZFA_BFILE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_BFILE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_LONG_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_LONG_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ROWID_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ROWID_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_UROWID_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_UROWID_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_BOOLEAN_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_BOOLEAN_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_JSON_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_JSON_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_XMLTYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_XMLTYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_GEOMETRY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_GEOMETRY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_TOPO_GEOMETRY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_TOPO_GEOMETRY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SDO_GEORASTER_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SDO_GEORASTER_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_VECTOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_VECTOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYDATA_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYDATA_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYTYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYTYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ANYDATASET_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ANYDATASET_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_URITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_URITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_DBURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_DBURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_XDBURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_XDBURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_HTTPURITYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_HTTPURITYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_EXPRESSION_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_EXPRESSION_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDAUDIO_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDAUDIO_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDIMAGE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDIMAGE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDVIDEO_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDVIDEO_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ORDDOC_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ORDDOC_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_STILLIMAGE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_STILLIMAGE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_COLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_COLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_AVERAGECOLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_AVERAGECOLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_COLORHISTOGRAM_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_COLORHISTOGRAM_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_POSITIONALCOLOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_POSITIONALCOLOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_TEXTURE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_TEXTURE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_SI_FEATURELIST_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_SI_FEATURELIST_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_MLSLABEL_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_MLSLABEL_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_REF_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_REF_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_REF_CURSOR_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_REF_CURSOR_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_VARRAY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_VARRAY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_NESTED_TABLE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_NESTED_TABLE_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_ASSOCIATIVE_ARRAY_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_ASSOCIATIVE_ARRAY_TABLE (ID, DUMMY) VALUES (1, 'test'); +CREATE TABLE T_RZFA_OBJECT_TYPE_TABLE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10)); +INSERT INTO T_RZFA_OBJECT_TYPE_TABLE (ID, DUMMY) VALUES (1, 'test'); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql new file mode 100644 index 0000000000..3c8cf1eee1 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..7c5c7a9e65 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,181 @@ +CREATE TABLE `LargeKey` ( + `pk_col1` STRING(255) NOT NULL , + `pk_col2` STRING(255) NOT NULL , + `pk_col3` STRING(255) NOT NULL , + `col1` STRING(255), + `col2` STRING(255), + `col3` STRING(255), + `value_col` STRING(MAX), +) PRIMARY KEY (`pk_col1`, `pk_col2`, `pk_col3`); + +CREATE INDEX `large_index` ON `LargeKey` (`col1`, `col2`, `col3`); + +CREATE TABLE `LargeCell` ( + `id` INT64 NOT NULL , + `max_string_col_to_bytes` BYTES(MAX), + `max_string_col_to_str` STRING(MAX), +) PRIMARY KEY (`id`); + +CREATE TABLE `WideRow` ( + `id` INT64 NOT NULL , + `col1` BYTES(MAX), + `col2` BYTES(MAX), + `col3` BYTES(MAX), + `col4` BYTES(MAX), + `col5` BYTES(MAX), + `col6` BYTES(MAX), + `col7` BYTES(MAX), + `col8` BYTES(MAX), + `col9` BYTES(MAX), + `col10` BYTES(MAX), + `col11` BYTES(MAX), + `col12` BYTES(MAX), + `col13` BYTES(MAX), + `col14` BYTES(MAX), + `col15` BYTES(MAX), + `col16` BYTES(MAX), + `col17` BYTES(MAX), + `col18` BYTES(MAX), + `col19` BYTES(MAX), + `col20` BYTES(MAX), + `col21` BYTES(MAX), + `col22` BYTES(MAX), + `col23` BYTES(MAX), + `col24` BYTES(MAX), + `col25` BYTES(MAX), + `col26` BYTES(MAX), + `col27` BYTES(MAX), + `col28` BYTES(MAX), + `col29` BYTES(MAX), + `col30` BYTES(MAX), + `col31` BYTES(MAX), + `col32` BYTES(MAX), + `col33` BYTES(MAX), + `col34` BYTES(MAX), + `col35` BYTES(MAX), + `col36` BYTES(MAX), + `col37` BYTES(MAX), + `col38` BYTES(MAX), + `col39` BYTES(MAX), + `col40` BYTES(MAX), + `col41` BYTES(MAX), + `col42` BYTES(MAX), + `col43` BYTES(MAX), + `col44` BYTES(MAX), + `col45` BYTES(MAX), + `col46` BYTES(MAX), + `col47` BYTES(MAX), + `col48` BYTES(MAX), + `col49` BYTES(MAX), + `col50` BYTES(MAX), + `col51` BYTES(MAX), + `col52` BYTES(MAX), + `col53` BYTES(MAX), + `col54` BYTES(MAX), + `col55` BYTES(MAX), + `col56` BYTES(MAX), + `col57` BYTES(MAX), + `col58` BYTES(MAX), + `col59` BYTES(MAX), + `col60` BYTES(MAX), + `col61` BYTES(MAX), + `col62` BYTES(MAX), + `col63` BYTES(MAX), + `col64` BYTES(MAX), + `col65` BYTES(MAX), + `col66` BYTES(MAX), + `col67` BYTES(MAX), + `col68` BYTES(MAX), + `col69` BYTES(MAX), + `col70` BYTES(MAX), + `col71` BYTES(MAX), + `col72` BYTES(MAX), + `col73` BYTES(MAX), + `col74` BYTES(MAX), + `col75` BYTES(MAX), + `col76` BYTES(MAX), + `col77` BYTES(MAX), + `col78` BYTES(MAX), + `col79` BYTES(MAX), + `col80` BYTES(MAX), + `col81` BYTES(MAX), + `col82` BYTES(MAX), + `col83` BYTES(MAX), + `col84` BYTES(MAX), + `col85` BYTES(MAX), + `col86` BYTES(MAX), + `col87` BYTES(MAX), + `col88` BYTES(MAX), + `col89` BYTES(MAX), + `col90` BYTES(MAX), + `col91` BYTES(MAX), + `col92` BYTES(MAX), + `col93` BYTES(MAX), + `col94` BYTES(MAX), + `col95` BYTES(MAX), + `col96` BYTES(MAX), + `col97` BYTES(MAX), + `col98` BYTES(MAX), + `col99` BYTES(MAX), + `col100` BYTES(MAX), + `col101` BYTES(MAX), + `col102` BYTES(MAX), + `col103` BYTES(MAX), + `col104` BYTES(MAX), + `col105` BYTES(MAX), + `col106` BYTES(MAX), + `col107` BYTES(MAX), + `col108` BYTES(MAX), + `col109` BYTES(MAX), + `col110` BYTES(MAX), + `col111` BYTES(MAX), + `col112` BYTES(MAX), + `col113` BYTES(MAX), + `col114` BYTES(MAX), + `col115` BYTES(MAX), + `col116` BYTES(MAX), + `col117` BYTES(MAX), + `col118` BYTES(MAX), + `col119` BYTES(MAX), + `col120` BYTES(MAX), + `col121` BYTES(MAX), + `col122` BYTES(MAX), + `col123` BYTES(MAX), + `col124` BYTES(MAX), + `col125` BYTES(MAX), + `col126` BYTES(MAX), + `col127` BYTES(MAX), + `col128` BYTES(MAX), + `col129` BYTES(MAX), + `col130` BYTES(MAX), + `col131` BYTES(MAX), + `col132` BYTES(MAX), + `col133` BYTES(MAX), + `col134` BYTES(MAX), + `col135` BYTES(MAX), + `col136` BYTES(MAX), + `col137` BYTES(MAX), + `col138` BYTES(MAX), + `col139` BYTES(MAX), + `col140` BYTES(MAX), + `col141` BYTES(MAX), + `col142` BYTES(MAX), + `col143` BYTES(MAX), + `col144` BYTES(MAX), + `col145` BYTES(MAX), + `col146` BYTES(MAX), + `col147` BYTES(MAX), + `col148` BYTES(MAX), + `col149` BYTES(MAX), + `col150` BYTES(MAX), + `col151` BYTES(MAX), + `col152` BYTES(MAX), + `col153` BYTES(MAX), + `col154` BYTES(MAX), + `col155` BYTES(MAX), + `col156` BYTES(MAX), + `col157` BYTES(MAX), + `col158` BYTES(MAX), + `col159` BYTES(MAX), + `col160` BYTES(MAX), +) PRIMARY KEY (`id`); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql new file mode 100644 index 0000000000..e18c79f955 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/oracle-schema.sql @@ -0,0 +1,28 @@ +CREATE TABLE "LargeKey" ( + "pk_col1" VARCHAR2(255) NOT NULL, + "pk_col2" VARCHAR2(255) NOT NULL, + "pk_col3" VARCHAR2(255) NOT NULL, + "col1" VARCHAR2(255), + "col2" VARCHAR2(255), + "col3" VARCHAR2(255), + "value_col" CLOB, + PRIMARY KEY ("pk_col1", "pk_col2", "pk_col3") +); + +CREATE INDEX "large_index" ON "LargeKey" ("col1", "col2", "col3"); + +INSERT INTO "LargeKey" VALUES ( + RPAD('A', 255, 'A'), + RPAD('B', 255, 'B'), + RPAD('C', 255, 'C'), + RPAD('A', 255, 'A'), + RPAD('B', 255, 'B'), + RPAD('C', 255, 'C'), + '3072 bytes of total size of table key as per the mysql limitation' +); + +CREATE TABLE "LargeCell" ( + "id" NUMBER PRIMARY KEY, + "max_string_col_to_bytes" BLOB, + "max_string_col_to_str" CLOB +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json new file mode 100644 index 0000000000..d613fee945 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT/session.json @@ -0,0 +1,11391 @@ +{ + "SpSchema": { + "t1": { + "Name": "LargeKey", + "ColIds": [ + "c2", + "c3", + "c4", + "c5", + "c6", + "c7", + "c8" + ], + "ShardIdColumn": "", + "ColDefs": { + "c2": { + "Name": "pk_col1", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col1 varchar(255)", + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c3": { + "Name": "pk_col2", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col2 varchar(255)", + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c4": { + "Name": "pk_col3", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: pk_col3 varchar(255)", + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c5": { + "Name": "col1", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col1 varchar(255)", + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c6": { + "Name": "col2", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col2 varchar(255)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c7": { + "Name": "col3", + "T": { + "Name": "STRING", + "Len": 255, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col3 varchar(255)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c8": { + "Name": "value_col", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: value_col text", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c3", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c4", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "Indexes": [ + { + "Name": "large_index", + "TableId": "t1", + "Unique": false, + "Keys": [ + { + "ColId": "c5", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 3 + } + ], + "Id": "i9", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table LargeKey", + "Id": "t1" + }, + "t10": { + "Name": "LargeCell", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ShardIdColumn": "", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c12": { + "Name": "max_string_col_to_bytes", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: max_string_col_to_bytes longtext", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c13": { + "Name": "max_string_col_to_str", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: max_string_col_to_str mediumtext", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table LargeCell", + "Id": "t10" + }, + "t14": { + "Name": "WideRow", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53", + "c54", + "c55", + "c56", + "c57", + "c58", + "c59", + "c60", + "c61", + "c62", + "c63", + "c64", + "c65", + "c66", + "c67", + "c68", + "c69", + "c70", + "c71", + "c72", + "c73", + "c74", + "c75", + "c76", + "c77", + "c78", + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107", + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135", + "c136", + "c137", + "c138", + "c139", + "c140", + "c141", + "c142", + "c143", + "c144", + "c145", + "c146", + "c147", + "c148", + "c149", + "c150", + "c151", + "c152", + "c153", + "c154", + "c155", + "c156", + "c157", + "c158", + "c159", + "c160", + "c161", + "c162", + "c163", + "c164", + "c165", + "c166", + "c167", + "c168", + "c169", + "c170", + "c171", + "c172", + "c173", + "c174", + "c175" + ], + "ShardIdColumn": "", + "ColDefs": { + "c100": { + "Name": "col85", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col85 mediumtext", + "Id": "c100", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c101": { + "Name": "col86", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col86 mediumtext", + "Id": "c101", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c102": { + "Name": "col87", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col87 mediumtext", + "Id": "c102", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c103": { + "Name": "col88", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col88 mediumtext", + "Id": "c103", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c104": { + "Name": "col89", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col89 mediumtext", + "Id": "c104", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c105": { + "Name": "col90", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col90 mediumtext", + "Id": "c105", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c106": { + "Name": "col91", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col91 mediumtext", + "Id": "c106", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c107": { + "Name": "col92", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col92 mediumtext", + "Id": "c107", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c108": { + "Name": "col93", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col93 mediumtext", + "Id": "c108", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c109": { + "Name": "col94", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col94 mediumtext", + "Id": "c109", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c110": { + "Name": "col95", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col95 mediumtext", + "Id": "c110", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c111": { + "Name": "col96", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col96 mediumtext", + "Id": "c111", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c112": { + "Name": "col97", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col97 mediumtext", + "Id": "c112", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c113": { + "Name": "col98", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col98 mediumtext", + "Id": "c113", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c114": { + "Name": "col99", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col99 mediumtext", + "Id": "c114", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c115": { + "Name": "col100", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col100 mediumtext", + "Id": "c115", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c116": { + "Name": "col101", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col101 mediumtext", + "Id": "c116", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c117": { + "Name": "col102", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col102 mediumtext", + "Id": "c117", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c118": { + "Name": "col103", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col103 mediumtext", + "Id": "c118", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c119": { + "Name": "col104", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col104 mediumtext", + "Id": "c119", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c120": { + "Name": "col105", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col105 mediumtext", + "Id": "c120", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c121": { + "Name": "col106", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col106 mediumtext", + "Id": "c121", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c122": { + "Name": "col107", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col107 mediumtext", + "Id": "c122", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c123": { + "Name": "col108", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col108 mediumtext", + "Id": "c123", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c124": { + "Name": "col109", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col109 mediumtext", + "Id": "c124", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c125": { + "Name": "col110", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col110 mediumtext", + "Id": "c125", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c126": { + "Name": "col111", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col111 mediumtext", + "Id": "c126", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c127": { + "Name": "col112", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col112 mediumtext", + "Id": "c127", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c128": { + "Name": "col113", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col113 mediumtext", + "Id": "c128", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c129": { + "Name": "col114", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col114 mediumtext", + "Id": "c129", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c130": { + "Name": "col115", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col115 mediumtext", + "Id": "c130", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c131": { + "Name": "col116", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col116 mediumtext", + "Id": "c131", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c132": { + "Name": "col117", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col117 mediumtext", + "Id": "c132", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c133": { + "Name": "col118", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col118 mediumtext", + "Id": "c133", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c134": { + "Name": "col119", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col119 mediumtext", + "Id": "c134", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c135": { + "Name": "col120", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col120 mediumtext", + "Id": "c135", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c136": { + "Name": "col121", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col121 mediumtext", + "Id": "c136", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c137": { + "Name": "col122", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col122 mediumtext", + "Id": "c137", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c138": { + "Name": "col123", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col123 mediumtext", + "Id": "c138", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c139": { + "Name": "col124", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col124 mediumtext", + "Id": "c139", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c140": { + "Name": "col125", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col125 mediumtext", + "Id": "c140", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c141": { + "Name": "col126", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col126 mediumtext", + "Id": "c141", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c142": { + "Name": "col127", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col127 mediumtext", + "Id": "c142", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c143": { + "Name": "col128", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col128 mediumtext", + "Id": "c143", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c144": { + "Name": "col129", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col129 mediumtext", + "Id": "c144", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c145": { + "Name": "col130", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col130 mediumtext", + "Id": "c145", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c146": { + "Name": "col131", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col131 mediumtext", + "Id": "c146", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c147": { + "Name": "col132", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col132 mediumtext", + "Id": "c147", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c148": { + "Name": "col133", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col133 mediumtext", + "Id": "c148", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c149": { + "Name": "col134", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col134 mediumtext", + "Id": "c149", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c15": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c150": { + "Name": "col135", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col135 mediumtext", + "Id": "c150", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c151": { + "Name": "col136", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col136 mediumtext", + "Id": "c151", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c152": { + "Name": "col137", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col137 mediumtext", + "Id": "c152", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c153": { + "Name": "col138", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col138 mediumtext", + "Id": "c153", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c154": { + "Name": "col139", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col139 mediumtext", + "Id": "c154", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c155": { + "Name": "col140", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col140 mediumtext", + "Id": "c155", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c156": { + "Name": "col141", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col141 mediumtext", + "Id": "c156", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c157": { + "Name": "col142", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col142 mediumtext", + "Id": "c157", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c158": { + "Name": "col143", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col143 mediumtext", + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c159": { + "Name": "col144", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col144 mediumtext", + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c16": { + "Name": "col1", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col1 mediumtext", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c160": { + "Name": "col145", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col145 mediumtext", + "Id": "c160", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c161": { + "Name": "col146", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col146 mediumtext", + "Id": "c161", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c162": { + "Name": "col147", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col147 mediumtext", + "Id": "c162", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c163": { + "Name": "col148", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col148 mediumtext", + "Id": "c163", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c164": { + "Name": "col149", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col149 mediumtext", + "Id": "c164", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c165": { + "Name": "col150", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col150 mediumtext", + "Id": "c165", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c166": { + "Name": "col151", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col151 mediumtext", + "Id": "c166", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c167": { + "Name": "col152", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col152 mediumtext", + "Id": "c167", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c168": { + "Name": "col153", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col153 mediumtext", + "Id": "c168", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c169": { + "Name": "col154", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col154 mediumtext", + "Id": "c169", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c17": { + "Name": "col2", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col2 mediumtext", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c170": { + "Name": "col155", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col155 mediumtext", + "Id": "c170", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c171": { + "Name": "col156", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col156 mediumtext", + "Id": "c171", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c172": { + "Name": "col157", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col157 mediumtext", + "Id": "c172", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c173": { + "Name": "col158", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col158 mediumtext", + "Id": "c173", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c174": { + "Name": "col159", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col159 mediumtext", + "Id": "c174", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c175": { + "Name": "col160", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col160 mediumtext", + "Id": "c175", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c18": { + "Name": "col3", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col3 mediumtext", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c19": { + "Name": "col4", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col4 mediumtext", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c20": { + "Name": "col5", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col5 mediumtext", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c21": { + "Name": "col6", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col6 mediumtext", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c22": { + "Name": "col7", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col7 mediumtext", + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c23": { + "Name": "col8", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col8 mediumtext", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c24": { + "Name": "col9", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col9 mediumtext", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c25": { + "Name": "col10", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col10 mediumtext", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c26": { + "Name": "col11", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col11 mediumtext", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c27": { + "Name": "col12", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col12 mediumtext", + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c28": { + "Name": "col13", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col13 mediumtext", + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c29": { + "Name": "col14", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col14 mediumtext", + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c30": { + "Name": "col15", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col15 mediumtext", + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c31": { + "Name": "col16", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col16 mediumtext", + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c32": { + "Name": "col17", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col17 mediumtext", + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c33": { + "Name": "col18", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col18 mediumtext", + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c34": { + "Name": "col19", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col19 mediumtext", + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c35": { + "Name": "col20", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col20 mediumtext", + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c36": { + "Name": "col21", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col21 mediumtext", + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c37": { + "Name": "col22", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col22 mediumtext", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c38": { + "Name": "col23", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col23 mediumtext", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c39": { + "Name": "col24", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col24 mediumtext", + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c40": { + "Name": "col25", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col25 mediumtext", + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c41": { + "Name": "col26", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col26 mediumtext", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c42": { + "Name": "col27", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col27 mediumtext", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c43": { + "Name": "col28", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col28 mediumtext", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c44": { + "Name": "col29", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col29 mediumtext", + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c45": { + "Name": "col30", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col30 mediumtext", + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c46": { + "Name": "col31", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col31 mediumtext", + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c47": { + "Name": "col32", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col32 mediumtext", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c48": { + "Name": "col33", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col33 mediumtext", + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c49": { + "Name": "col34", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col34 mediumtext", + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c50": { + "Name": "col35", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col35 mediumtext", + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c51": { + "Name": "col36", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col36 mediumtext", + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c52": { + "Name": "col37", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col37 mediumtext", + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c53": { + "Name": "col38", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col38 mediumtext", + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c54": { + "Name": "col39", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col39 mediumtext", + "Id": "c54", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c55": { + "Name": "col40", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col40 mediumtext", + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c56": { + "Name": "col41", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col41 mediumtext", + "Id": "c56", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c57": { + "Name": "col42", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col42 mediumtext", + "Id": "c57", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c58": { + "Name": "col43", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col43 mediumtext", + "Id": "c58", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c59": { + "Name": "col44", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col44 mediumtext", + "Id": "c59", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c60": { + "Name": "col45", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col45 mediumtext", + "Id": "c60", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c61": { + "Name": "col46", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col46 mediumtext", + "Id": "c61", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c62": { + "Name": "col47", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col47 mediumtext", + "Id": "c62", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c63": { + "Name": "col48", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col48 mediumtext", + "Id": "c63", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c64": { + "Name": "col49", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col49 mediumtext", + "Id": "c64", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c65": { + "Name": "col50", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col50 mediumtext", + "Id": "c65", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c66": { + "Name": "col51", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col51 mediumtext", + "Id": "c66", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c67": { + "Name": "col52", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col52 mediumtext", + "Id": "c67", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c68": { + "Name": "col53", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col53 mediumtext", + "Id": "c68", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c69": { + "Name": "col54", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col54 mediumtext", + "Id": "c69", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c70": { + "Name": "col55", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col55 mediumtext", + "Id": "c70", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c71": { + "Name": "col56", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col56 mediumtext", + "Id": "c71", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c72": { + "Name": "col57", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col57 mediumtext", + "Id": "c72", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c73": { + "Name": "col58", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col58 mediumtext", + "Id": "c73", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c74": { + "Name": "col59", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col59 mediumtext", + "Id": "c74", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c75": { + "Name": "col60", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col60 mediumtext", + "Id": "c75", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c76": { + "Name": "col61", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col61 mediumtext", + "Id": "c76", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c77": { + "Name": "col62", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col62 mediumtext", + "Id": "c77", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c78": { + "Name": "col63", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col63 mediumtext", + "Id": "c78", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c79": { + "Name": "col64", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col64 mediumtext", + "Id": "c79", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c80": { + "Name": "col65", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col65 mediumtext", + "Id": "c80", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c81": { + "Name": "col66", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col66 mediumtext", + "Id": "c81", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c82": { + "Name": "col67", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col67 mediumtext", + "Id": "c82", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c83": { + "Name": "col68", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col68 mediumtext", + "Id": "c83", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c84": { + "Name": "col69", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col69 mediumtext", + "Id": "c84", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c85": { + "Name": "col70", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col70 mediumtext", + "Id": "c85", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c86": { + "Name": "col71", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col71 mediumtext", + "Id": "c86", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c87": { + "Name": "col72", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col72 mediumtext", + "Id": "c87", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c88": { + "Name": "col73", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col73 mediumtext", + "Id": "c88", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c89": { + "Name": "col74", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col74 mediumtext", + "Id": "c89", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c90": { + "Name": "col75", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col75 mediumtext", + "Id": "c90", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c91": { + "Name": "col76", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col76 mediumtext", + "Id": "c91", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c92": { + "Name": "col77", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col77 mediumtext", + "Id": "c92", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c93": { + "Name": "col78", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col78 mediumtext", + "Id": "c93", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c94": { + "Name": "col79", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col79 mediumtext", + "Id": "c94", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c95": { + "Name": "col80", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col80 mediumtext", + "Id": "c95", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c96": { + "Name": "col81", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col81 mediumtext", + "Id": "c96", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c97": { + "Name": "col82", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col82 mediumtext", + "Id": "c97", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c98": { + "Name": "col83", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col83 mediumtext", + "Id": "c98", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + }, + "c99": { + "Name": "col84", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: col84 mediumtext", + "Id": "c99", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + }, + "Opts": null + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "", + "InterleaveType": "" + }, + "CheckConstraints": null, + "Comment": "Spanner schema for source table WideRow", + "Id": "t14" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "LargeKey", + "Schema": "", + "ColIds": [ + "c2", + "c3", + "c4", + "c5", + "c6", + "c7", + "c8" + ], + "ColDefs": { + "c2": { + "Name": "pk_col1", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c2", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c3": { + "Name": "pk_col2", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c3", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c4": { + "Name": "pk_col3", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c4", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c5": { + "Name": "col1", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c5", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c6": { + "Name": "col2", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c7": { + "Name": "col3", + "Type": { + "Name": "varchar", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c8": { + "Name": "value_col", + "Type": { + "Name": "text", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c2", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c3", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c4", + "Desc": false, + "Order": 3 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": [ + { + "Name": "large_index", + "Unique": false, + "Keys": [ + { + "ColId": "c5", + "Desc": false, + "Order": 1 + }, + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 3 + } + ], + "Id": "i9", + "StoredColumnIds": null + } + ], + "Id": "t1" + }, + "t10": { + "Name": "LargeCell", + "Schema": "", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c12": { + "Name": "max_string_col_to_bytes", + "Type": { + "Name": "longtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c13": { + "Name": "max_string_col_to_str", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t10" + }, + "t14": { + "Name": "WideRow", + "Schema": "", + "ColIds": [ + "c15", + "c16", + "c17", + "c18", + "c19", + "c20", + "c21", + "c22", + "c23", + "c24", + "c25", + "c26", + "c27", + "c28", + "c29", + "c30", + "c31", + "c32", + "c33", + "c34", + "c35", + "c36", + "c37", + "c38", + "c39", + "c40", + "c41", + "c42", + "c43", + "c44", + "c45", + "c46", + "c47", + "c48", + "c49", + "c50", + "c51", + "c52", + "c53", + "c54", + "c55", + "c56", + "c57", + "c58", + "c59", + "c60", + "c61", + "c62", + "c63", + "c64", + "c65", + "c66", + "c67", + "c68", + "c69", + "c70", + "c71", + "c72", + "c73", + "c74", + "c75", + "c76", + "c77", + "c78", + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107", + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135", + "c136", + "c137", + "c138", + "c139", + "c140", + "c141", + "c142", + "c143", + "c144", + "c145", + "c146", + "c147", + "c148", + "c149", + "c150", + "c151", + "c152", + "c153", + "c154", + "c155", + "c156", + "c157", + "c158", + "c159", + "c160", + "c161", + "c162", + "c163", + "c164", + "c165", + "c166", + "c167", + "c168", + "c169", + "c170", + "c171", + "c172", + "c173", + "c174", + "c175" + ], + "ColDefs": { + "c100": { + "Name": "col85", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c100", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c101": { + "Name": "col86", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c101", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c102": { + "Name": "col87", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c102", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c103": { + "Name": "col88", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c103", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c104": { + "Name": "col89", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c104", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c105": { + "Name": "col90", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c105", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c106": { + "Name": "col91", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c106", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c107": { + "Name": "col92", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c107", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c108": { + "Name": "col93", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c108", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c109": { + "Name": "col94", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c109", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c110": { + "Name": "col95", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c110", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c111": { + "Name": "col96", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c111", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c112": { + "Name": "col97", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c112", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c113": { + "Name": "col98", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c113", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c114": { + "Name": "col99", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c114", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c115": { + "Name": "col100", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c115", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c116": { + "Name": "col101", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c116", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c117": { + "Name": "col102", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c117", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c118": { + "Name": "col103", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c118", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c119": { + "Name": "col104", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c119", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c120": { + "Name": "col105", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c120", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c121": { + "Name": "col106", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c121", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c122": { + "Name": "col107", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c122", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c123": { + "Name": "col108", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c123", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c124": { + "Name": "col109", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c124", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c125": { + "Name": "col110", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c125", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c126": { + "Name": "col111", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c126", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c127": { + "Name": "col112", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c127", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c128": { + "Name": "col113", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c128", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c129": { + "Name": "col114", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c129", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c130": { + "Name": "col115", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c130", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c131": { + "Name": "col116", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c131", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c132": { + "Name": "col117", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c132", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c133": { + "Name": "col118", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c133", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c134": { + "Name": "col119", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c134", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c135": { + "Name": "col120", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c135", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c136": { + "Name": "col121", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c136", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c137": { + "Name": "col122", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c137", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c138": { + "Name": "col123", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c138", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c139": { + "Name": "col124", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c139", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c140": { + "Name": "col125", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c140", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c141": { + "Name": "col126", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c141", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c142": { + "Name": "col127", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c142", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c143": { + "Name": "col128", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c143", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c144": { + "Name": "col129", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c144", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c145": { + "Name": "col130", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c145", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c146": { + "Name": "col131", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c146", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c147": { + "Name": "col132", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c147", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c148": { + "Name": "col133", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c148", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c149": { + "Name": "col134", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c149", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c15": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 11 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c150": { + "Name": "col135", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c150", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c151": { + "Name": "col136", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c151", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c152": { + "Name": "col137", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c152", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c153": { + "Name": "col138", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c153", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c154": { + "Name": "col139", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c154", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c155": { + "Name": "col140", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c155", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c156": { + "Name": "col141", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c156", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c157": { + "Name": "col142", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c157", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c158": { + "Name": "col143", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c158", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c159": { + "Name": "col144", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c159", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c16": { + "Name": "col1", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c160": { + "Name": "col145", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c160", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c161": { + "Name": "col146", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c161", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c162": { + "Name": "col147", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c162", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c163": { + "Name": "col148", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c163", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c164": { + "Name": "col149", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c164", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c165": { + "Name": "col150", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c165", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c166": { + "Name": "col151", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c166", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c167": { + "Name": "col152", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c167", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c168": { + "Name": "col153", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c168", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c169": { + "Name": "col154", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c169", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c17": { + "Name": "col2", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c170": { + "Name": "col155", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c170", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c171": { + "Name": "col156", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c171", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c172": { + "Name": "col157", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c172", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c173": { + "Name": "col158", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c173", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c174": { + "Name": "col159", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c174", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c175": { + "Name": "col160", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c175", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c18": { + "Name": "col3", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c19": { + "Name": "col4", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c20": { + "Name": "col5", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c21": { + "Name": "col6", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c22": { + "Name": "col7", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c22", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c23": { + "Name": "col8", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c24": { + "Name": "col9", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c25": { + "Name": "col10", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c26": { + "Name": "col11", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c27": { + "Name": "col12", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c27", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c28": { + "Name": "col13", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c28", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c29": { + "Name": "col14", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c29", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c30": { + "Name": "col15", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c30", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c31": { + "Name": "col16", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c31", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c32": { + "Name": "col17", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c32", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c33": { + "Name": "col18", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c33", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c34": { + "Name": "col19", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c34", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c35": { + "Name": "col20", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c35", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c36": { + "Name": "col21", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c36", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c37": { + "Name": "col22", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c38": { + "Name": "col23", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c39": { + "Name": "col24", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c40": { + "Name": "col25", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c40", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c41": { + "Name": "col26", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c42": { + "Name": "col27", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c43": { + "Name": "col28", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c44": { + "Name": "col29", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c45": { + "Name": "col30", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c46": { + "Name": "col31", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c46", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c47": { + "Name": "col32", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c48": { + "Name": "col33", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c48", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c49": { + "Name": "col34", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c49", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c50": { + "Name": "col35", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c50", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c51": { + "Name": "col36", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c51", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c52": { + "Name": "col37", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c52", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c53": { + "Name": "col38", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c53", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c54": { + "Name": "col39", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c54", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c55": { + "Name": "col40", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c55", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c56": { + "Name": "col41", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c56", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c57": { + "Name": "col42", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c57", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c58": { + "Name": "col43", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c58", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c59": { + "Name": "col44", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c59", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c60": { + "Name": "col45", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c60", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c61": { + "Name": "col46", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c61", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c62": { + "Name": "col47", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c62", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c63": { + "Name": "col48", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c63", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c64": { + "Name": "col49", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c64", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c65": { + "Name": "col50", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c65", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c66": { + "Name": "col51", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c66", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c67": { + "Name": "col52", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c67", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c68": { + "Name": "col53", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c68", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c69": { + "Name": "col54", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c69", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c70": { + "Name": "col55", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c70", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c71": { + "Name": "col56", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c71", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c72": { + "Name": "col57", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c72", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c73": { + "Name": "col58", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c73", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c74": { + "Name": "col59", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c74", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c75": { + "Name": "col60", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c75", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c76": { + "Name": "col61", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c76", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c77": { + "Name": "col62", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c77", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c78": { + "Name": "col63", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c78", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c79": { + "Name": "col64", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c79", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c80": { + "Name": "col65", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c80", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c81": { + "Name": "col66", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c81", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c82": { + "Name": "col67", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c82", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c83": { + "Name": "col68", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c83", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c84": { + "Name": "col69", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c84", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c85": { + "Name": "col70", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c85", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c86": { + "Name": "col71", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c86", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c87": { + "Name": "col72", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c87", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c88": { + "Name": "col73", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c88", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c89": { + "Name": "col74", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c89", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c90": { + "Name": "col75", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c90", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c91": { + "Name": "col76", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c91", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c92": { + "Name": "col77", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c92", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c93": { + "Name": "col78", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c93", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c94": { + "Name": "col79", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c94", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c95": { + "Name": "col80", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c95", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c96": { + "Name": "col81", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c96", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c97": { + "Name": "col82", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c97", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c98": { + "Name": "col83", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c98", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + }, + "c99": { + "Name": "col84", + "Type": { + "Name": "mediumtext", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c99", + "AutoGen": { + "Name": "", + "GenerationType": "", + "IdentityOptions": { + "SkipRangeMin": "", + "SkipRangeMax": "", + "StartCounterWith": "" + } + }, + "DefaultValue": { + "IsPresent": false, + "Value": { + "ExpressionId": "", + "Statement": "" + } + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "CheckConstraints": null, + "Indexes": null, + "Id": "t14" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + }, + "t10": { + "ColumnLevelIssues": { + "c11": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t14": { + "ColumnLevelIssues": { + "c15": [ + 14 + ] + }, + "TableLevelIssues": null + } + }, + "InvalidCheckExp": null, + "ToSpanner": { + "LargeKey": { + "Name": "LargeKey", + "Cols": { + "col1": "col1", + "col2": "col2", + "col3": "col3", + "pk_col1": "pk_col1", + "pk_col2": "pk_col2", + "pk_col3": "pk_col3", + "value_col": "value_col" + } + }, + "LargeCell": { + "Name": "LargeCell", + "Cols": { + "id": "id", + "max_string_col_to_bytes": "max_string_col_to_bytes", + "max_string_col_to_str": "max_string_col_to_str" + } + }, + "WideRow": { + "Name": "WideRow", + "Cols": { + "col1": "col1", + "col10": "col10", + "col100": "col100", + "col101": "col101", + "col102": "col102", + "col103": "col103", + "col104": "col104", + "col105": "col105", + "col106": "col106", + "col107": "col107", + "col108": "col108", + "col109": "col109", + "col11": "col11", + "col110": "col110", + "col111": "col111", + "col112": "col112", + "col113": "col113", + "col114": "col114", + "col115": "col115", + "col116": "col116", + "col117": "col117", + "col118": "col118", + "col119": "col119", + "col12": "col12", + "col120": "col120", + "col121": "col121", + "col122": "col122", + "col123": "col123", + "col124": "col124", + "col125": "col125", + "col126": "col126", + "col127": "col127", + "col128": "col128", + "col129": "col129", + "col13": "col13", + "col130": "col130", + "col131": "col131", + "col132": "col132", + "col133": "col133", + "col134": "col134", + "col135": "col135", + "col136": "col136", + "col137": "col137", + "col138": "col138", + "col139": "col139", + "col14": "col14", + "col140": "col140", + "col141": "col141", + "col142": "col142", + "col143": "col143", + "col144": "col144", + "col145": "col145", + "col146": "col146", + "col147": "col147", + "col148": "col148", + "col149": "col149", + "col15": "col15", + "col150": "col150", + "col151": "col151", + "col152": "col152", + "col153": "col153", + "col154": "col154", + "col155": "col155", + "col156": "col156", + "col157": "col157", + "col158": "col158", + "col159": "col159", + "col16": "col16", + "col160": "col160", + "col17": "col17", + "col18": "col18", + "col19": "col19", + "col2": "col2", + "col20": "col20", + "col21": "col21", + "col22": "col22", + "col23": "col23", + "col24": "col24", + "col25": "col25", + "col26": "col26", + "col27": "col27", + "col28": "col28", + "col29": "col29", + "col3": "col3", + "col30": "col30", + "col31": "col31", + "col32": "col32", + "col33": "col33", + "col34": "col34", + "col35": "col35", + "col36": "col36", + "col37": "col37", + "col38": "col38", + "col39": "col39", + "col4": "col4", + "col40": "col40", + "col41": "col41", + "col42": "col42", + "col43": "col43", + "col44": "col44", + "col45": "col45", + "col46": "col46", + "col47": "col47", + "col48": "col48", + "col49": "col49", + "col5": "col5", + "col50": "col50", + "col51": "col51", + "col52": "col52", + "col53": "col53", + "col54": "col54", + "col55": "col55", + "col56": "col56", + "col57": "col57", + "col58": "col58", + "col59": "col59", + "col6": "col6", + "col60": "col60", + "col61": "col61", + "col62": "col62", + "col63": "col63", + "col64": "col64", + "col65": "col65", + "col66": "col66", + "col67": "col67", + "col68": "col68", + "col69": "col69", + "col7": "col7", + "col70": "col70", + "col71": "col71", + "col72": "col72", + "col73": "col73", + "col74": "col74", + "col75": "col75", + "col76": "col76", + "col77": "col77", + "col78": "col78", + "col79": "col79", + "col8": "col8", + "col80": "col80", + "col81": "col81", + "col82": "col82", + "col83": "col83", + "col84": "col84", + "col85": "col85", + "col86": "col86", + "col87": "col87", + "col88": "col88", + "col89": "col89", + "col9": "col9", + "col90": "col90", + "col91": "col91", + "col92": "col92", + "col93": "col93", + "col94": "col94", + "col95": "col95", + "col96": "col96", + "col97": "col97", + "col98": "col98", + "col99": "col99", + "id": "id" + } + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {}, + "Source": "mysqldump" +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..6b6168912a --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,131 @@ +CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsWithSizes ( + varchar_column STRING(30) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(50), + bool_column BOOL, + binary_column BYTES(30), + varbinary_column BYTES(30), + bit_column BYTES(10), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS DatatypeColumnsReducedSizes ( + varchar_column STRING(10) NOT NULL, + float_column FLOAT32, + decimal_column NUMERIC, + char_column STRING(20), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE IF NOT EXISTS Users ( + user_id INT64 NOT NULL, + first_name STRING(50), + last_name STRING(50), + age INT64, + full_name STRING(100) AS (ARRAY_TO_STRING([first_name, last_name], " ")) STORED, +) PRIMARY KEY (user_id); + +CREATE TABLE IF NOT EXISTS Authors ( + id INT64 NOT NULL, + name STRING(200), +) PRIMARY KEY (id); + +CREATE TABLE IF NOT EXISTS AllDatatypeTransformation ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + int_column INT64, + bigint_column INT64, + float_column FLOAT32, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + blob_column BYTES(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY (varchar_column); + +CREATE SEQUENCE sequence_id OPTIONS (sequence_kind='bit_reversed_positive', skip_range_min = 0, skip_range_max = 3); + +CREATE TABLE IF NOT EXISTS Singers ( + singer_id INT64 NOT NULL DEFAULT (GET_NEXT_SEQUENCE_VALUE(SEQUENCE sequence_id)) , + first_name STRING(1024), + last_name STRING(1024), +) PRIMARY KEY (singer_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE), + title STRING(200), +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql new file mode 100644 index 0000000000..fc1f4a088c --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql @@ -0,0 +1,4 @@ +INSERT INTO "AllDatatypeColumns" +("varchar_column", "tinyint_column", "text_column", "date_column", "smallint_column", "mediumint_column", "int_column", "bigint_column", "float_column", "double_column", "decimal_column", "datetime_column", "timestamp_column", "time_column", "year_column", "char_column", "tinyblob_column", "tinytext_column", "blob_column", "mediumblob_column", "mediumtext_column", "longblob_column", "longtext_column", "enum_column", "bool_column", "other_bool_column", "binary_column", "varbinary_column", "bit_column") +VALUES +('value1', 10, 'text_content', TO_DATE('2024-02-08', 'YYYY-MM-DD'), 50, 1000, 50000, 987654321, 45.67, 123.789, 456.12, TO_TIMESTAMP('2024-02-08 08:15:30', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP_TZ('2024-02-08 08:15:30 UTC', 'YYYY-MM-DD HH24:MI:SS TZR'), '08:15:30', 2022, 'char_1 ', HEXTORAW('74696e79626c6f625f646174615f31'), 'tinytext', HEXTORAW('626c6f625f646174615f31'), HEXTORAW('6d656469756d626c6f625f646174615f31'), 'mediumtext', HEXTORAW('6c6f6e67626c6f625f646174615f31'), 'longtext_content', '2', 0, 1, HEXTORAW('62696e6172795f31000000000000000000000000'), HEXTORAW('76617262696e6172795f646174615f31'), HEXTORAW('0102')); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql new file mode 100644 index 0000000000..830bc4ba29 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql @@ -0,0 +1,156 @@ + +CREATE TABLE "AllDatatypeColumns" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "other_bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "AllDatatypeColumns2" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "smallint_column" NUMBER, + "mediumint_column" NUMBER, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "char_column" CHAR(10), + "tinyblob_column" RAW(255), + "tinytext_column" VARCHAR2(255), + "blob_column" BLOB, + "mediumblob_column" BLOB, + "mediumtext_column" CLOB, + "longblob_column" BLOB, + "longtext_column" CLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsWithSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "DatatypeColumnsReducedSizes" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "float_column" FLOAT, + "decimal_column" NUMBER(10,2), + "char_column" CHAR(10), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "varbinary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Users" ( + "user_id" NUMBER NOT NULL, + "first_name" VARCHAR2(50), + "last_name" VARCHAR2(50), + "age" NUMBER, + PRIMARY KEY ("user_id") +); + +CREATE TABLE "Authors" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +CREATE TABLE "AllDatatypeTransformation" ( + "varchar_column" VARCHAR2(20) NOT NULL, + "tinyint_column" NUMBER, + "text_column" CLOB, + "date_column" DATE, + "int_column" NUMBER, + "bigint_column" NUMBER, + "float_column" FLOAT, + "double_column" DOUBLE PRECISION, + "decimal_column" NUMBER(10,2), + "datetime_column" TIMESTAMP WITH TIME ZONE, + "timestamp_column" TIMESTAMP WITH TIME ZONE, + "time_column" VARCHAR2(50), + "year_column" NUMBER, + "blob_column" BLOB, + "enum_column" VARCHAR2(255), + "bool_column" NUMBER(1), + "binary_column" RAW(20), + "bit_column" RAW(8), + PRIMARY KEY ("varchar_column") +); + +CREATE TABLE "Singers" ( + "singer_id" NUMBER NOT NULL , + "first_name" VARCHAR2(1024), + "last_name" VARCHAR2(1024), + PRIMARY KEY ("singer_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL , + "title" VARCHAR2(200), + PRIMARY KEY ("id") +) ; + +ALTER TABLE "AllDatatypeColumns" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeColumns2" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsWithSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "DatatypeColumnsReducedSizes" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "AllDatatypeTransformation" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Singers" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-Articles.avro new file mode 100644 index 0000000000000000000000000000000000000000..01421913ae369ac9a7a5b0b9f603a78d42f76ae5 GIT binary patch literal 2484 zcmdUvO^ee&7{|L|sVjI8l?88-bF)qQw#nW+2zu}Z-P=;aWM;CRb&`oQ6QL}HqA0j$ z4}Jr|gXm}Q=EvBR_yQ{IMLamOO{dwmTPt|b913av&piM6J^z{8(cX%+4jyv0Az%+D z;NarEE&Ca^w``6fz?wTyx!%!M`_9^>F zuI7tBSFSke2qtryjCX7kR$c~-nlqz4v z1iMq>j0qwMOO10=P;i?h7xqB9t8%6zIp%NFbCxF}M3`cUCjl|?h%t&ms^)THG8yPo z1}1tl-sy=LBhL~*Ct3LF$; z-)UjwI{}O%r;VHL@#DGfhu2TueS&Wehw68|bmLt~`r-j~Ac9UrQWdrrD;a0{su1;d ztKIVapzQ~4yG|JdI#!~d3kTvVGfDKi_|yVtT_eLhn;Kfa+?sb{ScRI%pj ztaCSR-Me;o+p1WWb*ZRPi&(?qvqy(VPpg)-3@z_JrgiXbIa*3rqP1G0)%Y`7UXiWg z_kTBABX4uE_3>yqT1r=ehEYY#&lop8f%tQ zr6Dh!(v{=F=v#=35hK74rad^lpEJ&xLX%oXB`Qhi|II0qV@kB^4*g>vIW-{GZr21- z$m2_PYp6`A$0u~WuBH-FTa6_@1?;ealwf0-rn&WF-mxWVfDW$#W}1f-G^{3DwtPmo z7O?8OhJ;g>YF@r(lP#5iTB|7~%R!De+oP*dD`Lhb&RDIHLsV3po4#fa4wuBGP)svX zxXxX`jN4*z$q^JIGiP=r=lq@en)FP?Ddkkt#eg_@A_S*U7+R-%Z@CB zt4_e>>5OZ~(3NF{M#C0LX$BS7Xy3A_0l6}zE@-~BN{ve3jToADSsdlA&LJKW{{;KKu1fPvUnU7AWDrFZIzgCV(_Al$x>?WiWwTljA7=}|TDiK16E{GA`#1J=eY`0E@7?6-)0R-JjWS!6F_S&=0+?|o4sspMj zA;iFh*qB%=#4jjfLHr8Dgv7wk0(VV(F=-k+FwjVelKB1Zd+&L_yL-|0f^``_;cQu; zEu6ylOS?c0a}3r1$B6M5v_S?_{hK=BO~!VF`X?cn#saKu?3NFUf@tZuR7g&;O|aKC z&pC#1C`pQiglTS+rWs{Q-v*Rz5(Me`$br$@DWQ}Iuvb}8u!1A}Z`WkeHb&C81E=gG z#d3?y-L5LcFb$8%&7m@49`4}byqxhOqqu< zl&IFN!WyZNF7;~FUF4}_GrkXQ7k|lqmp0PX;VT>u3crqYH9x+BS%v3IaB$LA- zWpJW5@lGc)5uQw0Wt$a&08FNB3U06eVSQtfuI*Z3@Z zkAB?v@aEw8wJUEwUEgngS-KtdqGZMO`gWY)!1iz)*nN~lwvXMeemUy>?eO5^cXagb zsrtNB-a=Q?p?FMf6eBw#nHqrC3pr={_==A2`JNN>{h;6VJCq@)qZXY)*a&BmL+W%! z2UK*n1 zj0g}TNGwc@ObDs!0%C!MfxiGKf(bEHg1aWUxJ{FGU?D@4IPbgfeeb>dawS+EGf#pQ zMyEMkLJ`UEY^_@74OLPSz`J{;v3At5USnd6ordUC{rCcx^ zFIw%q{+%HZHUy3k7a&SBQa2-1;`0`vix>hj-(jHfc7zGR+-he|rk>koj1fs^4awAmfXx_MmTBCY$%7nsj4brj!#(=8D0!7M?9%H4X6tupJ zV3IrtfdGSt?KHfraHe3;cL4&T5;4?x)hYmdBqm`v0<5*a8G9pUN0YH>G;xN=f)%HP zNM2nB>zUvva6&m2n8up9F6ipkPcB>n@uJL`iezt2jo#&W%o`yhNT7Z|G(4b`AP`Gm zwmFl47NMYz*Y@soqz@c)B6I(AKO76Ip0`5O1St_#O^K3GVp}|QQE#T; zgO)%ltJ^Pgh@f;?d6LMnFjwl=JXcyIFTC6@GrnK4D_{oLei?dp(J$)P`s252 z-{8{^Yw~@nw<6C*Eq%(PML=G&J#WH)z=CZluuu&BJ0V;YHa)j{aM{<*!-PxO&JqX7^ zun_!FXlG;Ve_$tKDYiD&mfF||md;#~NiM&%urh&Ve|+D(H=pmDcL#NrytA+;#j3*F zG=?u{_I=$=slVY%iiIS8!A~Hz&pd2CP*f`Oro|{H%HP=9AAZa-W~GBrr6o(+{z1Wg zmlQ}-vzRIkaq5&V85c%h@VRI+41BX+fzt;uKHAsCG^mMrn@nPykg9l4t2G0~lV_@STw{o$x+Ij!>&3XC!)VbZ8 zpYtD&U$4L#L8x8{0AmtVs^wNtqY(TO2-~QgWFSopvK5!AP zayBB+G7jM0g*8joB5X}s93#ekD{X}^(BG+3nHQJ~^-Gmt>I-XfVXbj7j)|7mONHbl zT(Z{F<~qmFuSgPLAz=_1rD;Z)(x)xTmI#8hoH#IgJ0O%2VXZfvC|Jx9{w3GY5W}EyK&}gwgn8vYu664v7?)b3$tQpfHpn^DwM61T&!gd3lT<(# zI{;JWp${c&Kdg1&b;3;r?Q@sVqu3nPXx}s@-O7Wat0v+kfV|e*J(?a(O;l_uRnlZ+ z!3twam9P2)7hA*;6GRdg8t0~<&h0R{Xc>k}DrY*94gN;m;dmk{KBidW!+;pM#~8&h zRC75pnF!V>gNJ%E-qwj63fm{q^!%BAB$U=-+VXJ~##CC98cGw1Bk?pvje9!24lZ>- zk$^kCNW`8%oOqqM{H->k3fi&oyY!Th%OaHRb`9c#*4-8l;!G``Hi~M1i|eVg4_!R) zKHRxJv$r(8!kClI<;EupRnN!%c+SZ=o;#M!jw3fW01{)V zx*=s|MEiE~VE4u8)R}{Sfqv?Me&VMmEQUQ4^>EldI`Ck`T-&c;K4C^=#|or~Oksg3`2@wNwC aN(3&?&lRthOSXwuD>YwJyJx7i?z2Bsr)3cU literal 0 HcmV?d00001 diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql new file mode 100644 index 0000000000..76cdeafdbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql @@ -0,0 +1,52 @@ +CREATE TABLE "Movie" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "actor" NUMBER, + "startTime" TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + "subscribed" NUMBER(1), + "plan" CHAR(1), + "startDate" DATE, + PRIMARY KEY ("id") +); + +CREATE TABLE "Authors" ( + "author_id" NUMBER NOT NULL, + "name" VARCHAR2(200), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Articles" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "published_date" DATE, + "author_id" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Articles" ADD FOREIGN KEY ("author_id") REFERENCES "Authors"("author_id"); + +CREATE INDEX author_id ON "Articles" ("author_id"); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD FOREIGN KEY ("author_id") REFERENCES "Authors"("author_id"); + +CREATE INDEX author_id_6 ON "Books" ("author_id"); + +ALTER TABLE "Movie" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Articles" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..71ee384cbb --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..fd61478626 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500), + "last_name1" VARCHAR2(500), + "first_name2" VARCHAR2(500), + "last_name2" VARCHAR2(500), + "first_name3" VARCHAR2(500), + "last_name3" VARCHAR2(500), + "ID" NUMBER NOT NULL PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json new file mode 100644 index 0000000000..c0b5fc0009 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json @@ -0,0 +1,10 @@ +{ + "renamedTables":{ + "person1":"human1" + }, + "renamedColumns":{ + "person1":{ + "first_name1":"name1" + } + } +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..62f8aaa748 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS Authors ( + author_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY (author_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), +) PRIMARY KEY (synth_id); + +CREATE INDEX author_id ON Books (author_id); + +ALTER TABLE Books ADD CONSTRAINT Books_ibfk_1 FOREIGN KEY (author_id) REFERENCES Authors (author_id) ON DELETE NO ACTION; + +CREATE TABLE IF NOT EXISTS Genre ( + name STRING(200), + genre_id INT64 NOT NULL, +) PRIMARY KEY (genre_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql new file mode 100644 index 0000000000..f58350d4a9 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql @@ -0,0 +1,26 @@ +CREATE TABLE "Authors" ( + "author_id" NUMBER(10,0) NOT NULL, + "name" VARCHAR2(25), + PRIMARY KEY ("author_id") +); + +CREATE TABLE "Books" ( + "id" NUMBER(10,0) NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER(10,0), + PRIMARY KEY ("id") +); + +ALTER TABLE "Books" ADD CONSTRAINT "Books_ibfk_1" FOREIGN KEY ("author_id") REFERENCES "Authors" ("author_id"); + +CREATE TABLE "Genre" ( + "name" VARCHAR2(200), + "genre_id" NUMBER(10,0) NOT NULL, + PRIMARY KEY ("genre_id") +); + +ALTER TABLE "Authors" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Books" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER TABLE "Genre" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json new file mode 100644 index 0000000000..b7ff136a40 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-session.json @@ -0,0 +1,506 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "e2e-it", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t34": { + "Name": "Authors", + "ColIds": [ + "c37", + "c38" + ], + "ShardIdColumn": "", + "ColDefs": { + "c37": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: author_id int(10)", + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Authors", + "Id": "t34" + }, + "t35": { + "Name": "Books", + "ColIds": [ + "c41", + "c42", + "c43", + "c47" + ], + "ShardIdColumn": "", + "ColDefs": { + "c41": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(10)", + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c47": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c47", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c47", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "Id": "f40", + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION" + } + ], + "Indexes": [ + { + "Name": "author_id", + "TableId": "t35", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t35" + } + }, + "SyntheticPKeys": { + "t35": { + "ColId": "c47", + "Sequence": 0 + } + }, + "SrcSchema": { + "t34": { + "Name": "Authors", + "Schema": "e2e-it", + "ColIds": [ + "c37", + "c38", + "c39" + ], + "ColDefs": { + "c37": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c37", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c38": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c38", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c39": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c39", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c37", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t34" + }, + "t35": { + "Name": "Books", + "Schema": "e2e-it", + "ColIds": [ + "c41", + "c42", + "c43" + ], + "ColDefs": { + "c41": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c41", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c42": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c42", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c43": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c43", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": [ + { + "Name": "Books_ibfk_1", + "ColIds": [ + "c43" + ], + "ReferTableId": "t34", + "ReferColumnIds": [ + "c37" + ], + "OnDelete": "NO ACTION", + "OnUpdate": "NO ACTION", + "Id": "f40" + } + ], + "Indexes": [ + { + "Name": "author_id", + "Unique": false, + "Keys": [ + { + "ColId": "c43", + "Desc": false, + "Order": 1 + } + ], + "Id": "i46", + "StoredColumnIds": null + } + ], + "Id": "t35" + }, + "t36": { + "Name": "Genre", + "Schema": "e2e-it", + "ColIds": [ + "c44", + "c45" + ], + "ColDefs": { + "c44": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c44", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c45": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c45", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c45", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t36" + } + }, + "SchemaIssues": { + "t34": { + "ColumnLevelIssues": { + "c37": [ + 14 + ], + "c38": [] + }, + "TableLevelIssues": null + }, + "t35": { + "ColumnLevelIssues": { + "c41": [ + 14 + ], + "c42": [], + "c43": [ + 14, + 24 + ], + "c47": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t36": { + "ColumnLevelIssues": {}, + "TableLevelIssues": [] + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..af959aef6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,28 @@ + +CREATE TABLE IF NOT EXISTS + MY_TABLE ( ID INT64 NOT NULL, + VAL INT64, + ) +PRIMARY KEY + (ID); + +CREATE TABLE IF NOT EXISTS + ALLTYPES ( BOOL_FIELD STRING(MAX) NOT NULL, + INT64_FIELD INT64 NOT NULL, + FLOAT64_FIELD FLOAT64 NOT NULL, + STRING_FIELD STRING(MAX) NOT NULL, + BYTES_FIELD BYTES(MAX) NOT NULL, + TIMESTAMP_FIELD TIMESTAMP NOT NULL, + DATE_FIELD DATE NOT NULL, + NUMERIC_FIELD NUMERIC NOT NULL, + VAL INT64, + ) +PRIMARY KEY + (BOOL_FIELD, + INT64_FIELD, + FLOAT64_FIELD, + STRING_FIELD, + BYTES_FIELD, + TIMESTAMP_FIELD, + DATE_FIELD, + NUMERIC_FIELD); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql new file mode 100644 index 0000000000..3560e1d435 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql @@ -0,0 +1,20 @@ + +CREATE TABLE MY_TABLE ( + ID NUMBER NOT NULL PRIMARY KEY, + VAL NUMBER +); +ALTER TABLE MY_TABLE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +CREATE TABLE ALLTYPES ( + BOOL_FIELD VARCHAR2(10) NOT NULL, + INT64_FIELD NUMBER NOT NULL, + FLOAT64_FIELD BINARY_DOUBLE NOT NULL, + STRING_FIELD VARCHAR2(4000) NOT NULL, + BYTES_FIELD RAW(2000) NOT NULL, + TIMESTAMP_FIELD TIMESTAMP NOT NULL, + DATE_FIELD DATE NOT NULL, + NUMERIC_FIELD NUMBER NOT NULL, + VAL NUMBER, + PRIMARY KEY (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD) +); +ALTER TABLE ALLTYPES ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql new file mode 100644 index 0000000000..7447a310a2 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE "Category" ( + "category_id" NUMBER NOT NULL PRIMARY KEY, + "full_name" VARCHAR2(25), + "last_update" TIMESTAMP +); + +CREATE TABLE "Books" ( + "id" NUMBER NOT NULL, + "title" VARCHAR2(200), + "author_id" NUMBER +); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json new file mode 100644 index 0000000000..f6e317ed00 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/oracle-session.json @@ -0,0 +1,448 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "XEPDB1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t15515": { + "Name": "Category", + "ColIds": [ + "c15516", + "c15517" + ], + "ShardIdColumn": "", + "ColDefs": { + "c15516": { + "Name": "category_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: category_id tinyint(4)", + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Category", + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "ColIds": [ + "c15520", + "c15521", + "c15522", + "c15523", + "c1" + ], + "ShardIdColumn": "", + "ColDefs": { + "c1": { + "Name": "extraCol1", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c1", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15520": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(11)", + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: title varchar(200)", + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: author_id int(11)", + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15523": { + "Name": "synth_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c15523", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15523", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentTable": { + "Id": "", + "OnDelete": "" + }, + "Comment": "Spanner schema for source table Books", + "Id": "t15519" + } + }, + "SyntheticPKeys": { + "t15519": { + "ColId": "c15523", + "Sequence": 0 + } + }, + "SrcSchema": { + "t15515": { + "Name": "Category", + "Schema": "", + "ColIds": [ + "c15516", + "c15517", + "c15518" + ], + "ColDefs": { + "c15516": { + "Name": "category_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15516", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15517": { + "Name": "full_name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 25 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15517", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15518": { + "Name": "last_update", + "Type": { + "Name": "TIMESTAMP", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15518", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c15516", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15515" + }, + "t15519": { + "Name": "Books", + "Schema": "", + "ColIds": [ + "c15520", + "c15521", + "c15522" + ], + "ColDefs": { + "c15520": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15520", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15521": { + "Name": "title", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15521", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15522": { + "Name": "author_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15522", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": null, + "ForeignKeys": null, + "Indexes": null, + "Id": "t15519" + }, + "t15520": { + "Name": "Genre", + "Schema": "", + "ColIds": [ + "c13", + "c14" + ], + "ColDefs": { + "c13": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + }, + "c14": { + "Name": "genre_id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14" + } + }, + "PrimaryKeys": [ + { + "ColId": "c14", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t15520" + } + }, + "SchemaIssues": { + "t15515": { + "ColumnLevelIssues": { + "c15516": [ + 14 + ], + "c15517": [] + }, + "TableLevelIssues": null + }, + "t15519": { + "ColumnLevelIssues": { + "c15520": [ + 14 + ], + "c15521": [], + "c15522": [ + 14 + ], + "c15523": [ + 2 + ] + }, + "TableLevelIssues": null + }, + "t15520": { + "ColumnLevelIssues": {}, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false, + "SpSequences": {}, + "SrcSequences": {} +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql new file mode 100644 index 0000000000..d57b1f3c9e --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSessionIT/spanner-schema.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS Category ( + category_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY(category_id); + +CREATE TABLE IF NOT EXISTS Books ( + id INT64 NOT NULL, + title STRING(200), + author_id INT64, + synth_id STRING(50), + extraCol1 INT64, +) PRIMARY KEY(synth_id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..921e1b3013 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64 +) PRIMARY KEY (id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql new file mode 100644 index 0000000000..bfe9b697af --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE Users ( + id INT64 NOT NULL, + name STRING(200), + age INT64, + migration_shard_id STRING(50) +) PRIMARY KEY (migration_shard_id, id); diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql new file mode 100644 index 0000000000..3264090a6f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE "Users" ( + "id" NUMBER NOT NULL, + "name" VARCHAR2(200), + "age" NUMBER, + PRIMARY KEY ("id") +); + +ALTER TABLE "Users" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json new file mode 100644 index 0000000000..cf062c681f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json @@ -0,0 +1,215 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "oracle", + "DatabaseName": "S1L1", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t2": { + "Name": "Users", + "ColIds": [ + "c11", + "c12", + "c13", + "c16" + ], + "ShardIdColumn": "c16", + "ColDefs": { + "c11": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id NUMBER", + "Id": "c11", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c12": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name VARCHAR2(200)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "age", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: age NUMBER", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c16": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Users", + "Id": "t2" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t2": { + "Name": "Users", + "Schema": "it_test", + "ColIds": [ + "c11", + "c12", + "c13" + ], + "ColDefs": { + "c11": { + "Name": "id", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c11" + }, + "c12": { + "Name": "name", + "Type": { + "Name": "VARCHAR2", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12" + }, + "c13": { + "Name": "age", + "Type": { + "Name": "NUMBER", + "Mods": [], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13" + } + }, + "PrimaryKeys": [ + { + "ColId": "c11", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + } + }, + "SchemaIssues": { + "t2": { + "ColumnLevelIssues": { + "c11": [], + "c12": [], + "c13": [], + "c16": [] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r17", + "Name": "r17", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql new file mode 100644 index 0000000000..3e8b59c4ff --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS human1 ( + name1 STRING(500), + last_name1 STRING(500), + first_name2 STRING(500), + last_name2 STRING(500), + first_name3 STRING(500), + last_name3 STRING(500), + ID INT64 NOT NULL, +) PRIMARY KEY(ID); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql new file mode 100644 index 0000000000..c87cdd07ab --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE "person1" ( + "first_name1" VARCHAR2(500) DEFAULT NULL, + "last_name1" VARCHAR2(500) DEFAULT NULL, + "first_name2" VARCHAR2(500) DEFAULT NULL, + "last_name2" VARCHAR2(500) DEFAULT NULL, + "first_name3" VARCHAR2(500) DEFAULT NULL, + "last_name3" VARCHAR2(500) DEFAULT NULL, + "ID" NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY +); +ALTER TABLE "person1" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java index 132b3bc84f..9263465ac0 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java @@ -59,7 +59,7 @@ public static ISpannerMigrationTransformer getApplyTransformationImpl( URL[] classLoaderUrls = JarFileReader.saveFilesLocally(customTransformation.jarPath()); // Create a new URLClassLoader - URLClassLoader urlClassLoader = new URLClassLoader(classLoaderUrls); + URLClassLoader urlClassLoader = new URLClassLoader(classLoaderUrls, Thread.currentThread().getContextClassLoader()); // Load the target class Class customTransformationClass = From fad52fffc1365f810535727df1211311bc696628 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Mon, 24 Aug 2026 12:23:15 +0000 Subject: [PATCH 02/13] Fix spotless, checkstyle, and compilation errors for Oracle ITs --- .../v2/templates/DataStreamToSpannerIT.java | 4 +- .../templates/DataStreamToSpannerITBase.java | 40 +- .../DataStreamToSpannerOracleDDLIT.java | 117 ++-- ...ataStreamToSpannerOracleRetryAllDLQIT.java | 64 +- .../DataStreamToSpannerOracleRetryDLQIT.java | 102 ++- ...treamToSpannerOracleStringOverridesIT.java | 68 +- ...OracleWideRowForMaxColumnsPerTablesIT.java | 42 +- ...amToSpannerShardedOracleRetryAllDLQIT.java | 194 ++++-- ...reamToSpannerReservedKeywordsOracleIT.java | 63 +- .../OracleDataStreamToSpannerEventsIT.java | 4 +- ...cleDataStreamToSpannerFileOverridesIT.java | 174 +++-- .../oracle/OracleDataStreamToSpannerIT.java | 232 ++++--- .../OracleDataStreamToSpannerMixedIT.java | 191 +++-- .../OracleDataStreamToSpannerSessionIT.java | 63 +- .../OracleDataStreamToSpannerTimezoneIT.java | 78 ++- ...rWideRowForMax16KeyTablePerDatabaseIT.java | 78 ++- ...RowForMaxTableNameWithMaxColumnNameIT.java | 63 +- .../OracleDatastreamToSpannerDataTypesIT.java | 656 ++++++++++-------- ...amToSpannerSingleDFShardedMigrationIT.java | 192 +++-- ...astreamToSpannerTableAndIndexLimitsIT.java | 106 ++- ...erWideRowForMax9MibTablePerDatabaseIT.java | 47 +- ...racleSeparateShadowTableDatabaseDDLIT.java | 135 ++-- ...leSeparateShadowTableDatabaseEventsIT.java | 348 +++++++--- ...ateShadowTableDatabaseFileOverridesIT.java | 182 +++-- ...cleSeparateShadowTableDatabaseMixedIT.java | 183 +++-- ...eparateShadowTableDatabasePKFocusedIT.java | 510 ++++++++------ ...eSeparateShadowTableDatabaseSessionIT.java | 209 ++++-- ...MigrationWithMigrationShardIdColumnIT.java | 335 ++++++--- ...rationWithoutMigrationShardIdColumnIT.java | 327 ++++++--- ...bleDatabaseSingleDFShardedMigrationIT.java | 322 ++++++--- ...eShadowTableDatabaseStringOverridesIT.java | 192 +++-- 31 files changed, 3434 insertions(+), 1887 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java index 75cfd32028..50dede9660 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java @@ -507,7 +507,7 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - flushOracleRedoLogs(cloudSqlResourceManager); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager).runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); } @@ -578,7 +578,7 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - flushOracleRedoLogs(cloudSqlResourceManager); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager).runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index bc61635ed2..7a61cacdb2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -89,8 +89,9 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { .maybeUseStaticInstance() .build(); } + public org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager setUpOracleResourceManager() { - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); @@ -628,31 +629,36 @@ protected ConditionCheck buildBaseConditionCheck( return combinedCondition; } - public void flushOracleRedoLogs(org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager) { + public void flushOracleRedoLogs( + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager) { boolean success = false; if (oracleResourceManager != null) { try { oracleResourceManager.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); - org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).info("Successfully flushed Oracle redo logs natively."); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .info("Successfully flushed Oracle redo logs natively."); success = true; } catch (Exception e) { - org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).warn("Failed to switch Oracle log via ResourceManager, attempting raw JDBC fallback...", e); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .warn( + "Failed to switch Oracle log via ResourceManager, attempting raw JDBC fallback...", + e); } } - + if (!success && System.getProperty("hostIp") != null) { - String url = "jdbc:oracle:thin:@//" + System.getProperty("hostIp") + ":1521/XE"; - String user = System.getProperty("cloudProxyUsername", "system"); - String pass = System.getProperty("cloudProxyPassword", "TestPassword123"); - try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); - java.sql.Statement stmt = conn.createStatement()) { - stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); - org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).info("Successfully flushed Oracle redo logs via raw JDBC fallback."); - } catch (Exception ex) { - org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class).error("Raw JDBC fallback log flush also failed.", ex); - } + String url = "jdbc:oracle:thin:@//" + System.getProperty("hostIp") + ":1521/XE"; + String user = System.getProperty("cloudProxyUsername", "system"); + String pass = System.getProperty("cloudProxyPassword", "TestPassword123"); + try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); + java.sql.Statement stmt = conn.createStatement()) { + stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .info("Successfully flushed Oracle redo logs via raw JDBC fallback."); + } catch (Exception ex) { + org.slf4j.LoggerFactory.getLogger(DataStreamToSpannerITBase.class) + .error("Raw JDBC fallback log flush also failed.", ex); + } } } - - } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java index 4ebebed7ce..28ade7e925 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -18,20 +18,19 @@ import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; -import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; -import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.util.Map; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; @@ -61,7 +60,8 @@ public class DataStreamToSpannerOracleDDLIT extends DataStreamToSpannerITBase { private static final Logger LOG = LoggerFactory.getLogger(DataStreamToSpannerOracleDDLIT.class); - private static final String SPANNER_DDL_RESOURCE = "oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/DataStreamToSpannerOracleDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; private static final String TABLE1 = "AllDatatypeColumns"; private static final String TABLE2 = "AllDatatypeColumns2"; @@ -81,34 +81,38 @@ public class DataStreamToSpannerOracleDDLIT extends DataStreamToSpannerITBase { public static SpannerResourceManager spannerResourceManager; public static GcsResourceManager gcsResourceManager; public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; - public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager + datastreamResourceManager; - private void flushOracleLogs() { - try { - Class.forName("oracle.jdbc.OracleDriver"); - try (Connection conn = DriverManager.getConnection("jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - } - } catch (Exception e) { - throw new RuntimeException("Failed to switch logfile", e); + try { + Class.forName("oracle.jdbc.OracleDriver"); + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } + } catch (Exception e) { + throw new RuntimeException("Failed to switch logfile", e); + } } @Before - public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; synchronized (DataStreamToSpannerOracleDDLIT.class) { testInstances.add(this); if (jobInfo == null) { oracleResourceManager = setUpOracleResourceManager(); - + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); spannerResourceManager = setUpSpannerResourceManager(); @@ -120,22 +124,33 @@ public void setUp() throws IOException, InterruptedException { CustomTransformation.builder( "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") .build(); - - String[] tables = {"AllDatatypeColumns", "AllDatatypeColumns2", "DatatypeColumnsWithSizes", "DatatypeColumnsReducedSizes", "Users", "Authors", "AllDatatypeTransformation", "Singers", "Books"}; - for (String table : tables) { - try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); - } catch (Exception e) { - // Ignore if doesn't exist - } - } - - // Also drop sequences - try { - oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); - } catch (Exception e) { - } -executeSqlScript(oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql"); + + String[] tables = { + "AllDatatypeColumns", + "AllDatatypeColumns2", + "DatatypeColumnsWithSizes", + "DatatypeColumnsReducedSizes", + "Users", + "Authors", + "AllDatatypeTransformation", + "Singers", + "Books" + }; + for (String table : tables) { + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); + } catch (Exception e) { + } + executeSqlScript( + oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql"); flushOracleLogs(); jobInfo = @@ -157,12 +172,17 @@ public void setUp() throws IOException, InterruptedException { datastreamResourceManager, null, // sessionResourceContent org.apache.beam.it.gcp.datastream.OracleSource.builder( - oracleResourceManager.getHost(), - oracleResourceManager.getUsername(), // Explicit Datastream logminer user instead of 'system' - oracleResourceManager.getPassword(), // Datastream user password - oracleResourceManager.getPort(), - "XEPDB1") - .setAllowedTables(java.util.Map.of(oracleResourceManager.getUsername().toUpperCase(), java.util.Arrays.asList(TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + oracleResourceManager.getHost(), + oracleResourceManager + .getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleResourceManager.getUsername().toUpperCase(), + java.util.Arrays.asList( + TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) .build()); } } @@ -189,11 +209,16 @@ public void migrationTestWithAllDatatypeConversionMapping() { List.of( new ConditionCheck() { @Override - protected String getDescription() { return "Run SQL inserts"; } + protected String getDescription() { + return "Run SQL inserts"; + } + @Override protected CheckResult check() { try { - executeSqlScript(oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql"); + executeSqlScript( + oracleResourceManager, + "oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql"); flushOracleLogs(); return new CheckResult(true, "Success"); } catch (Exception e) { @@ -225,7 +250,8 @@ private void assertAllDatatypeColumnsTableBackfillContents() { List> events = new ArrayList<>(); Map row = new HashMap<>(); row.put("varchar_column", "value1"); - // Only asserting primary key + simple int column to verify end-to-end sync without complex base64 logic + // Only asserting primary key + simple int column to verify end-to-end sync without complex + // base64 logic row.put("int_column", "50000"); events.add(row); @@ -235,7 +261,6 @@ private void assertAllDatatypeColumnsTableBackfillContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private void assertAllDatatypeColumnsTableCdcContents() { List> events = new ArrayList<>(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index 7c890f0d8a..c92f4f0643 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -71,7 +85,7 @@ public void setUp() throws IOException, InterruptedException { // Setup Oracle Source CloudOracleResourceManager sysUser = setUpOracleResourceManager(); - + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = CloudOracleResourceManager.builder(testName); sysBuilder.setHost(sysUser.getHost()); @@ -98,11 +112,13 @@ public void setUp() throws IOException, InterruptedException { executeSqlScript(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE); // Add Supplemental Log Data required by Datastream CDC - List.of("Customers", "Orders", "AllDataTypes").forEach( - tableName -> { - jdbcResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); - }); + List.of("Customers", "Orders", "AllDataTypes") + .forEach( + tableName -> { + jdbcResourceManager.runSQLUpdate( + String.format( + "ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); + }); gcsResourceManager = setUpSpannerITGcsResourceManager(); @@ -225,12 +241,18 @@ public void testDataStreamToSpannerRetryAllDLQ() throws Exception { assertThatResult(dlqWaitResult).meetsConditions(); assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); - assertTrue("id=999 should NOT exist yet in AllDataTypes", !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue( + "id=999 should NOT exist yet in AllDataTypes", + !rowExistsInSpanner("AllDataTypes", "id", 999)); assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); - assertTrue("id=1 should NOT exist yet in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue( + "id=1 should NOT exist yet in Customers", + !rowExistsInSpanner("Customers", "CustomerId", 1)); assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); - assertTrue("id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); - assertTrue("id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + assertTrue( + "id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue( + "id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); pipelineOperator().cancelJobAndFinish(createConfig(jobInfo, Duration.ofMinutes(15))); @@ -249,13 +271,16 @@ public void testDataStreamToSpannerRetryAllDLQ() throws Exception { Map retryJobParameters = new HashMap<>(); retryJobParameters.put("runMode", "retryAllDLQ"); - retryJobParameters.put("schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); - retryJobParameters.put("transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryJobParameters.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryJobParameters.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); retryJobParameters.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); retryJobParameters.put("transformationCustomParameters", "mode=good"); retryJobParameters.put("dlqMaxRetryCount", "20"); retryJobParameters.put("dlqRetryMinutes", "1"); - retryJobParameters.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryJobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); PipelineLauncher.LaunchInfo retryJobInfo = launchDataflowJob( @@ -285,7 +310,10 @@ public void testDataStreamToSpannerRetryAllDLQ() throws Exception { .setMinEvents(1) .setMaxEvents(1) .build()) - .and(SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes").setMinRows(2).build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build()) .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); PipelineOperator.Result retryResult = @@ -295,7 +323,8 @@ public void testDataStreamToSpannerRetryAllDLQ() throws Exception { assertThatResult(retryResult).meetsConditions(); - assertTrue("id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue( + "id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); List rows = spannerResourceManager.runQuery("SELECT * FROM AllDataTypes WHERE id = 999"); @@ -304,7 +333,8 @@ public void testDataStreamToSpannerRetryAllDLQ() throws Exception { .hasRecordsUnorderedCaseInsensitiveColumns(List.of(expectedRow)); assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); - assertTrue("id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue( + "id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index 08a0aee6cc..c7d9c068ae 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -68,7 +82,8 @@ public class DataStreamToSpannerOracleRetryDLQIT extends DataStreamToSpannerITBa private CloudOracleResourceManager setUpOracleResourceManagerForShard(String shardName) { if (System.getProperty("hostIp") != null) { - CloudOracleResourceManager.Builder shardBuilder = CloudOracleResourceManager.builder(testName + shardName); + CloudOracleResourceManager.Builder shardBuilder = + CloudOracleResourceManager.builder(testName + shardName); shardBuilder.setUsername(System.getProperty("cloudProxyUsername", "system")); shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); shardBuilder.setHost(System.getProperty("hostIp")); @@ -90,20 +105,33 @@ public void setUp() throws IOException, InterruptedException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); jdbcResourceManagerShardA = setUpOracleResourceManagerForShard("A"); - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) {} - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) {} - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) {} + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); + } catch (Exception e) { + } + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); + } catch (Exception e) { + } + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); + } catch (Exception e) { + } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); gcsResourceManager = setUpSpannerITGcsResourceManager(); datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); pubsubResourceManager = setUpPubSubResourceManager(); @@ -122,11 +150,15 @@ public void setUp() throws IOException, InterruptedException { jdbcResourceManagerShardA.getHost(), jdbcResourceManagerShardA.getUsername(), jdbcResourceManagerShardA.getPassword(), - jdbcResourceManagerShardA.getPort(), "XEPDB1") + jdbcResourceManagerShardA.getPort(), + "XEPDB1") .setAllowedTables( Map.of( jdbcResourceManagerShardA.getUsername().toUpperCase(), - List.of("Customers", "Orders", "AllDataTypes"))) // Assuming schemas are username + List.of( + "Customers", + "Orders", + "AllDataTypes"))) // Assuming schemas are username .build(); SourceConfig sourceConfigA = @@ -252,21 +284,28 @@ public void testDataStreamToSpannerShardedRetryDLQ() throws Exception { .build())); assertThatResult(dlqWaitResult).meetsConditions(); - assertTrue("id=1 should exist in AllDataTypes on shard1", + assertTrue( + "id=1 should exist in AllDataTypes on shard1", rowExistsInSpanner("AllDataTypes", "id", 1, "shard1")); - assertTrue("id=999 should NOT exist yet in AllDataTypes on shard1", + assertTrue( + "id=999 should NOT exist yet in AllDataTypes on shard1", !rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); - assertTrue("id=2 should exist in Customers on shard1", + assertTrue( + "id=2 should exist in Customers on shard1", rowExistsInSpanner("Customers", "CustomerId", 2, "shard1")); - assertTrue("id=1 should NOT exist yet in Customers on shard1", + assertTrue( + "id=1 should NOT exist yet in Customers on shard1", !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); - assertTrue("id=102 should exist in Orders on shard1", + assertTrue( + "id=102 should exist in Orders on shard1", rowExistsInSpanner("Orders", "OrderId", 102, "shard1")); - assertTrue("id=101 should NOT exist yet in Orders on shard1", + assertTrue( + "id=101 should NOT exist yet in Orders on shard1", !rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); - assertTrue("id=103 should NOT exist yet in Orders on shard1", + assertTrue( + "id=103 should NOT exist yet in Orders on shard1", !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); LOG.info("Launching retryDLQ job with session file to process DLQ"); @@ -344,7 +383,8 @@ public void testDataStreamToSpannerShardedRetryDLQ() throws Exception { assertTrue("id=999 should exist", rowExistsInSpanner("AllDataTypes", "id", 999, "shard1")); assertTrue("id=3 should exist", rowExistsInSpanner("Customers", "CustomerId", 3, "shard1")); - assertTrue("id=1 should NOT exist", !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); + assertTrue( + "id=1 should NOT exist", !rowExistsInSpanner("Customers", "CustomerId", 1, "shard1")); assertTrue("id=101 should exist", rowExistsInSpanner("Orders", "OrderId", 101, "shard1")); assertTrue("id=103 should NOT exist", !rowExistsInSpanner("Orders", "OrderId", 103, "shard1")); @@ -402,14 +442,26 @@ private String generateSourceConfig( return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " },\n" + " {\n" - + " \"logicalShardId\": \"" + shardB + "\",\n" - + " \"dbName\": \"" + dbB + "\",\n" - + " \"streamId\": \"" + streamB + "\"\n" + + " \"logicalShardId\": \"" + + shardB + + "\",\n" + + " \"dbName\": \"" + + dbB + + "\",\n" + + " \"streamId\": \"" + + streamB + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java index 9fafdbe667..330cefa4a5 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; @@ -48,7 +62,8 @@ public class DataStreamToSpannerOracleStringOverridesIT extends DataStreamToSpan private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = + new HashSet<>(); public static PubsubResourceManager pubsubResourceManager; public static SpannerResourceManager spannerResourceManager; @@ -67,34 +82,42 @@ public void setUp() throws IOException { pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); oracleResourceManager = setUpOracleResourceManager(); - datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + gcsPrefix = getGcsPath(testName + "/cdc/", gcsResourceManager) .replace("gs://" + gcsResourceManager.getBucket(), ""); - try { oracleResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch(Exception e) {} + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"person1\""); + } catch (Exception e) { + } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - + try { - oracleResourceManager.runSQLUpdate("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL"); - } catch(Exception e) { - // Might exist or fail if we are not SYS, ignore wrapper + oracleResourceManager.runSQLUpdate( + "GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL"); + } catch (Exception e) { + // Might exist or fail if we are not SYS, ignore wrapper } - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - OracleSource oracleSource = OracleSource.builder( - oracleResourceManager.getHost(), - oracleResourceManager.getUsername(), - oracleResourceManager.getPassword(), - oracleResourceManager.getPort(), - oracleResourceManager.getDatabaseName()) - .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("person1"))) - .build(); + OracleSource oracleSource = + OracleSource.builder( + oracleResourceManager.getHost(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword(), + oracleResourceManager.getPort(), + oracleResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); Map overridesMap = new HashMap<>(); overridesMap.put("inputFileFormat", "avro"); @@ -126,7 +149,11 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, gcsResourceManager, oracleResourceManager, datastreamResourceManager); + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + oracleResourceManager, + datastreamResourceManager); } @Test @@ -139,6 +166,7 @@ public void migrationTestWithRenameTableAndColumns() { protected String getDescription() { return "Insert records into Oracle"; } + @Override protected CheckResult check() { try { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index a8fc8d5a31..e8a0e08779 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -34,7 +48,6 @@ import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -44,7 +57,8 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT extends DataStreamToSpannerITBase { +public class DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT + extends DataStreamToSpannerITBase { private static final Integer NUM_EVENTS = 1; private static final Integer NUM_TABLES = 1; @@ -65,9 +79,12 @@ public class DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT extends Da static { for (int i = 1; i <= NUM_TABLES; i++) { - // According to instructions on quoting/identifiers: Oracle often folds to uppercase but Datastream captures whatever it is. Oh wait, Oracle tables are usually uppercase or need quotes. - // Let's use uppercase to avoid case folding issues if unquoted - TABLE_NAMES.add("DATASTREAMTOSPANNER_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + // According to instructions on quoting/identifiers: Oracle often folds to uppercase but + // Datastream captures whatever it is. Oh wait, Oracle tables are usually uppercase or need + // quotes. + // Let's use uppercase to avoid case folding issues if unquoted + TABLE_NAMES.add( + "DATASTREAMTOSPANNER_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); } for (int i = 1; i <= NUM_COLUMNS; i++) { COLUMNS.add("COL_" + i); @@ -88,7 +105,7 @@ public void setUp() throws IOException { spannerResourceManager = setUpSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { @@ -98,12 +115,15 @@ public void setUp() throws IOException { builder.setUsername(System.getProperty("cloudProxyUsername")); builder.setDatabaseName("XEPDB1"); } - cloudOracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + cloudOracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); String sessionContent = generateBaseSchema(); - sessionContent = sessionContent.replaceAll("SRC_DATABASE", cloudOracleResourceManager.getUsername().toUpperCase()) - .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()) - .replaceAll("\"mysql\"", "\"oracle\""); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", cloudOracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()) + .replaceAll("\"mysql\"", "\"oracle\""); for (int i = 1; i <= NUM_TABLES; i++) { sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index c218e40fa8..a33650473a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -74,41 +88,50 @@ public void setUp() throws IOException, InterruptedException { String oracleUser = System.getProperty("cloudProxyUsername", "system"); String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); - - jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) {} - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) {} - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) {} + jdbcResourceManagerShardA = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); + } catch (Exception e) { + } + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); + } catch (Exception e) { + } + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); + } catch (Exception e) { + } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - + + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + gcsResourceManager = setUpSpannerITGcsResourceManager(); datastreamResourceManager = - org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder(testName, PROJECT, REGION) + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder( + testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) .setPrivateConnectivity("datastream-connect-2") .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("hostIp"), - oracleUser, - oraclePassword, - 1521, - "XEPDB1") + System.getProperty("hostIp"), oracleUser, oraclePassword, 1521, "XEPDB1") .setAllowedTables( java.util.Map.of( - oracleUser, - java.util.List.of("Customers", "Orders", "AllDataTypes"))) + oracleUser, java.util.List.of("Customers", "Orders", "AllDataTypes"))) .build(); com.google.cloud.datastream.v1.SourceConfig sourceConfig = @@ -116,11 +139,17 @@ public void setUp() throws IOException, InterruptedException { com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), GCS_PATH_PREFIX + "/cdc/", org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + "gcs-profile", + gcsResourceManager.getBucket(), + GCS_PATH_PREFIX + "/cdc/", + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat + .AVRO_FILE_FORMAT); com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( - "test_stream_"+ org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + "test_stream_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5) + .toLowerCase(), sourceConfig, destinationConfig); @@ -129,21 +158,17 @@ public void setUp() throws IOException, InterruptedException { String shardConfig = generateSourceConfig( - streamNameA, - oracleUser, - "shard1", - streamNameA, - oracleUser, - "shard2"); + streamNameA, oracleUser, "shard1", streamNameA, oracleUser, "shard2"); gcsResourceManager.createArtifact( - "input/shardingConfig.conf", shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + "input/shardingConfig.conf", + shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); gcsResourceManager.uploadArtifact( "input/overrides.json", - com.google.common.io.Resources.getResource("oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json").getPath()); - gcsResourceManager.uploadArtifact( - "input/customShard.jar", - getCustomShardJarPath()); + com.google.common.io.Resources.getResource( + "oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT/overrides.json") + .getPath()); + gcsResourceManager.uploadArtifact("input/customShard.jar", getCustomShardJarPath()); java.util.Map jobParameters = new java.util.HashMap<>(); jobParameters.put( @@ -154,14 +179,18 @@ public void setUp() throws IOException, InterruptedException { jobParameters.put("transformationCustomParameters", "mode=bad"); jobParameters.put("dlqMaxRetryCount", "20"); jobParameters.put("dlqRetryMinutes", "60"); - jobParameters.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); - jobParameters.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParameters.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + jobParameters.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); // Explicitly set inputFilePattern for manual stream - jobParameters.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + GCS_PATH_PREFIX + "/cdc/"); + jobParameters.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/" + GCS_PATH_PREFIX + "/cdc/"); if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - jobParameters.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + jobParameters.put("jdbcDriverJars", driverPath); } jobInfo = @@ -178,12 +207,10 @@ public void setUp() throws IOException, InterruptedException { gcsResourceManager, datastreamResourceManager, null, - null - ); - - } + null); + } } - } + } @AfterClass public static void cleanUp() throws IOException { @@ -235,17 +262,17 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { .build()) .and( SpannerRowsCheck.builder(spannerResourceManager, "Orders") - .setMinRows(1) + .setMinRows(1) .setMaxRows(1) .build()) .and( SpannerRowsCheck.builder(spannerResourceManager, "Customers") - .setMinRows(1) + .setMinRows(1) .setMaxRows(1) .build()) .and( SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") - .setMinRows(1) + .setMinRows(1) .setMaxRows(1) .build())); assertThatResult(dlqWaitResult).meetsConditions(); @@ -254,12 +281,18 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { // Verify Spanner state LOG.info("Verifying Spanner state before retry job runs"); assertTrue("id=1 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 1)); - assertTrue("id=999 should NOT exist yet in AllDataTypes", !rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue( + "id=999 should NOT exist yet in AllDataTypes", + !rowExistsInSpanner("AllDataTypes", "id", 999)); assertTrue("id=2 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 2)); - assertTrue("id=1 should NOT exist yet in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue( + "id=1 should NOT exist yet in Customers", + !rowExistsInSpanner("Customers", "CustomerId", 1)); assertTrue("id=102 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 102)); - assertTrue("id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); - assertTrue("id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); + assertTrue( + "id=101 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 101)); + assertTrue( + "id=103 should NOT exist yet in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); // 4. Stop the regular pipeline LOG.info("Stopping regular pipeline"); @@ -285,19 +318,22 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { LOG.info("Launching retryAllDLQ job"); Map retryParams = new HashMap<>(); retryParams.put("runMode", "retryAllDLQ"); - retryParams.put("schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); - retryParams.put("transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); + retryParams.put( + "schemaOverridesFilePath", getGcsPath("input/overrides.json", gcsResourceManager)); + retryParams.put( + "transformationJarPath", getGcsPath("input/customShard.jar", gcsResourceManager)); retryParams.put("transformationClassName", "com.custom.CustomTransformationForDLQIT"); retryParams.put("transformationCustomParameters", "mode=good"); - retryParams.put("deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); + retryParams.put( + "deadLetterQueueDirectory", getGcsPath(GCS_PATH_PREFIX + "/dlq/", gcsResourceManager)); retryParams.put("dlqMaxRetryCount", "20"); retryParams.put("dlqRetryMinutes", "60"); retryParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); // Proprietary Drivers if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - retryParams.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + retryParams.put("jdbcDriverJars", driverPath); } PipelineLauncher.LaunchInfo retryJobInfo = @@ -314,8 +350,7 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { gcsResourceManager, null, null, - null - ); + null); LOG.info("RetryAllDLQ job launched: {}", retryJobInfo.jobId()); assertThatPipeline(retryJobInfo).isRunning(); @@ -329,7 +364,10 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { DlqEventsCountCheck.builder(gcsResourceManager, GCS_PATH_PREFIX + "/dlq/severe/") .setMinEvents(1) .build()) - .and(SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes").setMinRows(2).build()) + .and( + SpannerRowsCheck.builder(spannerResourceManager, "AllDataTypes") + .setMinRows(2) + .build()) .and(SpannerRowsCheck.builder(spannerResourceManager, "Orders").setMinRows(2).build()); PipelineOperator.Result retryResult = @@ -341,9 +379,11 @@ public void testDataStreamToSpannerShardedRetryAllDLQ() throws Exception { LOG.info("Retry job completed processing successfully"); // 8. Verify Spanner state - assertTrue("id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); + assertTrue( + "id=999 should exist in AllDataTypes", rowExistsInSpanner("AllDataTypes", "id", 999)); assertTrue("id=3 should exist in Customers", rowExistsInSpanner("Customers", "CustomerId", 3)); - assertTrue("id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); + assertTrue( + "id=1 should NOT exist in Customers", !rowExistsInSpanner("Customers", "CustomerId", 1)); assertTrue("id=101 should exist in Orders", rowExistsInSpanner("Orders", "OrderId", 101)); assertTrue("id=103 should NOT exist in Orders", !rowExistsInSpanner("Orders", "OrderId", 103)); } @@ -384,14 +424,26 @@ private String generateSourceConfig( return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " },\n" + " {\n" - + " \"logicalShardId\": \"" + shardB + "\",\n" - + " \"dbName\": \"" + dbB + "\",\n" - + " \"streamId\": \"" + streamB + "\"\n" + + " \"logicalShardId\": \"" + + shardB + + "\",\n" + + " \"dbName\": \"" + + dbB + + "\",\n" + + " \"streamId\": \"" + + streamB + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java index 832db20580..b2fbfeea0d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static java.util.Map.entry; @@ -8,14 +22,16 @@ import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; @@ -35,8 +51,6 @@ import org.junit.runners.JUnit4; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; -import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @@ -63,21 +77,21 @@ public class DatastreamToSpannerReservedKeywordsOracleIT extends DataStreamToSpa public void setUp() throws IOException { LOG.info("Setting up Oracle resource manager..."); oracleResourceManager = setUpOracleResourceManager(); - + LOG.info("Setting up Spanner resource manager..."); spannerResourceManager = setUpSpannerResourceManager(); LOG.info( "Spanner resource manager created with instance ID: {}", spannerResourceManager.getInstanceId()); - + LOG.info("Setting up GCS resource manager..."); gcsResourceManager = setUpSpannerITGcsResourceManager(); LOG.info("GCS resource manager created with bucket: {}", gcsResourceManager.getBucket()); - + LOG.info("Setting up Pub/Sub resource manager..."); pubsubResourceManager = setUpPubSubResourceManager(); LOG.info("Pub/Sub resource manager created."); - + LOG.info("Setting up Datastream resource manager..."); datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) @@ -100,15 +114,16 @@ public void cleanUp() { private void flushOracleLogs() { LOG.info("Flushing Oracle logs via raw JDBC..."); - try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - LOG.info("Successfully switched Oracle log file."); + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + LOG.info("Successfully switched Oracle log file."); } catch (Exception e) { - LOG.error("Failed to switch Oracle logfile", e); + LOG.error("Failed to switch Oracle logfile", e); } } @@ -121,7 +136,8 @@ public void testOracleReservedKeywords() throws Exception { LOG.info("Table true does not exist or could not be dropped: " + e.getMessage()); } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - oracleResourceManager.runSQLUpdate("ALTER TABLE \"true\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + oracleResourceManager.runSQLUpdate( + "ALTER TABLE \"true\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); flushOracleLogs(); LOG.info("Creating Spanner DDL..."); @@ -135,7 +151,7 @@ public void testOracleReservedKeywords() throws Exception { spannerResourceManager.getDatabaseId(), List.of("true"), // Table name SESSION_FILE); - + OracleSource oracleSource = OracleSource.builder( oracleResourceManager.getHost(), @@ -143,7 +159,8 @@ public void testOracleReservedKeywords() throws Exception { oracleResourceManager.getPassword(), oracleResourceManager.getPort(), oracleResourceManager.getDatabaseName()) - .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("true"))) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("true"))) .build(); Map jobParams = new HashMap<>(); @@ -174,7 +191,7 @@ public void testOracleReservedKeywords() throws Exception { createConfig(jobInfo, Duration.ofMinutes(20)), SpannerRowsCheck.builder(spannerResourceManager, "`true`").setMinRows(1).build()); assertThatResult(result).meetsConditions(); - + List rows = null; try { rows = spannerResourceManager.readTableRecords("true", List.of("id", "ALL", "AND", "AS")); @@ -182,7 +199,7 @@ public void testOracleReservedKeywords() throws Exception { LOG.error("Exception while reading spanner rows from `true`", e); throw e; } - + SpannerAsserts.assertThatStructs(rows) .hasRecordsUnorderedCaseInsensitiveColumns( List.of( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java index efdffc3bda..9b052cd89f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerEventsIT.java @@ -1,4 +1,3 @@ -/* GENERATED BY: Test Automation Skill */ /* * Copyright (C) 2023 Google LLC * @@ -56,7 +55,8 @@ @RunWith(JUnit4.class) public class OracleDataStreamToSpannerEventsIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerEventsIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerEventsIT.class); private static final String TABLE1 = "Users"; private static final String TABLE2 = "Movie"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index 85a051b683..dcdda1276d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -43,11 +57,15 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleDataStreamToSpannerFileOverridesIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerFileOverridesIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerFileOverridesIT.class); - private static final String ORACLE_SCHEMA = "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql"; - private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; - private static final String OVERRIDE_FILE = "oracle/OracleDataStreamToSpannerFileOverridesIT/override.json"; + private static final String ORACLE_SCHEMA = + "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDataStreamToSpannerFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = + "oracle/OracleDataStreamToSpannerFileOverridesIT/override.json"; private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; private static final String ORACLE_TABLE = "person1"; private static final String SPANNER_TABLE = "human1"; @@ -63,39 +81,63 @@ public class OracleDataStreamToSpannerFileOverridesIT extends DataStreamToSpanne private static HashSet testInstances = new HashSet<>(); private void setUpOracleUser(String user, String password) { - oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @@ -106,7 +148,7 @@ public void setUp() throws IOException { testInstances.add(this); if (!initialized) { LOG.info("Setting up Oracle sys resource manager..."); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); @@ -115,15 +157,16 @@ public void setUp() throws IOException { sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); } - oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); - + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); - + LOG.info("Provisioning isolated user: " + oracleUser); setUpOracleUser(oracleUser, oraclePassword); - - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { builder.setPassword(oraclePassword); @@ -132,8 +175,9 @@ public void setUp() throws IOException { builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); } - oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); - + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + spannerResourceManager = setUpSpannerResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); @@ -145,7 +189,7 @@ public void setUp() throws IOException { executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - + gcsResourceManager.uploadArtifact( GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); @@ -201,33 +245,41 @@ public void migrationTestWithRenameTableAndColumns() throws Exception { datastreamResourceManager, null, oracleSource); - + assertThatPipeline(jobInfo).isRunning(); - ConditionCheck sendDataCondition = new ConditionCheck() { - @Override - public String getDescription() { - return "Insert data into Oracle and flush logs"; - } + ConditionCheck sendDataCondition = + new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); - @Override - protected CheckResult check() { - try { - oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); - - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("hostIp", "localhost") + + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } } - return new CheckResult(true, "Data inserted and logs flushed"); - } catch (Exception e) { - LOG.error("Failed to insert data or flush logs", e); - return new CheckResult(false, e.getMessage()); - } - } - }; + }; ChainedConditionCheck conditionCheck = ChainedConditionCheck.builder( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index 994304eaa9..170df238bf 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -131,40 +145,66 @@ public void testDataStreamOracleToSpannerJson() throws IOException { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @@ -174,21 +214,29 @@ private void simpleOracleToSpannerTest( Function paramsAdder) throws IOException { - cloudOracleSysUser = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername("sys as sysdba") - .setPassword(System.getProperty("cloudProxyPassword")) - .setDatabaseName("XE").setHost(System.getProperty("hostIp")).setPort(1521) - .build(); + cloudOracleSysUser = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudProxyPassword")) + .setDatabaseName("XE") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); String oracleUser = System.getProperty("cloudProxyUsername"); String oraclePassword = System.getProperty("cloudProxyPassword"); - //setUpOracleUser(oracleUser, oraclePassword); -// setUpOracleUser(oracleUser, oraclePassword); - - cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1").setHost(System.getProperty("hostIp")).setPort(1521) - .build(); + // setUpOracleUser(oracleUser, oraclePassword); + // setUpOracleUser(oracleUser, oraclePassword); + + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); SpannerResourceManager.Builder spannerResourceManagerBuilder = SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) @@ -204,10 +252,9 @@ private void simpleOracleToSpannerTest( tableNames.forEach( tableName -> { - cloudSqlResourceManager.createTable( - tableName, createJdbcSchema()); - cloudSqlResourceManager.runSQLUpdate( - String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); + cloudSqlResourceManager.createTable(tableName, createJdbcSchema()); + cloudSqlResourceManager.runSQLUpdate( + String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); }); OracleSource jdbcSource = @@ -233,7 +280,10 @@ private void simpleOracleToSpannerTest( "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, fileFormat); Stream stream = - datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.createStream( + "stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); datastreamResourceManager.startStream(stream); createPubSubNotifications(); @@ -312,27 +362,39 @@ private void createSpannerTables(List tableNames, Dialect spannerDialect boolean usingPg = Dialect.POSTGRESQL.equals(spannerDialect); tableNames.forEach( tableName -> { - String q = usingPg ? "\"" : "`"; - spannerResourceManager.executeDdlStatement( - "CREATE TABLE " - + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") - + " (" - + q + ROW_ID + q - + (usingPg ? " bigint " : " INT64 ") - + "NOT NULL, " - + q + NAME + q - + (usingPg ? " character varying(200), " : " STRING(MAX), ") - + q + AGE + q - + (usingPg ? " bigint, " : " INT64, ") - + q + MEMBER + q - + (usingPg ? " character varying(200), " : " STRING(MAX), ") - + q + ENTRY_ADDED + q - + (usingPg ? " character varying(200)" : " STRING(MAX)") - + (usingPg ? ", " : ") ") - + "PRIMARY KEY (" - + q + ROW_ID + q - + ")" - + (usingPg ? ")" : "")); + String q = usingPg ? "\"" : "`"; + spannerResourceManager.executeDdlStatement( + "CREATE TABLE " + + (usingPg ? "\"" + tableName + "\"" : "`" + tableName + "`") + + " (" + + q + + ROW_ID + + q + + (usingPg ? " bigint " : " INT64 ") + + "NOT NULL, " + + q + + NAME + + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + + AGE + + q + + (usingPg ? " bigint, " : " INT64, ") + + q + + MEMBER + + q + + (usingPg ? " character varying(200), " : " STRING(MAX), ") + + q + + ENTRY_ADDED + + q + + (usingPg ? " character varying(200)" : " STRING(MAX)") + + (usingPg ? ", " : ") ") + + "PRIMARY KEY (" + + q + + ROW_ID + + q + + ")" + + (usingPg ? ")" : "")); }); } @@ -397,19 +459,19 @@ protected CheckResult check() { values.put(ENTRY_ADDED, Instant.now().toString()); rows.add(values); } - + List> cdcRows = new ArrayList<>(); for (Map row : rows) { - Map cdcRow = new HashMap<>(); - cdcRow.put(ROW_ID, row.get(ROW_ID)); - cdcRow.put(NAME, row.get(NAME)); - cdcRow.put(AGE, row.get(AGE)); - cdcRow.put(MEMBER, row.get(MEMBER)); - cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); - cdcRows.add(cdcRow); + Map cdcRow = new HashMap<>(); + cdcRow.put(ROW_ID, row.get(ROW_ID)); + cdcRow.put(NAME, row.get(NAME)); + cdcRow.put(AGE, row.get(AGE)); + cdcRow.put(MEMBER, row.get(MEMBER)); + cdcRow.put(ENTRY_ADDED, row.get(ENTRY_ADDED)); + cdcRows.add(cdcRow); } cdcEvents.put(tableName, cdcRows); - + success &= cloudSqlResourceManager.write(tableName, rows); messages.add(String.format("%d rows to %s", rows.size(), tableName)); } @@ -437,11 +499,11 @@ protected CheckResult check() { for (int i = 0; i < NUM_EVENTS; i++) { if (i % 2 == 0) { Map values = cdcEvents.get(tableName).get(i); - + String newName = values.get(NAME).toString().toUpperCase(); int newAge = new Random().nextInt(100); String newMember = Objects.equals(values.get(MEMBER).toString(), "Y") ? "N" : "Y"; - + values.put(NAME, newName); values.put(AGE, newAge); values.put(MEMBER, newMember); @@ -449,13 +511,21 @@ protected CheckResult check() { String updateSql = "UPDATE " + tableName - + " SET " + NAME + " = '" + + " SET " + + NAME + + " = '" + newName - + "', " + AGE + " = " + + "', " + + AGE + + " = " + newAge - + ", " + MEMBER + " = '" + + ", " + + MEMBER + + " = '" + newMember - + "' WHERE " + ROW_ID + " = " + + "' WHERE " + + ROW_ID + + " = " + i; cloudSqlResourceManager.runSQLUpdate(updateSql); newCdcEvents.add(values); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index a13bc64745..cadae8f1dc 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -49,7 +63,7 @@ public class OracleDataStreamToSpannerMixedIT extends DataStreamToSpannerITBase private String gcsPrefix; private SubscriptionName subscription; private SubscriptionName dlqSubscription; - + private CloudOracleResourceManager cloudOracleSysUser; private CloudOracleResourceManager cloudSqlResourceManager; private DatastreamResourceManager datastreamResourceManager; @@ -84,49 +98,78 @@ public void cleanUp() { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Test public void mixedMigrationTest() throws IOException { spannerResourceManager = setUpSpannerResourceManager(); - createSpannerDDL(spannerResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql"); + createSpannerDDL( + spannerResourceManager, + "oracle/OracleDataStreamToSpannerMixedIT/oracle-google_standard_sql-spanner-schema.sql"); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); @@ -137,15 +180,18 @@ public void mixedMigrationTest() throws IOException { String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); - cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - executeSqlScript(cloudSqlResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql"); + executeSqlScript( + cloudSqlResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql"); cloudSqlResourceManager.runSQLUpdate( String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Authors")); cloudSqlResourceManager.runSQLUpdate( @@ -170,10 +216,16 @@ public void mixedMigrationTest() throws IOException { datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), gcsPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + "gcs-profile", + gcsResourceManager.getBucket(), + gcsPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); Stream stream = - datastreamResourceManager.createStream("stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + datastreamResourceManager.createStream( + "stream" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); datastreamResourceManager.startStream(stream); pubsubResourceManager = @@ -184,7 +236,7 @@ public void mixedMigrationTest() throws IOException { TopicName dlqTopic = pubsubResourceManager.createTopic("it-dlq"); dlqSubscription = pubsubResourceManager.createSubscription(dlqTopic, "it-dlq-sub"); - + Map jobParams = new HashMap<>(); jobParams.put("inputFileFormat", "avro"); jobParams.put("gcsPubSubSubscription", subscription.toString()); @@ -198,21 +250,22 @@ public void mixedMigrationTest() throws IOException { jobParams.put("datastreamSourceType", "oracle"); PipelineLauncher.LaunchConfig.Builder options = - PipelineLauncher.LaunchConfig.builder(PipelineUtils.createJobName(testName), specPath) + PipelineLauncher.LaunchConfig.builder(PipelineUtils.createJobName(testName), specPath) .setParameters(jobParams) .addEnvironment("ipConfiguration", "WORKER_IP_PRIVATE"); - PipelineLauncher.LaunchInfo jobInfo = launchDataflowJob( - testName, - "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json", - null, - testName, - spannerResourceManager, - pubsubResourceManager, - jobParams, - null, - null, - gcsResourceManager); + PipelineLauncher.LaunchInfo jobInfo = + launchDataflowJob( + testName, + "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json", + null, + testName, + spannerResourceManager, + pubsubResourceManager, + jobParams, + null, + null, + gcsResourceManager); assertThatPipeline(jobInfo).isRunning(); @@ -221,14 +274,19 @@ public void mixedMigrationTest() throws IOException { ChainedConditionCheck.builder( List.of( writeInitialData(), - SpannerRowsCheck.builder(spannerResourceManager, "Authors").setMinRows(4).build(), - SpannerRowsCheck.builder(spannerResourceManager, "Books").setMinRows(3).build())) + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(3) + .build())) .build(); PipelineOperator.Result result = pipelineOperator() .waitForConditionAndCancel( - createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES + 10)), conditionCheck); + createConfig(jobInfo, Duration.ofMinutes(JOB_START_PROCESSING_WAIT_MINUTES + 10)), + conditionCheck); assertThatResult(result).meetsConditions(); @@ -250,25 +308,34 @@ protected CheckResult check() { authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); - + List> bookRows = new ArrayList<>(); bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); - + List> genreRows = new ArrayList<>(); genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); for (Map r : authorRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", r.get("author_id"), r.get("full_name"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", + r.get("author_id"), r.get("full_name"))); } for (Map r : bookRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", r.get("id"), r.get("title"), r.get("author_id"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", + r.get("id"), r.get("title"), r.get("author_id"))); } for (Map r : genreRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", r.get("genre_id"), r.get("name"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", + r.get("genre_id"), r.get("name"))); } - + cloudOracleSysUser.runSQLUpdate("COMMIT"); flushOracleRedoLogs(cloudOracleSysUser); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index 8e0082e2fd..710fc6e471 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -1,10 +1,23 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.spanner.Struct; -import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; @@ -16,15 +29,16 @@ import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Arrays; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; import org.apache.beam.it.gcp.datastream.JDBCSource; import org.apache.beam.it.gcp.datastream.OracleSource; @@ -33,7 +47,6 @@ import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.gcp.storage.GcsResourceManager; -import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -83,24 +96,31 @@ public void setUp() throws IOException { pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); oracleResourceManager = setUpOracleResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); try { oracleResourceManager.runSQLUpdate("DROP TABLE \"Category\""); - } catch (Exception e) {} + } catch (Exception e) { + } try { oracleResourceManager.runSQLUpdate("DROP TABLE \"Books\""); - } catch (Exception e) {} + } catch (Exception e) { + } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); // Pre-insert books - oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the Rings', 1)"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and Prejudice', 2)"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers Guide to the Galaxy', 3)"); - + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the Rings', 1)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and Prejudice', 2)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers Guide to the Galaxy', 3)"); + // Pre-insert categories - oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz', CURRENT_TIMESTAMP)"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc', CURRENT_TIMESTAMP)"); flushOracleLogs(); @@ -109,7 +129,7 @@ public void setUp() throws IOException { .setCredentialsProvider(credentialsProvider) .setPrivateConnectivity(System.getProperty("privateConnectivity")) .build(); - + JDBCSource jdbcSource = OracleSource.builder( System.getProperty("hostIp"), @@ -156,7 +176,11 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, gcsResourceManager, datastreamResourceManager, oracleResourceManager); + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + datastreamResourceManager, + oracleResourceManager); } private void flushOracleLogs() { @@ -193,9 +217,12 @@ public void migrationTestWithRenameAndDrops() { assertCategoryTableBackfillContents(); - oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def', CURRENT_TIMESTAMP)"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi', CURRENT_TIMESTAMP)"); - oracleResourceManager.runSQLUpdate("UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi', CURRENT_TIMESTAMP)"); + oracleResourceManager.runSQLUpdate( + "UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); oracleResourceManager.runSQLUpdate("DELETE FROM \"Category\" WHERE \"category_id\"=1"); flushOracleLogs(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index 4f5da9e4f2..53090d6d1a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -1,9 +1,22 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; -import com.google.cloud.spanner.Dialect; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; @@ -41,7 +54,8 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleDataStreamToSpannerTimezoneIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleDataStreamToSpannerTimezoneIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleDataStreamToSpannerTimezoneIT.class); private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDataStreamToSpannerTimezoneIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; @@ -69,9 +83,10 @@ public void setUp() throws IOException, InterruptedException { spannerResourceManager = setUpSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + oracleResourceManager = setUpOracleResourceManager(); - CloudOracleResourceManager.Builder sysBuilder = CloudOracleResourceManager.builder(testName); + CloudOracleResourceManager.Builder sysBuilder = + CloudOracleResourceManager.builder(testName); sysBuilder.setPassword("TestPassword123"); sysBuilder.setHost("" + System.getProperty("hostIp") + ""); sysBuilder.setPort(1521); @@ -105,7 +120,8 @@ public void setUp() throws IOException, InterruptedException { oracleResourceManager.getPassword(), oracleResourceManager.getPort(), oracleResourceManager.getDatabaseName()) - .setAllowedTables(Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("DateData"))) + .setAllowedTables( + Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of("DateData"))) .build(); jobInfo = @@ -137,31 +153,39 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, gcsResourceManager, oracleResourceManager, cloudOracleSysUser, datastreamResourceManager); + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + oracleResourceManager, + cloudOracleSysUser, + datastreamResourceManager); } @Test public void testTimezoneHandling() { - ConditionCheck insertData = new ConditionCheck() { - @Override - protected String getDescription() { - return "Insert data into Oracle"; - } - - @Override - protected CheckResult check() { - try { - oracleResourceManager.runSQLUpdate("INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02 20:00:00.000000')"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03 06:00:00.000000')"); - - flushOracleRedoLogs(cloudOracleSysUser); - return new CheckResult(true, "Data inserted successfully"); - } catch (Exception e) { - LOG.error("Failed to insert data into Oracle", e); - return new CheckResult(false, e.getMessage()); - } - } - }; + ConditionCheck insertData = + new ConditionCheck() { + @Override + protected String getDescription() { + return "Insert data into Oracle"; + } + + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02 20:00:00.000000')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03 06:00:00.000000')"); + + flushOracleRedoLogs(cloudOracleSysUser); + return new CheckResult(true, "Data inserted successfully"); + } catch (Exception e) { + LOG.error("Failed to insert data into Oracle", e); + return new CheckResult(false, e.getMessage()); + } + } + }; ChainedConditionCheck conditionCheck = ChainedConditionCheck.builder( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java index 20206f917b..69c9a20b85 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java @@ -1,4 +1,3 @@ -/* GENERATED BY: Test Automation Skill */ /* * Copyright (C) 2025 Google LLC * @@ -20,9 +19,9 @@ import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; -import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; -import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import java.io.IOException; @@ -38,7 +37,6 @@ import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ChainedConditionCheck; import org.apache.beam.it.conditions.ConditionCheck; -import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.cloudsql.CloudSqlResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; import org.apache.beam.it.gcp.datastream.OracleSource; @@ -50,7 +48,6 @@ import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -72,8 +69,8 @@ public class OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT private static SpannerResourceManager spannerResourceManager; private static PubsubResourceManager pubsubResourceManager; private static GcsResourceManager gcsResourceManager; - private static HashSet testInstances = - new HashSet<>(); + private static HashSet + testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; private static final List TABLE_NAMES = new ArrayList<>(); private static DatastreamResourceManager datastreamResourceManager; @@ -103,9 +100,10 @@ public void setUp() throws IOException { gcsResourceManager = setUpSpannerITGcsResourceManager(); cloudSqlResourceManager = setUpOracleResourceManager(); String sessionContent = generateBaseSchema(); - sessionContent = sessionContent - .replaceAll("SRC_DATABASE", cloudSqlResourceManager.getDatabaseName()) - .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", cloudSqlResourceManager.getDatabaseName()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); for (int i = 1; i <= NUM_TABLES; i++) { sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); } @@ -132,7 +130,8 @@ public void setUp() throws IOException { cloudSqlResourceManager.getHost(), cloudSqlResourceManager.getUsername(), cloudSqlResourceManager.getPassword(), - cloudSqlResourceManager.getPort(), cloudSqlResourceManager.getDatabaseName()) + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) .setAllowedTables( Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), TABLE_NAMES)) .build()); @@ -178,7 +177,14 @@ public void testDataStreamOracleToSpannerForMax16KeyTablesPerDatabase() throws I PipelineOperator.Result result = pipelineOperator() .waitForConditionAndCancel( - PipelineOperator.Config.builder().setJobId(jobInfo.jobId()).setProject(PROJECT).setRegion(REGION).setTimeoutAfter(Duration.ofMinutes(45)).setCheckAfter(Duration.ofSeconds(5)).build(), conditionCheck); + PipelineOperator.Config.builder() + .setJobId(jobInfo.jobId()) + .setProject(PROJECT) + .setRegion(REGION) + .setTimeoutAfter(Duration.ofMinutes(45)) + .setCheckAfter(Duration.ofSeconds(5)) + .build(), + conditionCheck); // Assert assertThatResult(result).meetsConditions(); @@ -228,7 +234,9 @@ public static Map createSessionTemplate() { for (int ci = 1; ci <= NUM_COLUMNS; ci++) { colIds.add("c" + ci); } - Map sessionTemplate = createSessionTemplate(NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); + Map sessionTemplate = + createSessionTemplate( + NUM_TABLES, createColumnDefinitions(colIds), createPrimaryKeys(colIds)); sessionTemplate.put("DatabaseType", "oracle"); return sessionTemplate; } @@ -346,29 +354,31 @@ private ConditionCheck writeJdbcData(Map>> cdcE success &= cloudSqlResourceManager.write(tableName, rows); try { - String dynamicHost = cloudSqlResourceManager.getHost(); - int dynamicPort = cloudSqlResourceManager.getPort(); - - // Set system property because the builder internally relies on it! - System.setProperty("cloudOracleHost", dynamicHost); - - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder("oracle_sys"); - sysBuilder.setUsername("sys as sysdba"); - sysBuilder.setPassword("TestPassword123"); - sysBuilder.setHost(dynamicHost); - sysBuilder.setPort(dynamicPort); - sysBuilder.setDatabaseName("XE"); - - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudOracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); - flushOracleRedoLogs(cloudOracleSysUser); - cloudOracleSysUser.cleanupAll(); + String dynamicHost = cloudSqlResourceManager.getHost(); + int dynamicPort = cloudSqlResourceManager.getPort(); + + // Set system property because the builder internally relies on it! + System.setProperty("cloudOracleHost", dynamicHost); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder("oracle_sys"); + sysBuilder.setUsername("sys as sysdba"); + sysBuilder.setPassword("TestPassword123"); + sysBuilder.setHost(dynamicHost); + sysBuilder.setPort(dynamicPort); + sysBuilder.setDatabaseName("XE"); + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudOracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + flushOracleRedoLogs(cloudOracleSysUser); + cloudOracleSysUser.cleanupAll(); } catch (Throwable e) { - org.slf4j.LoggerFactory.getLogger(OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class).error("FAILED TO EXECUTE SYSDBA SWITCH LOGFILE", e); - e.printStackTrace(); + org.slf4j.LoggerFactory.getLogger( + OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.class) + .error("FAILED TO EXECUTE SYSDBA SWITCH LOGFILE", e); + e.printStackTrace(); } - + messages.add(String.format("%d rows to %s", rows.size(), tableName)); } return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java index 549161ee45..b5bc08bf27 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java @@ -1,4 +1,3 @@ -/* GENERATED BY: Test Automation Skill */ /* * Copyright (C) 2025 Google LLC * @@ -16,14 +15,13 @@ */ package com.google.cloud.teleport.v2.templates.oracle; -import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; -import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; - import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import java.io.IOException; @@ -39,9 +37,7 @@ import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ChainedConditionCheck; import org.apache.beam.it.conditions.ConditionCheck; -import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; -import org.apache.beam.it.gcp.datastream.OracleSource; import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; import org.apache.beam.it.gcp.spanner.SpannerResourceManager; import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; @@ -50,7 +46,6 @@ import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -106,14 +101,18 @@ public void setUp() throws IOException { pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); cloudSqlResourceManager = setUpOracleResourceManager(); - oracleSchema = ("TEST_U" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5)).toUpperCase(); - cloudSqlResourceManager.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s1A", oracleSchema, oracleSchema)); + oracleSchema = + ("TEST_U" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5)) + .toUpperCase(); + cloudSqlResourceManager.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s1A", oracleSchema, oracleSchema)); cloudSqlResourceManager.runSQLUpdate(String.format("GRANT DBA TO %s", oracleSchema)); String sessionContent = generateBaseSchema(); sessionContent = sessionContent.replaceAll("SRC_DATABASE", oracleSchema); - sessionContent = sessionContent.replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + sessionContent = + sessionContent.replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); for (int i = 1; i <= NUM_TABLES; i++) { - sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); + sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); } setupSchema(); jobInfo = @@ -138,9 +137,9 @@ public void setUp() throws IOException { cloudSqlResourceManager.getHost(), cloudSqlResourceManager.getUsername(), cloudSqlResourceManager.getPassword(), - cloudSqlResourceManager.getPort(), cloudSqlResourceManager.getDatabaseName()) - .setAllowedTables( - Map.of(oracleSchema, TABLE_NAMES)) + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables(Map.of(oracleSchema, TABLE_NAMES)) .build()); } } @@ -148,7 +147,8 @@ public void setUp() throws IOException { @After public void cleanUp() throws IOException { - for (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT instance : testInstances) { + for (OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT instance : + testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -163,7 +163,12 @@ private void setupSchema() { TABLE_NAMES.forEach( tableName -> { cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName)); - cloudSqlResourceManager.runSQLUpdate("ALTER TABLE \"" + oracleSchema + "\".\"" + tableName + "\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + cloudSqlResourceManager.runSQLUpdate( + "ALTER TABLE \"" + + oracleSchema + + "\".\"" + + tableName + + "\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); }); createSpannerTables(); } @@ -244,7 +249,11 @@ private static List> createPrimaryKeys(List colIds) private String getJDBCSchema(String tableName) { StringBuilder sb = new StringBuilder(); - sb.append("CREATE TABLE \"").append(oracleSchema).append("\".\"").append(tableName).append("\" ("); + sb.append("CREATE TABLE \"") + .append(oracleSchema) + .append("\".\"") + .append(tableName) + .append("\" ("); for (int i = 1; i <= NUM_COLUMNS; i++) { sb.append("\"").append(COLUMNS.get(i - 1)).append("\" NUMERIC NOT NULL"); if (i != NUM_COLUMNS) { @@ -343,14 +352,28 @@ protected CheckResult check() { } } rows.add(values); - String sql = "INSERT INTO \"" + oracleSchema + "\".\"" + tableName + "\" (" + cols.toString() + ") VALUES (" + vals.toString() + ")"; - try { cloudSqlResourceManager.runSQLUpdate(sql); cloudSqlResourceManager.runSQLUpdate("COMMIT"); } catch (Exception e) { success = false; } + String sql = + "INSERT INTO \"" + + oracleSchema + + "\".\"" + + tableName + + "\" (" + + cols.toString() + + ") VALUES (" + + vals.toString() + + ")"; + try { + cloudSqlResourceManager.runSQLUpdate(sql); + cloudSqlResourceManager.runSQLUpdate("COMMIT"); + } catch (Exception e) { + success = false; + } } cdcEvents.put(tableName, rows); messages.add(String.format("%d rows to %s", rows.size(), tableName)); } flushOracleRedoLogs(cloudSqlResourceManager); - return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); + return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); } }; } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index fc84a1fef3..7bbc856a82 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -17,7 +31,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; @@ -30,6 +43,7 @@ import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.gcp.storage.GcsResourceManager; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -43,52 +57,54 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleDatastreamToSpannerDataTypesIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleDatastreamToSpannerDataTypesIT.class); - private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleDatastreamToSpannerDataTypesIT/oracle-google_standard_sql-spanner-schema.sql"; private static final String TABLE_PREFIX = "T_SPRG_"; - private static final List UNSUPPORTED_TYPE_TABLES = List.of( - "T_SPRG_R27_INTERVAL_Y", - "T_SPRG_R28_INTERVAL_D", - "T_SPRG_R30_LONG_RAW", - "T_SPRG_R34_BFILE", - "T_SPRG_R35_LONG", - "T_SPRG_R36_ROWID", - "T_SPRG_R37_UROWID", - "T_SPRG_R38_BOOLEAN", - "T_SPRG_R39_JSON", - "T_SPRG_R40_XMLTYPE", - "T_SPRG_R41_SDO_GEOMET", - "T_SPRG_R42_SDO_TOPO_G", - "T_SPRG_R43_SDO_GEORAS", - "T_SPRG_R44_VECTOR", - "T_SPRG_R45_ANYDATA", - "T_SPRG_R46_ANYTYPE", - "T_SPRG_R47_ANYDATASET", - "T_SPRG_R48_URITYPE", - "T_SPRG_R49_DBURITYPE", - "T_SPRG_R50_XDBURITYPE", - "T_SPRG_R51_HTTPURITYP", - "T_SPRG_R52_EXPRESSION", - "T_SPRG_R53_ORDAUDIO", - "T_SPRG_R54_ORDIMAGE", - "T_SPRG_R55_ORDVIDEO", - "T_SPRG_R56_ORDDOC", - "T_SPRG_R57_SI_STILLIM", - "T_SPRG_R58_SI_COLOR", - "T_SPRG_R59_SI_AVERAGE", - "T_SPRG_R60_SI_COLORHI", - "T_SPRG_R61_SI_POSITIO", - "T_SPRG_R62_SI_TEXTURE", - "T_SPRG_R63_SI_FEATURE", - "T_SPRG_R64_MLSLABEL", - "T_SPRG_R65_REF", - "T_SPRG_R66_REF_CURSOR", - "T_SPRG_R67_VARRAY", - "T_SPRG_R68_NESTED_TAB", - "T_SPRG_R69_ASSOCIATIV", - "T_SPRG_R70_OBJECT_TYP" - ); + private static final List UNSUPPORTED_TYPE_TABLES = + List.of( + "T_SPRG_R27_INTERVAL_Y", + "T_SPRG_R28_INTERVAL_D", + "T_SPRG_R30_LONG_RAW", + "T_SPRG_R34_BFILE", + "T_SPRG_R35_LONG", + "T_SPRG_R36_ROWID", + "T_SPRG_R37_UROWID", + "T_SPRG_R38_BOOLEAN", + "T_SPRG_R39_JSON", + "T_SPRG_R40_XMLTYPE", + "T_SPRG_R41_SDO_GEOMET", + "T_SPRG_R42_SDO_TOPO_G", + "T_SPRG_R43_SDO_GEORAS", + "T_SPRG_R44_VECTOR", + "T_SPRG_R45_ANYDATA", + "T_SPRG_R46_ANYTYPE", + "T_SPRG_R47_ANYDATASET", + "T_SPRG_R48_URITYPE", + "T_SPRG_R49_DBURITYPE", + "T_SPRG_R50_XDBURITYPE", + "T_SPRG_R51_HTTPURITYP", + "T_SPRG_R52_EXPRESSION", + "T_SPRG_R53_ORDAUDIO", + "T_SPRG_R54_ORDIMAGE", + "T_SPRG_R55_ORDVIDEO", + "T_SPRG_R56_ORDDOC", + "T_SPRG_R57_SI_STILLIM", + "T_SPRG_R58_SI_COLOR", + "T_SPRG_R59_SI_AVERAGE", + "T_SPRG_R60_SI_COLORHI", + "T_SPRG_R61_SI_POSITIO", + "T_SPRG_R62_SI_TEXTURE", + "T_SPRG_R63_SI_FEATURE", + "T_SPRG_R64_MLSLABEL", + "T_SPRG_R65_REF", + "T_SPRG_R66_REF_CURSOR", + "T_SPRG_R67_VARRAY", + "T_SPRG_R68_NESTED_TAB", + "T_SPRG_R69_ASSOCIATIV", + "T_SPRG_R70_OBJECT_TYP"); private static boolean initialized = false; private static CloudOracleResourceManager oracleSysUser; @@ -99,259 +115,281 @@ public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITB private static DatastreamResourceManager datastreamResourceManager; private static HashSet testInstances = new HashSet<>(); - private static final String[] ORACLE_DDL_COMMANDS = new String[] { - "CREATE TABLE T_SPRG_R0_VARCHAR2 (ID NUMBER PRIMARY KEY, COL_0 VARCHAR2(255))", - "ALTER TABLE T_SPRG_R0_VARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R0_VARCHAR2 (ID, COL_0) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R1_VARCHAR (ID NUMBER PRIMARY KEY, COL_1 VARCHAR(255))", - "ALTER TABLE T_SPRG_R1_VARCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R1_VARCHAR (ID, COL_1) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R2_CHAR (ID NUMBER PRIMARY KEY, COL_2 CHAR(255))", - "ALTER TABLE T_SPRG_R2_CHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R2_CHAR (ID, COL_2) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R3_CHARACTER (ID NUMBER PRIMARY KEY, COL_3 CHARACTER(255))", - "ALTER TABLE T_SPRG_R3_CHARACTER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R3_CHARACTER (ID, COL_3) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID NUMBER PRIMARY KEY, COL_4 NVARCHAR2(255))", - "ALTER TABLE T_SPRG_R4_NVARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R4_NVARCHAR2 (ID, COL_4) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R5_NCHAR (ID NUMBER PRIMARY KEY, COL_5 NCHAR(255))", - "ALTER TABLE T_SPRG_R5_NCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R5_NCHAR (ID, COL_5) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID NUMBER PRIMARY KEY, COL_6 NCHAR VARYING(255))", - "ALTER TABLE T_SPRG_R6_NCHAR_VARY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R6_NCHAR_VARY (ID, COL_6) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R7_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_7 NATIONAL CHARACTER(255))", - "ALTER TABLE T_SPRG_R7_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R7_NATIONAL_C (ID, COL_7) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R8_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_8 NATIONAL CHAR(255))", - "ALTER TABLE T_SPRG_R8_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R8_NATIONAL_C (ID, COL_8) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER VARYING(255))", - "ALTER TABLE T_SPRG_R9_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R9_NATIONAL_C (ID, COL_9) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR VARYING(255))", - "ALTER TABLE T_SPRG_R10_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R10_NATIONAL_C (ID, COL_10) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R11_NUMBER (ID NUMBER PRIMARY KEY, COL_11 NUMBER)", - "ALTER TABLE T_SPRG_R11_NUMBER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R11_NUMBER (ID, COL_11) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R12_NUMERIC (ID NUMBER PRIMARY KEY, COL_12 NUMERIC)", - "ALTER TABLE T_SPRG_R12_NUMERIC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R12_NUMERIC (ID, COL_12) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R13_DECIMAL (ID NUMBER PRIMARY KEY, COL_13 DECIMAL)", - "ALTER TABLE T_SPRG_R13_DECIMAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R13_DECIMAL (ID, COL_13) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R14_DEC (ID NUMBER PRIMARY KEY, COL_14 DEC)", - "ALTER TABLE T_SPRG_R14_DEC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R14_DEC (ID, COL_14) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R15_FLOAT (ID NUMBER PRIMARY KEY, COL_15 FLOAT)", - "ALTER TABLE T_SPRG_R15_FLOAT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R15_FLOAT (ID, COL_15) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID NUMBER PRIMARY KEY, COL_16 DOUBLE PRECISION)", - "ALTER TABLE T_SPRG_R16_DOUBLE_PRE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R16_DOUBLE_PRE (ID, COL_16) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R17_REAL (ID NUMBER PRIMARY KEY, COL_17 REAL)", - "ALTER TABLE T_SPRG_R17_REAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R17_REAL (ID, COL_17) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R18_BINARY_FLO (ID NUMBER PRIMARY KEY, COL_18 BINARY_FLOAT)", - "ALTER TABLE T_SPRG_R18_BINARY_FLO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R18_BINARY_FLO (ID, COL_18) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R19_BINARY_DOU (ID NUMBER PRIMARY KEY, COL_19 BINARY_DOUBLE)", - "ALTER TABLE T_SPRG_R19_BINARY_DOU ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R19_BINARY_DOU (ID, COL_19) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R20_INTEGER (ID NUMBER PRIMARY KEY, COL_20 INTEGER)", - "ALTER TABLE T_SPRG_R20_INTEGER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R20_INTEGER (ID, COL_20) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R21_INT (ID NUMBER PRIMARY KEY, COL_21 INT)", - "ALTER TABLE T_SPRG_R21_INT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R21_INT (ID, COL_21) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R22_SMALLINT (ID NUMBER PRIMARY KEY, COL_22 SMALLINT)", - "ALTER TABLE T_SPRG_R22_SMALLINT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R22_SMALLINT (ID, COL_22) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R23_DATE (ID NUMBER PRIMARY KEY, COL_23 DATE)", - "ALTER TABLE T_SPRG_R23_DATE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R23_DATE (ID, COL_23) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R24_TIMESTAMP (ID NUMBER PRIMARY KEY, COL_24 TIMESTAMP)", - "ALTER TABLE T_SPRG_R24_TIMESTAMP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R24_TIMESTAMP (ID, COL_24) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME ZONE)", - "ALTER TABLE T_SPRG_R25_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R25_TIMESTAMP_ (ID, COL_25) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL TIME ZONE)", - "ALTER TABLE T_SPRG_R26_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R26_TIMESTAMP_ (ID, COL_26) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R27_INTERVAL_Y ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R27_INTERVAL_Y (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R28_INTERVAL_D (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R28_INTERVAL_D ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R28_INTERVAL_D (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R29_RAW (ID NUMBER PRIMARY KEY, COL_29 RAW(255))", - "ALTER TABLE T_SPRG_R29_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R29_RAW (ID, COL_29) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R30_LONG_RAW (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R30_LONG_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R30_LONG_RAW (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R31_BLOB (ID NUMBER PRIMARY KEY, COL_31 BLOB)", - "ALTER TABLE T_SPRG_R31_BLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R31_BLOB (ID, COL_31) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R32_CLOB (ID NUMBER PRIMARY KEY, COL_32 CLOB)", - "ALTER TABLE T_SPRG_R32_CLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R32_CLOB (ID, COL_32) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R33_NCLOB (ID NUMBER PRIMARY KEY, COL_33 NCLOB)", - "ALTER TABLE T_SPRG_R33_NCLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R33_NCLOB (ID, COL_33) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R34_BFILE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R34_BFILE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R34_BFILE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R35_LONG (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R35_LONG ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R35_LONG (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R36_ROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R36_ROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R36_ROWID (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R37_UROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R37_UROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R37_UROWID (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R38_BOOLEAN (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R38_BOOLEAN ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R38_BOOLEAN (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R39_JSON (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R39_JSON ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R39_JSON (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R40_XMLTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R40_XMLTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R40_XMLTYPE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R41_SDO_GEOMET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R41_SDO_GEOMET (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R42_SDO_TOPO_G ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R42_SDO_TOPO_G (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R43_SDO_GEORAS ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R43_SDO_GEORAS (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R44_VECTOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R44_VECTOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R44_VECTOR (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R45_ANYDATA (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R45_ANYDATA ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R45_ANYDATA (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R46_ANYTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R46_ANYTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R46_ANYTYPE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R47_ANYDATASET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R47_ANYDATASET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R47_ANYDATASET (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R48_URITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R48_URITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R48_URITYPE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R49_DBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R49_DBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R49_DBURITYPE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R50_XDBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R50_XDBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R50_XDBURITYPE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R51_HTTPURITYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R51_HTTPURITYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R51_HTTPURITYP (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R52_EXPRESSION (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R52_EXPRESSION ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R52_EXPRESSION (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R53_ORDAUDIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R53_ORDAUDIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R53_ORDAUDIO (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R54_ORDIMAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R54_ORDIMAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R54_ORDIMAGE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R55_ORDVIDEO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R55_ORDVIDEO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R55_ORDVIDEO (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R56_ORDDOC (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R56_ORDDOC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R56_ORDDOC (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R57_SI_STILLIM (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R57_SI_STILLIM ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R57_SI_STILLIM (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R58_SI_COLOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R58_SI_COLOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R58_SI_COLOR (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R59_SI_AVERAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R59_SI_AVERAGE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R60_SI_COLORHI (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R60_SI_COLORHI ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R60_SI_COLORHI (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R61_SI_POSITIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R61_SI_POSITIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R61_SI_POSITIO (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R62_SI_TEXTURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R62_SI_TEXTURE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R63_SI_FEATURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R63_SI_FEATURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R63_SI_FEATURE (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R64_MLSLABEL (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R64_MLSLABEL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R64_MLSLABEL (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R65_REF (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R65_REF ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R65_REF (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R66_REF_CURSOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R66_REF_CURSOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R66_REF_CURSOR (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R67_VARRAY (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R67_VARRAY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R67_VARRAY (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R68_NESTED_TAB (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R68_NESTED_TAB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R68_NESTED_TAB (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R69_ASSOCIATIV ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R69_ASSOCIATIV (ID, DUMMY) VALUES (1, 'test')", - "CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", - "ALTER TABLE T_SPRG_R70_OBJECT_TYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", - "INSERT INTO T_SPRG_R70_OBJECT_TYP (ID, DUMMY) VALUES (1, 'test')" - }; + private static final String[] ORACLE_DDL_COMMANDS = + new String[] { + "CREATE TABLE T_SPRG_R0_VARCHAR2 (ID NUMBER PRIMARY KEY, COL_0 VARCHAR2(255))", + "ALTER TABLE T_SPRG_R0_VARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R0_VARCHAR2 (ID, COL_0) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R1_VARCHAR (ID NUMBER PRIMARY KEY, COL_1 VARCHAR(255))", + "ALTER TABLE T_SPRG_R1_VARCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R1_VARCHAR (ID, COL_1) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R2_CHAR (ID NUMBER PRIMARY KEY, COL_2 CHAR(255))", + "ALTER TABLE T_SPRG_R2_CHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R2_CHAR (ID, COL_2) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R3_CHARACTER (ID NUMBER PRIMARY KEY, COL_3 CHARACTER(255))", + "ALTER TABLE T_SPRG_R3_CHARACTER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R3_CHARACTER (ID, COL_3) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R4_NVARCHAR2 (ID NUMBER PRIMARY KEY, COL_4 NVARCHAR2(255))", + "ALTER TABLE T_SPRG_R4_NVARCHAR2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R4_NVARCHAR2 (ID, COL_4) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R5_NCHAR (ID NUMBER PRIMARY KEY, COL_5 NCHAR(255))", + "ALTER TABLE T_SPRG_R5_NCHAR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R5_NCHAR (ID, COL_5) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R6_NCHAR_VARY (ID NUMBER PRIMARY KEY, COL_6 NCHAR VARYING(255))", + "ALTER TABLE T_SPRG_R6_NCHAR_VARY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R6_NCHAR_VARY (ID, COL_6) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R7_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_7 NATIONAL CHARACTER(255))", + "ALTER TABLE T_SPRG_R7_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R7_NATIONAL_C (ID, COL_7) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R8_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_8 NATIONAL CHAR(255))", + "ALTER TABLE T_SPRG_R8_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R8_NATIONAL_C (ID, COL_8) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER VARYING(255))", + "ALTER TABLE T_SPRG_R9_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R9_NATIONAL_C (ID, COL_9) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR VARYING(255))", + "ALTER TABLE T_SPRG_R10_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R10_NATIONAL_C (ID, COL_10) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R11_NUMBER (ID NUMBER PRIMARY KEY, COL_11 NUMBER)", + "ALTER TABLE T_SPRG_R11_NUMBER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R11_NUMBER (ID, COL_11) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R12_NUMERIC (ID NUMBER PRIMARY KEY, COL_12 NUMERIC)", + "ALTER TABLE T_SPRG_R12_NUMERIC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R12_NUMERIC (ID, COL_12) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R13_DECIMAL (ID NUMBER PRIMARY KEY, COL_13 DECIMAL)", + "ALTER TABLE T_SPRG_R13_DECIMAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R13_DECIMAL (ID, COL_13) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R14_DEC (ID NUMBER PRIMARY KEY, COL_14 DEC)", + "ALTER TABLE T_SPRG_R14_DEC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R14_DEC (ID, COL_14) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R15_FLOAT (ID NUMBER PRIMARY KEY, COL_15 FLOAT)", + "ALTER TABLE T_SPRG_R15_FLOAT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R15_FLOAT (ID, COL_15) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R16_DOUBLE_PRE (ID NUMBER PRIMARY KEY, COL_16 DOUBLE PRECISION)", + "ALTER TABLE T_SPRG_R16_DOUBLE_PRE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R16_DOUBLE_PRE (ID, COL_16) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R17_REAL (ID NUMBER PRIMARY KEY, COL_17 REAL)", + "ALTER TABLE T_SPRG_R17_REAL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R17_REAL (ID, COL_17) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R18_BINARY_FLO (ID NUMBER PRIMARY KEY, COL_18 BINARY_FLOAT)", + "ALTER TABLE T_SPRG_R18_BINARY_FLO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R18_BINARY_FLO (ID, COL_18) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R19_BINARY_DOU (ID NUMBER PRIMARY KEY, COL_19 BINARY_DOUBLE)", + "ALTER TABLE T_SPRG_R19_BINARY_DOU ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R19_BINARY_DOU (ID, COL_19) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R20_INTEGER (ID NUMBER PRIMARY KEY, COL_20 INTEGER)", + "ALTER TABLE T_SPRG_R20_INTEGER ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R20_INTEGER (ID, COL_20) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R21_INT (ID NUMBER PRIMARY KEY, COL_21 INT)", + "ALTER TABLE T_SPRG_R21_INT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R21_INT (ID, COL_21) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R22_SMALLINT (ID NUMBER PRIMARY KEY, COL_22 SMALLINT)", + "ALTER TABLE T_SPRG_R22_SMALLINT ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R22_SMALLINT (ID, COL_22) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R23_DATE (ID NUMBER PRIMARY KEY, COL_23 DATE)", + "ALTER TABLE T_SPRG_R23_DATE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R23_DATE (ID, COL_23) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R24_TIMESTAMP (ID NUMBER PRIMARY KEY, COL_24 TIMESTAMP)", + "ALTER TABLE T_SPRG_R24_TIMESTAMP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R24_TIMESTAMP (ID, COL_24) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME ZONE)", + "ALTER TABLE T_SPRG_R25_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R25_TIMESTAMP_ (ID, COL_25) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL TIME ZONE)", + "ALTER TABLE T_SPRG_R26_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R26_TIMESTAMP_ (ID, COL_26) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R27_INTERVAL_Y ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R27_INTERVAL_Y (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R28_INTERVAL_D (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R28_INTERVAL_D ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R28_INTERVAL_D (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R29_RAW (ID NUMBER PRIMARY KEY, COL_29 RAW(255))", + "ALTER TABLE T_SPRG_R29_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R29_RAW (ID, COL_29) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R30_LONG_RAW (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R30_LONG_RAW ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R30_LONG_RAW (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R31_BLOB (ID NUMBER PRIMARY KEY, COL_31 BLOB)", + "ALTER TABLE T_SPRG_R31_BLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R31_BLOB (ID, COL_31) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R32_CLOB (ID NUMBER PRIMARY KEY, COL_32 CLOB)", + "ALTER TABLE T_SPRG_R32_CLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R32_CLOB (ID, COL_32) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R33_NCLOB (ID NUMBER PRIMARY KEY, COL_33 NCLOB)", + "ALTER TABLE T_SPRG_R33_NCLOB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R33_NCLOB (ID, COL_33) VALUES (1, NULL)", + "CREATE TABLE T_SPRG_R34_BFILE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R34_BFILE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R34_BFILE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R35_LONG (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R35_LONG ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R35_LONG (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R36_ROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R36_ROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R36_ROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R37_UROWID (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R37_UROWID ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R37_UROWID (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R38_BOOLEAN (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R38_BOOLEAN ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R38_BOOLEAN (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R39_JSON (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R39_JSON ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R39_JSON (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R40_XMLTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R40_XMLTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R40_XMLTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R41_SDO_GEOMET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R41_SDO_GEOMET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R41_SDO_GEOMET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R42_SDO_TOPO_G (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R42_SDO_TOPO_G ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R42_SDO_TOPO_G (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R43_SDO_GEORAS (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R43_SDO_GEORAS ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R43_SDO_GEORAS (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R44_VECTOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R44_VECTOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R44_VECTOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R45_ANYDATA (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R45_ANYDATA ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R45_ANYDATA (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R46_ANYTYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R46_ANYTYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R46_ANYTYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R47_ANYDATASET (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R47_ANYDATASET ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R47_ANYDATASET (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R48_URITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R48_URITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R48_URITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R49_DBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R49_DBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R49_DBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R50_XDBURITYPE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R50_XDBURITYPE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R50_XDBURITYPE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R51_HTTPURITYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R51_HTTPURITYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R51_HTTPURITYP (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R52_EXPRESSION (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R52_EXPRESSION ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R52_EXPRESSION (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R53_ORDAUDIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R53_ORDAUDIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R53_ORDAUDIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R54_ORDIMAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R54_ORDIMAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R54_ORDIMAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R55_ORDVIDEO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R55_ORDVIDEO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R55_ORDVIDEO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R56_ORDDOC (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R56_ORDDOC ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R56_ORDDOC (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R57_SI_STILLIM (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R57_SI_STILLIM ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R57_SI_STILLIM (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R58_SI_COLOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R58_SI_COLOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R58_SI_COLOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R59_SI_AVERAGE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R59_SI_AVERAGE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R59_SI_AVERAGE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R60_SI_COLORHI (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R60_SI_COLORHI ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R60_SI_COLORHI (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R61_SI_POSITIO (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R61_SI_POSITIO ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R61_SI_POSITIO (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R62_SI_TEXTURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R62_SI_TEXTURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R62_SI_TEXTURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R63_SI_FEATURE (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R63_SI_FEATURE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R63_SI_FEATURE (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R64_MLSLABEL (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R64_MLSLABEL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R64_MLSLABEL (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R65_REF (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R65_REF ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R65_REF (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R66_REF_CURSOR (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R66_REF_CURSOR ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R66_REF_CURSOR (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R67_VARRAY (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R67_VARRAY ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R67_VARRAY (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R68_NESTED_TAB (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R68_NESTED_TAB ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R68_NESTED_TAB (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R69_ASSOCIATIV (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R69_ASSOCIATIV ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R69_ASSOCIATIV (ID, DUMMY) VALUES (1, 'test')", + "CREATE TABLE T_SPRG_R70_OBJECT_TYP (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", + "ALTER TABLE T_SPRG_R70_OBJECT_TYP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", + "INSERT INTO T_SPRG_R70_OBJECT_TYP (ID, DUMMY) VALUES (1, 'test')" + }; private void setUpOracleUser(String user, String password) { oracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @@ -362,7 +400,7 @@ public void setUp() throws IOException { testInstances.add(this); if (!initialized) { LOG.info("Setting up Oracle sys resource manager..."); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); @@ -371,16 +409,17 @@ public void setUp() throws IOException { sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); } - oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); - + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); - + LOG.info("Provisioning isolated user: " + oracleUser); setUpOracleUser(oracleUser, oraclePassword); - + // Build isolated RM - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { builder.setPassword(oraclePassword); @@ -389,17 +428,18 @@ public void setUp() throws IOException { builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); } - oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); - + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + LOG.info("Setting up Spanner resource manager..."); spannerResourceManager = setUpSpannerResourceManager(); - + LOG.info("Setting up GCS resource manager..."); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + LOG.info("Setting up Pub/Sub resource manager..."); pubsubResourceManager = setUpPubSubResourceManager(); - + LOG.info("Setting up Datastream resource manager..."); datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) @@ -408,14 +448,18 @@ public void setUp() throws IOException { .build(); LOG.info("Executing Oracle DDL commands efficiently using a single JDBC connection..."); - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - oracleResourceManager.getUri(), oracleResourceManager.getUsername(), oracleResourceManager.getPassword()); - java.sql.Statement stmt = conn.createStatement()) { - for (String cmd : ORACLE_DDL_COMMANDS) { - stmt.execute(cmd); - } + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + java.sql.Statement stmt = conn.createStatement()) { + for (String cmd : ORACLE_DDL_COMMANDS) { + stmt.execute(cmd); + } } catch (Exception e) { - throw new RuntimeException("Failed to efficiently upload Oracle DDL via single connection", e); + throw new RuntimeException( + "Failed to efficiently upload Oracle DDL via single connection", e); } initialized = true; @@ -451,7 +495,9 @@ public void testOracleDataTypes() throws Exception { oracleResourceManager.getPort(), oracleResourceManager.getDatabaseName()) .setAllowedTables( - Map.of(oracleResourceManager.getUsername().toUpperCase(), getAllowedTables(expectedData))) + Map.of( + oracleResourceManager.getUsername().toUpperCase(), + getAllowedTables(expectedData))) .build(); LOG.info("Launching Dataflow job..."); @@ -502,10 +548,10 @@ private List> createRows(String colName, Object... values) { List vals = Arrays.asList(values); List> rows = new ArrayList<>(); for (int i = 0; i < vals.size(); i++) { - Map row = new HashMap<>(); - row.put("ID", i + 1L); - row.put(colName, vals.get(i)); - rows.add(row); + Map row = new HashMap<>(); + row.put("ID", i + 1L); + row.put(colName, vals.get(i)); + rows.add(row); } return rows; } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index 9f44934ec7..98dc215b77 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -18,7 +32,6 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.conditions.ChainedConditionCheck; import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; @@ -37,7 +50,8 @@ import org.slf4j.LoggerFactory; /** - * Integration test for sharded data migration using a single Dataflow job for Oracle Source targeting Spanner. + * Integration test for sharded data migration using a single Dataflow job for Oracle Source + * targeting Spanner. */ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @@ -53,7 +67,7 @@ public class OracleDatastreamToSpannerSingleDFShardedMigrationIT extends DataStr private static final String SPANNER_DDL_RESOURCE = "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-google_standard_sql-spanner-schema.sql"; - + private static final String ORACLE_SCHEMA_FILE_RESOURCE = "oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT/oracle-schema.sql"; @@ -85,39 +99,39 @@ public void setUp() throws IOException, InterruptedException { } if (jobInfo == null) { - + String oracleUser = System.getProperty("cloudProxyUsername", "system"); String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); - jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} + jdbcResourceManagerShardA = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); + } catch (Exception e) { + } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - - jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - + + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + datastreamResourceManager = - org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder(testName, PROJECT, REGION) + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder( + testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) .setPrivateConnectivity("datastream-connect-2") .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("hostIp"), - oracleUser, - oraclePassword, - 1521, - "XEPDB1") - .setAllowedTables( - java.util.Map.of( - oracleUser, - java.util.List.of("Users"))) + System.getProperty("hostIp"), oracleUser, oraclePassword, 1521, "XEPDB1") + .setAllowedTables(java.util.Map.of(oracleUser, java.util.List.of("Users"))) .build(); com.google.cloud.datastream.v1.SourceConfig sourceConfig = @@ -125,41 +139,49 @@ public void setUp() throws IOException, InterruptedException { com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), "oracle-shard-cdc/cdc/", org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + "gcs-profile", + gcsResourceManager.getBucket(), + "oracle-shard-cdc/cdc/", + org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat + .AVRO_FILE_FORMAT); com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( - "test_stream_"+ org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + "test_stream_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5) + .toLowerCase(), sourceConfig, destinationConfig); datastreamResourceManager.startStream(stream); streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); - String shardConfig = - generateSourceConfig( - streamNameA, - oracleUser, - "L1" - ); + String shardConfig = generateSourceConfig(streamNameA, oracleUser, "L1"); gcsResourceManager.createArtifact( - "input/shardingConfig.conf", shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); - + "input/shardingConfig.conf", + shardConfig.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + Map jobParams = new HashMap<>(); jobParams.put("inputFileFormat", "avro"); - jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); jobParams.put("datastreamSourceType", "oracle"); - jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - jobParams.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); } String sessionFileContent = - com.google.common.io.Resources.toString(com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), java.nio.charset.StandardCharsets.UTF_8); - sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + com.google.common.io.Resources.toString( + com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); jobInfo = launchDataflowJob( getClass().getSimpleName() + "shard1", @@ -185,12 +207,16 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager); + spannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager); } @Test public void multiShardMigration() throws Exception { - + // Check if pipeline is running assertThatPipeline(jobInfo).isRunning(); @@ -202,11 +228,11 @@ public void multiShardMigration() throws Exception { .setMinRows(12) .setMaxRows(12) .build(); - + PipelineOperator.Result result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); - + assertThatResult(result).meetsConditions(); // Sleep for cutover time to wait till all CDCs propagate. @@ -214,35 +240,54 @@ public void multiShardMigration() throws Exception { Thread.sleep(CUTOVER_MILLIS); } catch (InterruptedException e) { } - + // Assert specific rows assertUsersTableContents(); } private void insertDataInOracle() { LOG.info("Inserting rows into Users table in Oracle"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); LOG.info("Wait, executing Hard-boot raw JDBC switch logfile..."); - // Force log file archive - needed so Datastream can see changes which are read from archived log files. - // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + System.getProperty("hostIp") + ":1521 (User: system, Pass: TestPassword123) calling ALTER SYSTEM SWITCH LOGFILE. - try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XEPDB1", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + // Force log file archive - needed so Datastream can see changes which are read from archived + // log files. + // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + + // System.getProperty("hostIp") + ":1521 (User: system, Pass: TestPassword123) calling ALTER + // SYSTEM SWITCH LOGFILE. + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } catch (Exception e) { - LOG.warn("Error while executing ALTER SYSTEM SWITCH LOGFILE. Using framework fallback...", e); - flushOracleRedoLogs(jdbcResourceManagerShardA); + LOG.warn("Error while executing ALTER SYSTEM SWITCH LOGFILE. Using framework fallback...", e); + flushOracleRedoLogs(jdbcResourceManagerShardA); } } @@ -337,14 +382,19 @@ private void assertUsersTableContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private String generateSourceConfig( - String streamA, String dbA, String shardA) { + private String generateSourceConfig(String streamA, String dbA, String shardA) { return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java index 2aee8ce1a1..4ba158d4c0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static java.util.Map.entry; @@ -11,16 +25,16 @@ import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; -import java.time.Duration; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.Statement; +import java.time.Duration; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Arrays; -import java.nio.charset.StandardCharsets; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; @@ -34,7 +48,6 @@ import org.apache.beam.it.gcp.storage.GcsResourceManager; import org.junit.AfterClass; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -59,8 +72,7 @@ public class OracleDatastreamToSpannerTableAndIndexLimitsIT extends DataStreamTo private static final String LARGE_KEY_TABLE = "LargeKey"; private static final String LARGE_CELL_TABLE = "LargeCell"; - private static final List TABLES = - List.of(LARGE_KEY_TABLE, LARGE_CELL_TABLE); + private static final List TABLES = List.of(LARGE_KEY_TABLE, LARGE_CELL_TABLE); private static CloudOracleResourceManager oracleResourceManager; private static SpannerResourceManager spannerResourceManager; @@ -80,9 +92,15 @@ public void setUp() throws IOException { LOG.info("Setting up Oracle resource manager..."); oracleResourceManager = setUpOracleResourceManager(); LOG.info("Oracle resource manager created with URI: {}", oracleResourceManager.getUri()); - - try { oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeKey\" CASCADE CONSTRAINTS"); } catch (Exception e) {} - try { oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeCell\" CASCADE CONSTRAINTS"); } catch (Exception e) {} + + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeKey\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + } + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeCell\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + } LOG.info("Setting up Spanner resource manager..."); spannerResourceManager = setUpSpannerResourceManager(); @@ -105,40 +123,56 @@ public void setUp() throws IOException { LOG.info("Executing Oracle DDL script..."); executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - + LOG.info("Adding supplemental logging to tables..."); - try { oracleResourceManager.runSQLUpdate("ALTER TABLE \"LargeKey\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} - try { oracleResourceManager.runSQLUpdate("ALTER TABLE \"LargeCell\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} + try { + oracleResourceManager.runSQLUpdate( + "ALTER TABLE \"LargeKey\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + } catch (Exception e) { + } + try { + oracleResourceManager.runSQLUpdate( + "ALTER TABLE \"LargeCell\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + } catch (Exception e) { + } LOG.info("Creating Spanner DDL..."); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); // Pre-insert testing data for LargeCell - try (Connection conn = DriverManager.getConnection( - oracleResourceManager.getUri(), - oracleResourceManager.getUsername(), - oracleResourceManager.getPassword()); - PreparedStatement pstmt = conn.prepareStatement("INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\", \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { - - byte[] bytes = new byte[4000]; - Arrays.fill(bytes, (byte)'b'); - pstmt.setBytes(1, bytes); - pstmt.setString(2, new String(bytes, StandardCharsets.UTF_8)); - pstmt.executeUpdate(); + try (Connection conn = + DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + PreparedStatement pstmt = + conn.prepareStatement( + "INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\", \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { + + byte[] bytes = new byte[4000]; + Arrays.fill(bytes, (byte) 'b'); + pstmt.setBytes(1, bytes); + pstmt.setString(2, new String(bytes, StandardCharsets.UTF_8)); + pstmt.executeUpdate(); } catch (Exception e) { - LOG.error("Failed to insert LargeCell locally: ", e); - throw new RuntimeException("Failed to insert LargeCell", e); + LOG.error("Failed to insert LargeCell locally: ", e); + throw new RuntimeException("Failed to insert LargeCell", e); } // Force a log switch and insert to LargeCell. - try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@" + oracleResourceManager.getHost() + ":" + oracleResourceManager.getPort() + "/XEPDB1", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement(); ) { - flushOracleRedoLogs(null); + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@" + + oracleResourceManager.getHost() + + ":" + + oracleResourceManager.getPort() + + "/XEPDB1", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement(); ) { + flushOracleRedoLogs(null); } catch (Exception e) { - LOG.warn("Failed to switch log file natively: ", e); + LOG.warn("Failed to switch log file natively: ", e); } LOG.info("Generating session file content..."); @@ -209,9 +243,7 @@ public void testKeySize() { List rows = null; try { rows = - spannerResourceManager.readTableRecords( - LARGE_KEY_TABLE, - List.of("pk_col1", "pk_col2")); + spannerResourceManager.readTableRecords(LARGE_KEY_TABLE, List.of("pk_col1", "pk_col2")); } catch (Exception e) { LOG.error("Exception while reading spanner rows from {}", LARGE_KEY_TABLE, e); throw e; @@ -247,6 +279,6 @@ public void testCellSize() { throw e; } SpannerAsserts.assertThatStructs(rows) - .hasRecordsUnorderedCaseInsensitiveColumns(List.of(Map.ofEntries(entry("id", 1)))); + .hasRecordsUnorderedCaseInsensitiveColumns(List.of(Map.ofEntries(entry("id", 1)))); } } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java index f783d6179d..352b3363ff 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -67,14 +81,15 @@ public class OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT private static PubsubResourceManager pubsubResourceManager; private static GcsResourceManager gcsResourceManager; private static DatastreamResourceManager datastreamResourceManager; - private static HashSet testInstances = - new HashSet<>(); + private static HashSet + testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; private static final List TABLE_NAMES = new ArrayList<>(); static { for (int i = 1; i <= NUM_TABLES; i++) { - TABLE_NAMES.add("DATASTREAM_WIDETB_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); + TABLE_NAMES.add( + "DATASTREAM_WIDETB_" + i + "_" + RandomStringUtils.randomAlphanumeric(5).toUpperCase()); } } @@ -94,9 +109,10 @@ public void setUp() throws IOException { gcsResourceManager = setUpSpannerITGcsResourceManager(); oracleResourceManager = setUpOracleResourceManager(); String sessionContent = generateBaseSchema(); - sessionContent = sessionContent - .replaceAll("SRC_DATABASE", oracleResourceManager.getUsername().toUpperCase()) - .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); + sessionContent = + sessionContent + .replaceAll("SRC_DATABASE", oracleResourceManager.getUsername().toUpperCase()) + .replaceAll("SP_DATABASE", spannerResourceManager.getDatabaseId()); for (int i = 1; i <= NUM_TABLES; i++) { sessionContent = sessionContent.replaceAll("TABLE" + i, TABLE_NAMES.get(i - 1)); } @@ -204,7 +220,8 @@ private Map createSessionTemplate() { } List> primaryKeys = List.of(Map.of("ColId", colIds.get(0), "Desc", false, "Order", 1)); - Map sessionTemplate = createSessionTemplate(NUM_TABLES, colTypeConfigs, primaryKeys); + Map sessionTemplate = + createSessionTemplate(NUM_TABLES, colTypeConfigs, primaryKeys); sessionTemplate.put("DatabaseType", "oracle"); return sessionTemplate; } @@ -296,10 +313,16 @@ protected CheckResult check() { rows.add(values); } - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - oracleResourceManager.getUri(), oracleResourceManager.getUsername(), oracleResourceManager.getPassword()); - java.sql.PreparedStatement pstmt = conn.prepareStatement( - "INSERT INTO " + tableName + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?, ?, ?, ?)")) { + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + oracleResourceManager.getUri(), + oracleResourceManager.getUsername(), + oracleResourceManager.getPassword()); + java.sql.PreparedStatement pstmt = + conn.prepareStatement( + "INSERT INTO " + + tableName + + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?, ?, ?, ?)")) { for (Map values : rows) { pstmt.setLong(1, ((Number) values.get(COLUMNS.get(0))).longValue()); pstmt.setString(2, (String) values.get(COLUMNS.get(1))); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java index 0c16372671..0253ebef33 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -18,20 +18,19 @@ import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; -import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; -import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; +import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.util.Map; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; @@ -59,9 +58,11 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleSeparateShadowTableDatabaseDDLIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseDDLIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseDDLIT.class); - private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; private static final String TABLE1 = "AllDatatypeColumns"; private static final String TABLE2 = "AllDatatypeColumns2"; @@ -82,34 +83,38 @@ public class OracleSeparateShadowTableDatabaseDDLIT extends DataStreamToSpannerI public static SpannerResourceManager shadowSpannerResourceManager; public static GcsResourceManager gcsResourceManager; public static org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager oracleResourceManager; - public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; + public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager + datastreamResourceManager; - private void flushOracleLogs() { - try { - Class.forName("oracle.jdbc.OracleDriver"); - try (Connection conn = DriverManager.getConnection("jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - } - } catch (Exception e) { - throw new RuntimeException("Failed to switch logfile", e); + try { + Class.forName("oracle.jdbc.OracleDriver"); + try (Connection conn = + DriverManager.getConnection( + "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", + "system", + "TestPassword123"); + Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } + } catch (Exception e) { + throw new RuntimeException("Failed to switch logfile", e); + } } @Before - public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; synchronized (OracleSeparateShadowTableDatabaseDDLIT.class) { testInstances.add(this); if (jobInfo == null) { oracleResourceManager = setUpOracleResourceManager(); - + datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); spannerResourceManager = setUpSpannerResourceManager(); @@ -122,22 +127,34 @@ public void setUp() throws IOException, InterruptedException { CustomTransformation.builder( "customTransformation.jar", "com.custom.CustomTransformationWithShardForLiveIT") .build(); - - String[] tables = {"AllDatatypeColumns", "AllDatatypeColumns2", "DatatypeColumnsWithSizes", "DatatypeColumnsReducedSizes", "Users", "Authors", "AllDatatypeTransformation", "Singers", "Books"}; - for (String table : tables) { - try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); - } catch (Exception e) { - // Ignore if doesn't exist - } - } - - // Also drop sequences - try { - oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); - } catch (Exception e) { - } -executeSqlScript(oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql"); + + String[] tables = { + "AllDatatypeColumns", + "AllDatatypeColumns2", + "DatatypeColumnsWithSizes", + "DatatypeColumnsReducedSizes", + "Users", + "Authors", + "AllDatatypeTransformation", + "Singers", + "Books" + }; + for (String table : tables) { + try { + oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); + } catch (Exception e) { + // Ignore if doesn't exist + } + } + + // Also drop sequences + try { + oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); + } catch (Exception e) { + } + executeSqlScript( + oracleResourceManager, + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql"); flushOracleLogs(); jobInfo = @@ -151,8 +168,12 @@ public void setUp() throws IOException, InterruptedException { new HashMap<>() { { put("inputFileFormat", "avro"); - put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); - put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); } }, customTransformation, // customTransformation @@ -161,12 +182,17 @@ public void setUp() throws IOException, InterruptedException { datastreamResourceManager, null, // sessionResourceContent org.apache.beam.it.gcp.datastream.OracleSource.builder( - oracleResourceManager.getHost(), - oracleResourceManager.getUsername(), // Explicit Datastream logminer user instead of 'system' - oracleResourceManager.getPassword(), // Datastream user password - oracleResourceManager.getPort(), - "XEPDB1") - .setAllowedTables(java.util.Map.of(oracleResourceManager.getUsername().toUpperCase(), java.util.Arrays.asList(TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) + oracleResourceManager.getHost(), + oracleResourceManager + .getUsername(), // Explicit Datastream logminer user instead of 'system' + oracleResourceManager.getPassword(), // Datastream user password + oracleResourceManager.getPort(), + "XEPDB1") + .setAllowedTables( + java.util.Map.of( + oracleResourceManager.getUsername().toUpperCase(), + java.util.Arrays.asList( + TABLE1, TABLE2, TABLE3, TABLE4, TABLE5, TABLE6, TABLE7, TABLE8))) .build()); } } @@ -183,7 +209,10 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, shadowSpannerResourceManager, gcsResourceManager); + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager); } @Test @@ -193,13 +222,19 @@ public void migrationTestWithAllDatatypeConversionMapping() { List.of( new ConditionCheck() { boolean executed = false; + @Override - protected String getDescription() { return "Run SQL inserts"; } + protected String getDescription() { + return "Run SQL inserts"; + } + @Override protected CheckResult check() { if (!executed) { try { - executeSqlScript(oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql"); + executeSqlScript( + oracleResourceManager, + "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql"); executed = true; // Repositioned ABOVE flushOracleLogs flushOracleLogs(); } catch (Exception e) { @@ -233,7 +268,8 @@ private void assertAllDatatypeColumnsTableBackfillContents() { List> events = new ArrayList<>(); Map row = new HashMap<>(); row.put("varchar_column", "value1"); - // Only asserting primary key + simple int column to verify end-to-end sync without complex base64 logic + // Only asserting primary key + simple int column to verify end-to-end sync without complex + // base64 logic row.put("int_column", "50000"); events.add(row); @@ -243,7 +279,6 @@ private void assertAllDatatypeColumnsTableBackfillContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private void assertAllDatatypeColumnsTableCdcContents() { List> events = new ArrayList<>(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index a91e16610b..b353f8ef8d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; @@ -47,10 +61,10 @@ public class OracleSeparateShadowTableDatabaseEventsIT extends DataStreamToSpann private static final String TABLE1 = "Users"; private static final String TABLE2 = "Movie"; - + private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; - + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; @@ -73,7 +87,7 @@ public void setUp() throws IOException { .setCredentialsProvider(credentialsProvider) .setPrivateConnectivity(System.getProperty("privateConnectivity")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); @@ -81,7 +95,7 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); // SYSTEM AUTHORIZATIONS - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -94,15 +108,19 @@ public void setUp() throws IOException { String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); - cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql"); + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseEventsIT/oracle-schema.sql"); OracleSource jdbcSource = OracleSource.builder( @@ -127,8 +145,12 @@ public void setUp() throws IOException { pubsubResourceManager, new HashMap<>() { { - put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); - put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); put("inputFileFormat", "avro"); put("datastreamSourceType", "oracle"); } @@ -144,41 +166,70 @@ public void setUp() throws IOException { } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - + // Supplement logging requirement cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @@ -217,9 +268,7 @@ public void migrationTestWithUpdatesAndDeletes() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); @@ -244,9 +293,7 @@ public void migrationTestWithInsertsOnly() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); assertMovieTableContents(); @@ -286,13 +333,19 @@ public void interleavedAndFKAndIndexTest() { private ConditionCheck writeUsersInitialData() { return new ConditionCheck() { boolean executed = false; + @Override - protected String getDescription() { return "Write initial Users data"; } + protected String getDescription() { + return "Write initial Users data"; + } + @Override protected CheckResult check() { if (!executed) { - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -304,12 +357,17 @@ protected CheckResult check() { private ConditionCheck writeUsersNextData() { return new ConditionCheck() { boolean executed = false; + @Override - protected String getDescription() { return "Write next Users data"; } + protected String getDescription() { + return "Write next Users data"; + } + @Override protected CheckResult check() { if (!executed) { - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -321,13 +379,19 @@ protected CheckResult check() { private ConditionCheck writeMovieInitialData() { return new ConditionCheck() { boolean executed = false; + @Override - protected String getDescription() { return "Write Movie data"; } + protected String getDescription() { + return "Write Movie data"; + } + @Override protected CheckResult check() { if (!executed) { - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1', TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2', TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1', TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2', TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -339,25 +403,41 @@ protected CheckResult check() { private ConditionCheck writeArticlesInitialData() { return new ConditionCheck() { boolean executed = false; + @Override - protected String getDescription() { return "Write Articles data"; } + protected String getDescription() { + return "Write Articles data"; + } + @Override protected CheckResult check() { if (!executed) { - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (1, 'a1')"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (2, 'a2')"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (3, 'a3')"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (4, 'a4')"); - - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); - - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (1, 'Book005', 3)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (2, 'Book002', 3)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (3, 'Book004', 4)"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (4, 'Book005', 2)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (1, 'a1')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (2, 'a2')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (3, 'a3')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (4, 'a4')"); + + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); + + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (1, 'Book005', 3)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (2, 'Book002', 3)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (3, 'Book004', 4)"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (4, 'Book005', 2)"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -368,18 +448,74 @@ protected CheckResult check() { private void assertUsersTableContents() { List> events = new ArrayList<>(); - events.add(Map.of("id", 1, "name", "Tester Kumar", "age", 30, "subscribed", false, "plan", "A", "startDate", Date.parseDate("2023-01-01"))); - events.add(Map.of("id", 3, "name", "Tester Gupta", "age", 50, "subscribed", false, "plan", "Z", "startDate", Date.parseDate("2023-06-07"))); - events.add(Map.of("id", 4, "name", "Tester", "age", 38, "subscribed", true, "plan", "D", "startDate", Date.parseDate("2023-09-10"))); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) + events.add( + Map.of( + "id", + 1, + "name", + "Tester Kumar", + "age", + 30, + "subscribed", + false, + "plan", + "A", + "startDate", + Date.parseDate("2023-01-01"))); + events.add( + Map.of( + "id", + 3, + "name", + "Tester Gupta", + "age", + 50, + "subscribed", + false, + "plan", + "Z", + "startDate", + Date.parseDate("2023-06-07"))); + events.add( + Map.of( + "id", + 4, + "name", + "Tester", + "age", + 38, + "subscribed", + true, + "plan", + "D", + "startDate", + Date.parseDate("2023-09-10"))); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Users where id in (1, 3, 4)")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } private void assertMovieTableContents() { List> events = new ArrayList<>(); - events.add(Map.of("id", 1, "name", "movie1", "startTime", Timestamp.parseTimestamp("2023-01-01T12:12:12Z"))); - events.add(Map.of("id", 2, "name", "movie2", "startTime", Timestamp.parseTimestamp("2023-11-25T17:10:12Z"))); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select id, name, startTime from Movie where id in (1, 2)")) + events.add( + Map.of( + "id", + 1, + "name", + "movie1", + "startTime", + Timestamp.parseTimestamp("2023-01-01T12:12:12Z"))); + events.add( + Map.of( + "id", + 2, + "name", + "movie2", + "startTime", + Timestamp.parseTimestamp("2023-11-25T17:10:12Z"))); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select id, name, startTime from Movie where id in (1, 2)")) .hasRecordsUnorderedCaseInsensitiveColumns(events); ImmutableList numericVals = @@ -404,17 +540,55 @@ private void assertBooksTable() { events.add(Map.of("id", 2, "title", "Book002", "author_id", 3)); events.add(Map.of("id", 3, "title", "Book004", "author_id", 4)); events.add(Map.of("id", 4, "title", "Book005", "author_id", 2)); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Books@{FORCE_INDEX=author_id_6}")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } private void assertArticlesTable() { List> events = new ArrayList<>(); - events.add(Map.of("id", 1, "name", "Article001", "published_date", Date.parseDate("2024-01-01"), "author_id", 1)); - events.add(Map.of("id", 2, "name", "Article002", "published_date", Date.parseDate("2024-01-01"), "author_id", 1)); - events.add(Map.of("id", 3, "name", "Article004", "published_date", Date.parseDate("2024-01-01"), "author_id", 4)); - events.add(Map.of("id", 4, "name", "Article005", "published_date", Date.parseDate("2024-01-01"), "author_id", 3)); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) + events.add( + Map.of( + "id", + 1, + "name", + "Article001", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 1)); + events.add( + Map.of( + "id", + 2, + "name", + "Article002", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 1)); + events.add( + Map.of( + "id", + 3, + "name", + "Article004", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 4)); + events.add( + Map.of( + "id", + 4, + "name", + "Article005", + "published_date", + Date.parseDate("2024-01-01"), + "author_id", + 3)); + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select * from Articles@{FORCE_INDEX=author_id}")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index ffc78ea637..b5a53c6adc 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -43,11 +57,15 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleSeparateShadowTableDatabaseFileOverridesIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseFileOverridesIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseFileOverridesIT.class); - private static final String ORACLE_SCHEMA = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql"; - private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; - private static final String OVERRIDE_FILE = "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json"; + private static final String ORACLE_SCHEMA = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String OVERRIDE_FILE = + "oracle/OracleSeparateShadowTableDatabaseFileOverridesIT/override.json"; private static final String GCS_PATH_PREFIX = "OracleFileOverridesIT"; private static final String ORACLE_TABLE = "person1"; private static final String SPANNER_TABLE = "human1"; @@ -61,42 +79,67 @@ public class OracleSeparateShadowTableDatabaseFileOverridesIT extends DataStream private static DatastreamResourceManager datastreamResourceManager; private static boolean initialized = false; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = + new HashSet<>(); private void setUpOracleUser(String user, String password) { - oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @@ -107,7 +150,7 @@ public void setUp() throws IOException { testInstances.add(this); if (!initialized) { LOG.info("Setting up Oracle sys resource manager..."); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); @@ -116,15 +159,16 @@ public void setUp() throws IOException { sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); } - oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); - + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); - + LOG.info("Provisioning isolated user: " + oracleUser); setUpOracleUser(oracleUser, oraclePassword); - - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { builder.setPassword(oraclePassword); @@ -133,8 +177,9 @@ public void setUp() throws IOException { builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); } - oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); - + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); @@ -147,7 +192,7 @@ public void setUp() throws IOException { executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - + gcsResourceManager.uploadArtifact( GCS_PATH_PREFIX + "/override.json", Resources.getResource(OVERRIDE_FILE).getPath()); @@ -206,34 +251,41 @@ public void migrationTestWithRenameTableAndColumns() throws Exception { datastreamResourceManager, null, oracleSource); - - assertThatPipeline(jobInfo).isRunning(); - ConditionCheck sendDataCondition = new ConditionCheck() { - @Override - public String getDescription() { - return "Insert data into Oracle and flush logs"; - } + assertThatPipeline(jobInfo).isRunning(); - @Override - protected CheckResult check() { - try { - oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); - oracleResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); - - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XE", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + ConditionCheck sendDataCondition = + new ConditionCheck() { + @Override + public String getDescription() { + return "Insert data into Oracle and flush logs"; } - return new CheckResult(true, "Data inserted and logs flushed"); - } catch (Exception e) { - LOG.error("Failed to insert data or flush logs", e); - return new CheckResult(false, e.getMessage()); - } - } - }; + @Override + protected CheckResult check() { + try { + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + oracleResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); + + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + + System.getProperty("hostIp", "localhost") + + ":1521/XE", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); + } + return new CheckResult(true, "Data inserted and logs flushed"); + } catch (Exception e) { + LOG.error("Failed to insert data or flush logs", e); + return new CheckResult(false, e.getMessage()); + } + } + }; ChainedConditionCheck conditionCheck = ChainedConditionCheck.builder( @@ -247,9 +299,7 @@ protected CheckResult check() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); assertHumanTableContents(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index e98e65bf67..b3e18a6549 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; @@ -38,12 +53,12 @@ @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleSeparateShadowTableDatabaseMixedIT extends DataStreamToSpannerITBase { - private static final String SESSION_FILE_RESOURCE = "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json"; - + private static final String SESSION_FILE_RESOURCE = + "oracle/OracleDataStreamToSpannerMixedIT/oracle-session.json"; private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-google_standard_sql-spanner-schema.sql"; - + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; @@ -66,14 +81,14 @@ public void setUp() throws IOException { .setCredentialsProvider(credentialsProvider) .setPrivateConnectivity(System.getProperty("privateConnectivity")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -86,15 +101,19 @@ public void setUp() throws IOException { String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); - cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql"); + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseMixedIT/oracle-schema.sql"); OracleSource jdbcSource = OracleSource.builder( @@ -119,8 +138,12 @@ public void setUp() throws IOException { pubsubResourceManager, new HashMap<>() { { - put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); - put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + put( + "shadowTableSpannerInstanceId", + shadowSpannerResourceManager.getInstanceId()); + put( + "shadowTableSpannerDatabaseId", + shadowSpannerResourceManager.getDatabaseId()); put("inputFileFormat", "avro"); put("datastreamSourceType", "oracle"); } @@ -136,42 +159,70 @@ public void setUp() throws IOException { } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); - } @AfterClass @@ -195,15 +246,23 @@ public void mixedMigrationTest() { ChainedConditionCheck.builder( List.of( writeInitialData(), - SpannerRowsCheck.builder(spannerResourceManager, "Authors").setMinRows(4).setMaxRows(4).build(), - SpannerRowsCheck.builder(spannerResourceManager, "Books").setMinRows(3).setMaxRows(3).build(), - SpannerRowsCheck.builder(spannerResourceManager, "Genre").setMinRows(1).setMaxRows(1).build())) + SpannerRowsCheck.builder(spannerResourceManager, "Authors") + .setMinRows(4) + .setMaxRows(4) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Books") + .setMinRows(3) + .setMaxRows(3) + .build(), + SpannerRowsCheck.builder(spannerResourceManager, "Genre") + .setMinRows(1) + .setMaxRows(1) + .build())) .build(); PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); @@ -214,6 +273,7 @@ public void mixedMigrationTest() { private ConditionCheck writeInitialData() { return new ConditionCheck() { boolean executed = false; + @Override protected String getDescription() { return "Write JDBC data"; @@ -222,31 +282,40 @@ protected String getDescription() { @Override protected CheckResult check() { if (!executed) { - + List> authorRows = new ArrayList<>(); authorRows.add(Map.of("author_id", 4, "full_name", "Stephen King")); authorRows.add(Map.of("author_id", 1, "full_name", "Jane Austen")); authorRows.add(Map.of("author_id", 2, "full_name", "Charles Dickens")); authorRows.add(Map.of("author_id", 3, "full_name", "Leo Tolstoy")); - + List> bookRows = new ArrayList<>(); bookRows.add(Map.of("id", 1, "title", "Pride and Prejudice", "author_id", 1)); bookRows.add(Map.of("id", 2, "title", "Oliver Twist", "author_id", 2)); bookRows.add(Map.of("id", 3, "title", "War and Peace", "author_id", 3)); - + List> genreRows = new ArrayList<>(); genreRows.add(Map.of("genre_id", 1, "name", "Fiction")); for (Map r : authorRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", r.get("author_id"), r.get("full_name"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (%d, '%s')", + r.get("author_id"), r.get("full_name"))); } for (Map r : bookRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", r.get("id"), r.get("title"), r.get("author_id"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (%d, '%s', %d)", + r.get("id"), r.get("title"), r.get("author_id"))); } for (Map r : genreRows) { - cloudSqlResourceManager.runSQLUpdate(String.format("INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", r.get("genre_id"), r.get("name"))); + cloudSqlResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"Genre\"(\"genre_id\",\"name\") VALUES (%d, '%s')", + r.get("genre_id"), r.get("name"))); } - + cloudSqlResourceManager.runSQLUpdate("COMMIT"); cloudOracleSysUser.runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); executed = true; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index 865bd2a39d..0531765e64 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -1,4 +1,18 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -37,18 +51,20 @@ import org.junit.runners.JUnit4; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.io.Resources; @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) public class OracleSeparateShadowTableDatabasePKFocusedIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabasePKFocusedIT.class); + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabasePKFocusedIT.class); - private static final String ORACLE_SCHEMA = "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql"; - private static final String SPANNER_DDL_RESOURCE = "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql"; + private static final String ORACLE_SCHEMA = + "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-schema.sql"; + private static final String SPANNER_DDL_RESOURCE = + "oracle/OracleSeparateShadowTableDatabasePKFocusedIT/oracle-google_standard_sql-spanner-schema.sql"; private static final String GCS_PATH_PREFIX = "OracleSeparateShadowTableDatabasePKFocusedIT"; - + // Oracle defaults to uppercase unquoted identifiers private static final String ORACLE_TABLE_MY_TABLE = "MY_TABLE"; private static final String ORACLE_TABLE_ALLTYPES = "ALLTYPES"; @@ -64,43 +80,68 @@ public class OracleSeparateShadowTableDatabasePKFocusedIT extends DataStreamToSp private static DatastreamResourceManager datastreamResourceManager; private static boolean initialized = false; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = + new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; private void setUpOracleUser(String user, String password) { - oracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); + oracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @@ -111,7 +152,7 @@ public void setUp() throws IOException { testInstances.add(this); if (!initialized) { LOG.info("Setting up Oracle sys resource manager..."); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); @@ -120,15 +161,16 @@ public void setUp() throws IOException { sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); } - oracleSysUser = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); - + oracleSysUser = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); - + LOG.info("Provisioning isolated user: " + oracleUser); setUpOracleUser(oracleUser, oraclePassword); - - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { builder.setPassword(oraclePassword); @@ -137,8 +179,9 @@ public void setUp() throws IOException { builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); } - oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); - + oracleResourceManager = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); @@ -151,7 +194,7 @@ public void setUp() throws IOException { executeSqlScript(oracleResourceManager, ORACLE_SCHEMA); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - + OracleSource oracleSource = OracleSource.builder( oracleResourceManager.getHost(), @@ -160,29 +203,34 @@ public void setUp() throws IOException { oracleResourceManager.getPort(), oracleResourceManager.getDatabaseName()) .setAllowedTables( - Map.of(oracleResourceManager.getUsername().toUpperCase(), List.of(ORACLE_TABLE_MY_TABLE, ORACLE_TABLE_ALLTYPES))) + Map.of( + oracleResourceManager.getUsername().toUpperCase(), + List.of(ORACLE_TABLE_MY_TABLE, ORACLE_TABLE_ALLTYPES))) .build(); Map overridesMap = new HashMap<>(); overridesMap.put("inputFileFormat", "avro"); - overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); - overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); - - jobInfo = launchDataflowJob( - getClass().getSimpleName(), - null, - null, - GCS_PATH_PREFIX, - spannerResourceManager, - pubsubResourceManager, - overridesMap, - null, - null, - gcsResourceManager, - datastreamResourceManager, - null, - oracleSource); - + overridesMap.put( + "shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put( + "shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + null, + null, + GCS_PATH_PREFIX, + spannerResourceManager, + pubsubResourceManager, + overridesMap, + null, + null, + gcsResourceManager, + datastreamResourceManager, + null, + oracleSource); + assertThatPipeline(jobInfo).isRunning(); initialized = true; } @@ -204,70 +252,94 @@ public static void cleanUp() throws IOException { } private void flushOracleLogs() { - try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XE", "system", System.getProperty("cloudProxyPassword")); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XE", + "system", + System.getProperty("cloudProxyPassword")); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } catch (Exception e) { - LOG.warn("Log switch failed. Continuing...", e); + LOG.warn("Log switch failed. Continuing...", e); } } @Test public void migrationTestSimpleTable() throws Exception { - ConditionCheck insertRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "Insert 3 records into MY_TABLE"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (1, 10)"); - oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (2, 20)"); - oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (3, 30)"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data inserted"); - } - }; - - ConditionCheck updateDeleteRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "Update and Delete in MY_TABLE"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET VAL = 10 WHERE ID = 2"); - oracleResourceManager.runSQLUpdate("DELETE FROM MY_TABLE WHERE ID = 1"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data updated/deleted"); - } - }; - - ConditionCheck pkUpdateRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "PK Update in MY_TABLE"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET ID = 10 WHERE ID = 3"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data PK updated"); - } - }; + ConditionCheck insertRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Insert 3 records into MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (1, 10)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (2, 20)"); + oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (3, 30)"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update and Delete in MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET VAL = 10 WHERE ID = 2"); + oracleResourceManager.runSQLUpdate("DELETE FROM MY_TABLE WHERE ID = 1"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "PK Update in MY_TABLE"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET ID = 10 WHERE ID = 3"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data PK updated"); + } + }; ChainedConditionCheck conditionCheck = ChainedConditionCheck.builder( @@ -285,15 +357,22 @@ protected CheckResult check() { pkUpdateRecords, new ConditionCheck() { @Override - public String getDescription() { return "Check if MY_TABLE PK updated to 10"; } + public String getDescription() { + return "Check if MY_TABLE PK updated to 10"; + } + @Override protected CheckResult check() { try { - com.google.common.collect.ImmutableList rows = spannerResourceManager.runQuery("SELECT ID FROM MY_TABLE WHERE ID = 10"); + com.google.common.collect.ImmutableList + rows = + spannerResourceManager.runQuery( + "SELECT ID FROM MY_TABLE WHERE ID = 10"); if (!rows.isEmpty()) { return new CheckResult(true, "PK updated to 10"); } - } catch (Exception e) {} + } catch (Exception e) { + } return new CheckResult(false, "Not yet"); } })) @@ -301,88 +380,116 @@ protected CheckResult check() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); - try { Thread.sleep(CUTOVER_MILLIS); } catch (Exception e) {} - + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (Exception e) { + } + List> events = new ArrayList<>(); Map row1 = new HashMap<>(); row1.put("ID", 2L); - row1.put("VAL", 10L); + row1.put("VAL", 10L); Map row2 = new HashMap<>(); row2.put("ID", 10L); - row2.put("VAL", 30L); + row2.put("VAL", 30L); events.add(row1); events.add(row2); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select ID, VAL from MY_TABLE")) + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select ID, VAL from MY_TABLE")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @Test public void migrationTestAllDataType() throws Exception { - ConditionCheck insertRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "Insert records into ALLTYPES"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - // 1 - oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + - "VALUES ('true', 1, 3.14, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 10)"); - // 2 - oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + - "VALUES ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 20)"); - // 3 - oracleResourceManager.runSQLUpdate("INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + - "VALUES ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 30)"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data inserted"); - } - }; - - ConditionCheck updateDeleteRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "Update and Delete in ALLTYPES"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - oracleResourceManager.runSQLUpdate("DELETE FROM ALLTYPES WHERE INT64_FIELD = 1"); - oracleResourceManager.runSQLUpdate("UPDATE ALLTYPES SET VAL = 10 WHERE INT64_FIELD = 2"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data updated/deleted"); - } - }; - - ConditionCheck pkUpdateRecords = new ConditionCheck() { - boolean executed = false; - @Override - public String getDescription() { return "Update PK in ALLTYPES"; } - @Override - protected CheckResult check() { - if (!executed) { - try { - oracleResourceManager.runSQLUpdate("UPDATE ALLTYPES SET INT64_FIELD = 10 WHERE INT64_FIELD = 3"); - flushOracleLogs(); - executed = true; - } catch (Exception e) { return new CheckResult(false, e.getMessage()); } - } - return new CheckResult(true, "Data PK updated"); - } - }; + ConditionCheck insertRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Insert records into ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + // 1 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 1, 3.14, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 10)"); + // 2 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 20)"); + // 3 + oracleResourceManager.runSQLUpdate( + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " + + "VALUES ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 30)"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data inserted"); + } + }; + + ConditionCheck updateDeleteRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update and Delete in ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate("DELETE FROM ALLTYPES WHERE INT64_FIELD = 1"); + oracleResourceManager.runSQLUpdate( + "UPDATE ALLTYPES SET VAL = 10 WHERE INT64_FIELD = 2"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data updated/deleted"); + } + }; + + ConditionCheck pkUpdateRecords = + new ConditionCheck() { + boolean executed = false; + + @Override + public String getDescription() { + return "Update PK in ALLTYPES"; + } + + @Override + protected CheckResult check() { + if (!executed) { + try { + oracleResourceManager.runSQLUpdate( + "UPDATE ALLTYPES SET INT64_FIELD = 10 WHERE INT64_FIELD = 3"); + flushOracleLogs(); + executed = true; + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); + } + } + return new CheckResult(true, "Data PK updated"); + } + }; ChainedConditionCheck conditionCheck = ChainedConditionCheck.builder( @@ -400,15 +507,22 @@ protected CheckResult check() { pkUpdateRecords, new ConditionCheck() { @Override - public String getDescription() { return "Check if ALLTYPES PK updated to 10"; } + public String getDescription() { + return "Check if ALLTYPES PK updated to 10"; + } + @Override protected CheckResult check() { try { - com.google.common.collect.ImmutableList rows = spannerResourceManager.runQuery("SELECT INT64_FIELD FROM ALLTYPES WHERE INT64_FIELD = 10"); + com.google.common.collect.ImmutableList + rows = + spannerResourceManager.runQuery( + "SELECT INT64_FIELD FROM ALLTYPES WHERE INT64_FIELD = 10"); if (!rows.isEmpty()) { return new CheckResult(true, "PK updated to 10"); } - } catch (Exception e) {} + } catch (Exception e) { + } return new CheckResult(false, "Not yet"); } })) @@ -416,24 +530,26 @@ protected CheckResult check() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); - try { Thread.sleep(CUTOVER_MILLIS); } catch (Exception e) {} - + try { + Thread.sleep(CUTOVER_MILLIS); + } catch (Exception e) { + } + List> events = new ArrayList<>(); Map r1 = new HashMap<>(); r1.put("INT64_FIELD", 2L); r1.put("VAL", 10L); Map r2 = new HashMap<>(); r2.put("INT64_FIELD", 10L); - r2.put("VAL", 30L); + r2.put("VAL", 30L); events.add(r1); events.add(r2); - SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select INT64_FIELD, VAL from ALLTYPES")) + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select INT64_FIELD, VAL from ALLTYPES")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java index 01efebed2b..232f039432 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -1,7 +1,20 @@ -/* GENERATED BY: Test Automation Skill */ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; -import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; import com.google.cloud.spanner.Struct; @@ -11,7 +24,6 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -19,20 +31,17 @@ import java.util.Map; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; -import org.apache.beam.it.common.utils.PipelineUtils; import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ChainedConditionCheck; import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; -import org.apache.beam.it.gcp.datastream.DatastreamResourceManager.DestinationOutputFormat; import org.apache.beam.it.gcp.datastream.OracleSource; import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; import org.apache.beam.it.gcp.spanner.SpannerResourceManager; import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.gcp.storage.GcsResourceManager; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -53,8 +62,9 @@ public class OracleSeparateShadowTableDatabaseSessionIT extends DataStreamToSpan private static final String TABLE1 = "Category"; private static final String TABLE2 = "Books"; private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); - + private static HashSet testInstances = + new HashSet<>(); + public static DatastreamResourceManager datastreamResourceManager; public static CloudOracleResourceManager cloudOracleSysUser; public static CloudOracleResourceManager cloudOracleResourceManager; @@ -80,23 +90,34 @@ public void setUp() throws IOException { datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + cloudOracleResourceManager = setUpOracleResourceManager(); - try { cloudOracleResourceManager.runSQLUpdate(String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE1)); } catch (Exception e) {} - try { cloudOracleResourceManager.runSQLUpdate(String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE2)); } catch (Exception e) {} + try { + cloudOracleResourceManager.runSQLUpdate( + String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE1)); + } catch (Exception e) { + } + try { + cloudOracleResourceManager.runSQLUpdate( + String.format("DROP TABLE \"%s\" CASCADE CONSTRAINTS", TABLE2)); + } catch (Exception e) { + } executeSqlScript(cloudOracleResourceManager, ORACLE_DDL_RESOURCE); - cloudOracleResourceManager.runSQLUpdate(String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE1)); - cloudOracleResourceManager.runSQLUpdate(String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE2)); + cloudOracleResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE2)); spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - + OracleSource jdbcSource = OracleSource.builder( cloudOracleResourceManager.getHost(), @@ -109,11 +130,11 @@ public void setUp() throws IOException { cloudOracleResourceManager.getUsername().toUpperCase(), List.of(TABLE1, TABLE2))) .build(); - + Map jobParams = new HashMap<>(); jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); - + jobInfo = launchDataflowJob( getClass().getSimpleName(), @@ -132,27 +153,40 @@ public void setUp() throws IOException { } } } - - public static void setUpOracleUser(CloudOracleResourceManager sysUser, String user, String password) { - sysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + + public static void setUpOracleUser( + CloudOracleResourceManager sysUser, String user, String password) { + sysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); sysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); @@ -168,9 +202,10 @@ public static void setUpOracleUser(CloudOracleResourceManager sysUser, String us sysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - + // Add c##datastream grants based on requirements - sysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL")); + sysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL")); } @AfterClass @@ -206,9 +241,7 @@ public void migrationTestWithRenameAndDrops() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); assertCategoryTableBackfillContents(); @@ -250,90 +283,114 @@ public void migrationTestWithSyntheticPKAndExtraColumn() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); assertBooksBackfillContents(); } - + private ConditionCheck writeInitialCategoryData() { return new ConditionCheck() { private boolean executed = false; @Override - protected String getDescription() { return "Write initial Category rows"; } + protected String getDescription() { + return "Write initial Category rows"; + } @Override protected CheckResult check() { if (executed) { - return new CheckResult(true, "Sent category initial"); + return new CheckResult(true, "Sent category initial"); } try { - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (1, 'xyz')", TABLE1)); - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (2, 'abc')", TABLE1)); - cloudOracleResourceManager.runSQLUpdate("COMMIT"); - flushOracleRedoLogs(cloudOracleSysUser); - executed = true; - return new CheckResult(true, "Sent category initial"); - } catch(Exception e) { - return new CheckResult(false, e.getMessage()); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (1, 'xyz')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (2, 'abc')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category initial"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); } } }; } - + private ConditionCheck writeCdcCategoryData() { return new ConditionCheck() { private boolean executed = false; @Override - protected String getDescription() { return "Write cdc Category rows"; } + protected String getDescription() { + return "Write cdc Category rows"; + } @Override protected CheckResult check() { if (executed) { - return new CheckResult(true, "Sent category cdc"); + return new CheckResult(true, "Sent category cdc"); } try { - cloudOracleResourceManager.runSQLUpdate(String.format("DELETE FROM \"%s\" WHERE \"category_id\" = 1", TABLE1)); - cloudOracleResourceManager.runSQLUpdate(String.format("UPDATE \"%s\" SET \"full_name\" = 'abc1' WHERE \"category_id\" = 2", TABLE1)); - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (3, 'def')", TABLE1)); - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (4, 'ghi')", TABLE1)); - cloudOracleResourceManager.runSQLUpdate("COMMIT"); - flushOracleRedoLogs(cloudOracleSysUser); - executed = true; - return new CheckResult(true, "Sent category cdc"); - } catch(Exception e) { - return new CheckResult(false, e.getMessage()); + cloudOracleResourceManager.runSQLUpdate( + String.format("DELETE FROM \"%s\" WHERE \"category_id\" = 1", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "UPDATE \"%s\" SET \"full_name\" = 'abc1' WHERE \"category_id\" = 2", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (3, 'def')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"category_id\", \"full_name\") VALUES (4, 'ghi')", TABLE1)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent category cdc"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); } } }; } - + private ConditionCheck writeBooksData() { return new ConditionCheck() { private boolean executed = false; @Override - protected String getDescription() { return "Write Books rows"; } + protected String getDescription() { + return "Write Books rows"; + } @Override protected CheckResult check() { if (executed) { - return new CheckResult(true, "Sent books"); + return new CheckResult(true, "Sent books"); } try { - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of the Rings', 1)", TABLE2)); - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and Prejudice', 2)", TABLE2)); - cloudOracleResourceManager.runSQLUpdate(String.format("INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The Hitchhikers Guide to the Galaxy', 3)", TABLE2)); - cloudOracleResourceManager.runSQLUpdate("COMMIT"); - flushOracleRedoLogs(cloudOracleSysUser); - executed = true; - return new CheckResult(true, "Sent books"); - } catch(Exception e) { - return new CheckResult(false, e.getMessage()); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of the Rings', 1)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and Prejudice', 2)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate( + String.format( + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The Hitchhikers Guide to the Galaxy', 3)", + TABLE2)); + cloudOracleResourceManager.runSQLUpdate("COMMIT"); + flushOracleRedoLogs(cloudOracleSysUser); + executed = true; + return new CheckResult(true, "Sent books"); + } catch (Exception e) { + return new CheckResult(false, e.getMessage()); } } }; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 3301de1711..9440532331 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -17,8 +32,8 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; import org.apache.beam.it.gcp.datastream.OracleSource; @@ -40,8 +55,11 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class); +public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger( + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class); private static final String TABLE = "Users"; private static final String SPANNER_DDL_RESOURCE = @@ -50,7 +68,9 @@ public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShard "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-session.json"; private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); + private static HashSet< + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT> + testInstances = new HashSet<>(); public static PubsubResourceManager pubsubResourceManager; public static SpannerResourceManager spannerResourceManager; @@ -64,22 +84,25 @@ public class OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShard @Before public void setUp() throws Exception { skipBaseCleanup = true; - synchronized (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class) { + synchronized ( + OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.class) { testInstances.add(this); if (jobInfo == null) { - datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -92,55 +115,87 @@ public void setUp() throws Exception { String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); - jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql"); - try { jdbcResourceManagerShardA.runSQLUpdate("ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); } catch (Exception e) {} + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql"); + try { + jdbcResourceManagerShardA.runSQLUpdate( + "ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + } catch (Exception e) { + } - OracleSource jdbcSource = OracleSource.builder( - jdbcResourceManagerShardA.getHost(), - jdbcResourceManagerShardA.getUsername(), - jdbcResourceManagerShardA.getPassword(), - jdbcResourceManagerShardA.getPort(), - jdbcResourceManagerShardA.getDatabaseName()) - .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) - .build(); + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); - com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); - String uniqueCdcPrefix = "oracle-shard-cdc_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + "/cdc/"; - com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), uniqueCdcPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); - com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( - "test_stream_"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = + "oracle-shard-cdc_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + uniqueCdcPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); datastreamResourceManager.startStream(stream); streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); - gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); - + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + Map jobParams = new HashMap<>(); jobParams.put("inputFileFormat", "avro"); - jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put( + "inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); jobParams.put("datastreamSourceType", "oracle"); - jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - jobParams.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); } java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); - if (!fileLocal.exists()) { fileLocal = new java.io.File("v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); } - String sessionFileContent = new String(java.nio.file.Files.readAllBytes(fileLocal.toPath()), java.nio.charset.StandardCharsets.UTF_8); - sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); - + if (!fileLocal.exists()) { + fileLocal = + new java.io.File( + "v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); + } + String sessionFileContent = + new String( + java.nio.file.Files.readAllBytes(fileLocal.toPath()), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + jobInfo = launchDataflowJob( getClass().getSimpleName() + "shard1", @@ -161,50 +216,86 @@ public void setUp() throws Exception { } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass public static void cleanUp() throws IOException { - for (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT instance : testInstances) { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT instance : + testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); } @Test @@ -215,29 +306,32 @@ public void multiShardMigration() throws Exception { ChainedConditionCheck.builder( List.of( new ConditionCheck() { - boolean executed = false; - @Override - protected String getDescription() { return "Insert Data into Oracle"; } - @Override - protected CheckResult check() { - if (!executed) { - insertDataInOracle(); - executed = true; - } - return new CheckResult(true, "Inserted successfully"); + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; } + return new CheckResult(true, "Inserted successfully"); + } }, SpannerRowsCheck.builder(spannerResourceManager, TABLE) .setMinRows(12) .setMaxRows(12) - .build() - ) - ).build(); - + .build())) + .build(); + PipelineOperator.Result result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); - + assertThatResult(result).meetsConditions(); assertUsersTableContents(); @@ -245,25 +339,41 @@ protected CheckResult check() { private void insertDataInOracle() { LOG.info("Inserting rows into Users table in Oracle"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); - try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } catch (Exception e) { - flushOracleRedoLogs(jdbcResourceManagerShardA); + flushOracleRedoLogs(jdbcResourceManagerShardA); } } @@ -286,14 +396,19 @@ private void assertUsersTableContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private String generateSourceConfig( - String streamA, String dbA, String shardA) { + private String generateSourceConfig(String streamA, String dbA, String shardA) { return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index a489684de8..06f8261700 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -39,8 +54,11 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class); +public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger( + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class); private static final String TABLE = "Users"; private static final String SPANNER_DDL_RESOURCE = @@ -49,7 +67,9 @@ public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationSh "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-session.json"; private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); + private static HashSet< + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT> + testInstances = new HashSet<>(); public static PubsubResourceManager pubsubResourceManager; public static SpannerResourceManager spannerResourceManager; @@ -63,22 +83,25 @@ public class OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationSh @Before public void setUp() throws Exception { skipBaseCleanup = true; - synchronized (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class) { + synchronized ( + OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.class) { testInstances.add(this); if (jobInfo == null) { - datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -91,54 +114,82 @@ public void setUp() throws Exception { String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); - jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql"); + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT/oracle-schema.sql"); - OracleSource jdbcSource = OracleSource.builder( - jdbcResourceManagerShardA.getHost(), - jdbcResourceManagerShardA.getUsername(), - jdbcResourceManagerShardA.getPassword(), - jdbcResourceManagerShardA.getPort(), - jdbcResourceManagerShardA.getDatabaseName()) - .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) - .build(); + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); - com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); - String uniqueCdcPrefix = "oracle-shard-cdc_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + "/cdc/"; - com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), uniqueCdcPrefix, DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); - com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( - "test_stream_"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + String uniqueCdcPrefix = + "oracle-shard-cdc_" + + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toLowerCase() + + "/cdc/"; + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + uniqueCdcPrefix, + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test_stream_" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); datastreamResourceManager.startStream(stream); streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); - gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); - + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, oracleUser, "L1")); + Map jobParams = new HashMap<>(); jobParams.put("inputFileFormat", "avro"); - jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); + jobParams.put( + "inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/" + uniqueCdcPrefix); jobParams.put("datastreamSourceType", "oracle"); - jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - jobParams.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); } java.io.File fileLocal = new java.io.File("src/test/resources/" + SESSION_FILE_RESOURCE); - if (!fileLocal.exists()) { fileLocal = new java.io.File("v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); } - String sessionFileContent = new String(java.nio.file.Files.readAllBytes(fileLocal.toPath()), java.nio.charset.StandardCharsets.UTF_8); - sessionFileContent = sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); - + if (!fileLocal.exists()) { + fileLocal = + new java.io.File( + "v2/datastream-to-spanner/src/test/resources/" + SESSION_FILE_RESOURCE); + } + String sessionFileContent = + new String( + java.nio.file.Files.readAllBytes(fileLocal.toPath()), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", oracleUser).replace("shard_1", "L1"); + jobInfo = launchDataflowJob( getClass().getSimpleName() + "shard1", @@ -159,50 +210,86 @@ public void setUp() throws Exception { } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass public static void cleanUp() throws IOException { - for (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT instance : testInstances) { + for (OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT instance : + testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); } @Test @@ -213,29 +300,32 @@ public void multiShardMigration() throws Exception { org.apache.beam.it.conditions.ChainedConditionCheck.builder( List.of( new ConditionCheck() { - boolean executed = false; - @Override - protected String getDescription() { return "Insert Data into Oracle"; } - @Override - protected CheckResult check() { - if (!executed) { - insertDataInOracle(); - executed = true; - } - return new CheckResult(true, "Inserted successfully"); + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; } + return new CheckResult(true, "Inserted successfully"); + } }, SpannerRowsCheck.builder(spannerResourceManager, TABLE) .setMinRows(12) .setMaxRows(12) - .build() - ) - ).build(); - + .build())) + .build(); + PipelineOperator.Result result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); - + assertThatResult(result).meetsConditions(); assertUsersTableContents(); @@ -243,25 +333,41 @@ protected CheckResult check() { private void insertDataInOracle() { LOG.info("Inserting rows into Users table in Oracle"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); - try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } catch (Exception e) { - flushOracleRedoLogs(jdbcResourceManagerShardA); + flushOracleRedoLogs(jdbcResourceManagerShardA); } } @@ -284,14 +390,19 @@ private void assertUsersTableContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private String generateSourceConfig( - String streamA, String dbA, String shardA) { + private String generateSourceConfig(String streamA, String dbA, String shardA) { return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index 2900e3262c..d789402d3c 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; @@ -39,9 +54,10 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT extends DataStreamToSpannerITBase { - private static final Logger LOG = LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class); - +public class OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT + extends DataStreamToSpannerITBase { + private static final Logger LOG = + LoggerFactory.getLogger(OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class); private static final String TABLE = "Users"; private static final String SPANNER_DDL_RESOURCE = @@ -50,7 +66,8 @@ public class OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT extends "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-session.json"; private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); + private static HashSet + testInstances = new HashSet<>(); public static PubsubResourceManager pubsubResourceManager; public static SpannerResourceManager spannerResourceManager; @@ -67,19 +84,21 @@ public void setUp() throws Exception { synchronized (OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.class) { testInstances.add(this); if (jobInfo == null) { - datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -88,57 +107,84 @@ public void setUp() throws Exception { builder.setDatabaseName("XE"); cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); - jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(System.getProperty("cloudProxyUsername", "system")) - .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + jdbcResourceManagerShardA = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudProxyUsername", "system")) + .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); + } catch (Exception e) { + } - try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) {} - executeSqlScript(jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql"); + try { + jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); + } catch (Exception e) { + } + executeSqlScript( + jdbcResourceManagerShardA, + "oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT/oracle-schema.sql"); - OracleSource jdbcSource = OracleSource.builder( - jdbcResourceManagerShardA.getHost(), - jdbcResourceManagerShardA.getUsername(), - jdbcResourceManagerShardA.getPassword(), - jdbcResourceManagerShardA.getPort(), - jdbcResourceManagerShardA.getDatabaseName()) - .setAllowedTables(Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) - .build(); + OracleSource jdbcSource = + OracleSource.builder( + jdbcResourceManagerShardA.getHost(), + jdbcResourceManagerShardA.getUsername(), + jdbcResourceManagerShardA.getPassword(), + jdbcResourceManagerShardA.getPort(), + jdbcResourceManagerShardA.getDatabaseName()) + .setAllowedTables( + Map.of(jdbcResourceManagerShardA.getUsername().toUpperCase(), List.of("Users"))) + .build(); - com.google.cloud.datastream.v1.SourceConfig sourceConfig = datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); - com.google.cloud.datastream.v1.DestinationConfig destinationConfig = datastreamResourceManager.buildGCSDestinationConfig( - "gcs-profile", gcsResourceManager.getBucket(), "oracle-shard-cdc/cdc/", DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); - com.google.cloud.datastream.v1.Stream stream = datastreamResourceManager.createStream( - "test-stream-"+ RandomStringUtils.randomAlphanumeric(5).toLowerCase(), sourceConfig, destinationConfig); + com.google.cloud.datastream.v1.SourceConfig sourceConfig = + datastreamResourceManager.buildJDBCSourceConfig("jdbc-profile", jdbcSource); + com.google.cloud.datastream.v1.DestinationConfig destinationConfig = + datastreamResourceManager.buildGCSDestinationConfig( + "gcs-profile", + gcsResourceManager.getBucket(), + "oracle-shard-cdc/cdc/", + DatastreamResourceManager.DestinationOutputFormat.AVRO_FILE_FORMAT); + com.google.cloud.datastream.v1.Stream stream = + datastreamResourceManager.createStream( + "test-stream-" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(), + sourceConfig, + destinationConfig); datastreamResourceManager.startStream(stream); streamNameA = stream.getName().substring(stream.getName().lastIndexOf('/') + 1); - gcsResourceManager.createArtifact("input/shardingConfig.conf", generateSourceConfig(streamNameA, "system", "L1")); - + gcsResourceManager.createArtifact( + "input/shardingConfig.conf", generateSourceConfig(streamNameA, "system", "L1")); + Map jobParams = new HashMap<>(); jobParams.put("inputFileFormat", "avro"); - jobParams.put("inputFilePattern", "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); + jobParams.put( + "inputFilePattern", + "gs://" + gcsResourceManager.getBucket() + "/oracle-shard-cdc/cdc/"); jobParams.put("datastreamSourceType", "oracle"); - jobParams.put("sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); + jobParams.put( + "sourceConfigURL", getGcsPath("input/shardingConfig.conf", gcsResourceManager)); jobParams.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); jobParams.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); if (System.getProperty("jdbcDriverJars") != null) { - String driverPath = System.getProperty("jdbcDriverJars"); - jobParams.put("jdbcDriverJars", driverPath); + String driverPath = System.getProperty("jdbcDriverJars"); + jobParams.put("jdbcDriverJars", driverPath); } String sessionFileContent = - com.google.common.io.Resources.toString(com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), java.nio.charset.StandardCharsets.UTF_8); - sessionFileContent = sessionFileContent.replace("it_test", "system").replace("shard_1", "L1"); - + com.google.common.io.Resources.toString( + com.google.common.io.Resources.getResource(SESSION_FILE_RESOURCE), + java.nio.charset.StandardCharsets.UTF_8); + sessionFileContent = + sessionFileContent.replace("it_test", "system").replace("shard_1", "L1"); + jobInfo = - launchDataflowJob( + launchDataflowJob( getClass().getSimpleName() + "shard1", null, null, @@ -152,45 +198,73 @@ public void setUp() throws Exception { datastreamResourceManager, sessionFileContent, jdbcSource); - } } } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @@ -201,7 +275,13 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, shadowSpannerResourceManager, pubsubResourceManager, gcsResourceManager, jdbcResourceManagerShardA, datastreamResourceManager, cloudOracleSysUser); + spannerResourceManager, + shadowSpannerResourceManager, + pubsubResourceManager, + gcsResourceManager, + jdbcResourceManagerShardA, + datastreamResourceManager, + cloudOracleSysUser); } @Test @@ -212,29 +292,32 @@ public void multiShardMigration() throws Exception { org.apache.beam.it.conditions.ChainedConditionCheck.builder( List.of( new ConditionCheck() { - boolean executed = false; - @Override - protected String getDescription() { return "Insert Data into Oracle"; } - @Override - protected CheckResult check() { - if (!executed) { - insertDataInOracle(); - executed = true; - } - return new CheckResult(true, "Inserted successfully"); + boolean executed = false; + + @Override + protected String getDescription() { + return "Insert Data into Oracle"; + } + + @Override + protected CheckResult check() { + if (!executed) { + insertDataInOracle(); + executed = true; } + return new CheckResult(true, "Inserted successfully"); + } }, SpannerRowsCheck.builder(spannerResourceManager, TABLE) .setMinRows(12) .setMaxRows(12) - .build() - ) - ).build(); - + .build())) + .build(); + PipelineOperator.Result result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), rowsConditionCheck); - + assertThatResult(result).meetsConditions(); assertUsersTableContents(); @@ -242,25 +325,41 @@ protected CheckResult check() { private void insertDataInOracle() { LOG.info("Inserting rows into Users table in Oracle"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); - jdbcResourceManagerShardA.runSQLUpdate("INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (1, 'Tester1', 20)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (3, 'Tester3', 103)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (13, 'Tester13', 113)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (4, 'Tester4', 104)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (5, 'Tester5', 105)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (6, 'Tester6', 106)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (7, 'Tester7', 107)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (8, 'Tester8', 108)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (9, 'Tester9', 109)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (10, 'Tester10', 110)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (11, 'Tester11', 111)"); + jdbcResourceManagerShardA.runSQLUpdate( + "INSERT INTO \"Users\" (\"id\", \"name\", \"age\") VALUES (12, 'Tester12', 112)"); jdbcResourceManagerShardA.runSQLUpdate("COMMIT"); - try (java.sql.Connection conn = java.sql.DriverManager.getConnection("jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); + try (java.sql.Connection conn = + java.sql.DriverManager.getConnection( + "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "system", + "TestPassword123"); + java.sql.Statement stmt = conn.createStatement()) { + flushOracleRedoLogs(null); } catch (Exception e) { - flushOracleRedoLogs(jdbcResourceManagerShardA); + flushOracleRedoLogs(jdbcResourceManagerShardA); } } @@ -283,14 +382,19 @@ private void assertUsersTableContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } - private String generateSourceConfig( - String streamA, String dbA, String shardA) { + private String generateSourceConfig(String streamA, String dbA, String shardA) { return "{\n" + " \"shardConfigs\": [\n" + " {\n" - + " \"logicalShardId\": \"" + shardA + "\",\n" - + " \"dbName\": \"" + dbA + "\",\n" - + " \"streamId\": \"" + streamA + "\"\n" + + " \"logicalShardId\": \"" + + shardA + + "\",\n" + + " \"dbName\": \"" + + dbA + + "\",\n" + + " \"streamId\": \"" + + streamA + + "\"\n" + " }\n" + " ]\n" + "}"; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index fec02abe26..148ee060a9 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.templates.oracle; import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; @@ -17,7 +32,6 @@ import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; import org.apache.beam.it.conditions.ChainedConditionCheck; -import org.apache.beam.it.conditions.ConditionCheck; import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; import org.apache.beam.it.gcp.datastream.DatastreamResourceManager; import org.apache.beam.it.gcp.datastream.OracleSource; @@ -26,7 +40,6 @@ import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; import org.apache.beam.it.gcp.storage.GcsResourceManager; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -46,7 +59,8 @@ public class OracleSeparateShadowTableDatabaseStringOverridesIT extends DataStre "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-GOOGLE_STANDARD_SQL-spanner-schema.sql"; private static PipelineLauncher.LaunchInfo jobInfo; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = + new HashSet<>(); public static PubsubResourceManager pubsubResourceManager; public static SpannerResourceManager spannerResourceManager; @@ -62,19 +76,21 @@ public void setUp() throws IOException { synchronized (OracleSeparateShadowTableDatabaseStringOverridesIT.class) { testInstances.add(this); if (jobInfo == null) { - datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) + datastreamResourceManager = + DatastreamResourceManager.builder(testName, PROJECT, REGION) .setCredentialsProvider(credentialsProvider) - .setPrivateConnectivity(System.getProperty("privateConnectivity", "datastream-connect-2")) + .setPrivateConnectivity( + System.getProperty("privateConnectivity", "datastream-connect-2")) .build(); - + spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); pubsubResourceManager = setUpPubSubResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); - + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword")); @@ -83,34 +99,44 @@ public void setUp() throws IOException { builder.setDatabaseName("XE"); cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); - cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(System.getProperty("cloudProxyUsername", "system")) - .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) - .setDatabaseName("XE") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + cloudSqlResourceManager = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(testName) + .setUsername(System.getProperty("cloudProxyUsername", "system")) + .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setDatabaseName("XE") + .setHost(System.getProperty("hostIp")) + .setPort(1521) + .build(); + try { + cloudSqlResourceManager.runSQLUpdate("DROP TABLE \"person1\""); + } catch (Exception e) { + } + executeSqlScript( + cloudSqlResourceManager, + "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql"); - try { cloudSqlResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch (Exception e) {} - executeSqlScript(cloudSqlResourceManager, "oracle/OracleSeparateShadowTableDatabaseStringOverridesIT/oracle-schema.sql"); - - OracleSource jdbcSource = OracleSource.builder( - cloudSqlResourceManager.getHost(), - cloudSqlResourceManager.getUsername(), - cloudSqlResourceManager.getPassword(), - cloudSqlResourceManager.getPort(), - cloudSqlResourceManager.getDatabaseName()) - .setAllowedTables(Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), List.of("person1"))) - .build(); + OracleSource jdbcSource = + OracleSource.builder( + cloudSqlResourceManager.getHost(), + cloudSqlResourceManager.getUsername(), + cloudSqlResourceManager.getPassword(), + cloudSqlResourceManager.getPort(), + cloudSqlResourceManager.getDatabaseName()) + .setAllowedTables( + Map.of(cloudSqlResourceManager.getUsername().toUpperCase(), List.of("person1"))) + .build(); Map overridesMap = new HashMap<>(); overridesMap.put("inputFileFormat", "avro"); overridesMap.put("datastreamSourceType", "oracle"); overridesMap.put("tableOverrides", "[{person1, human1}]"); overridesMap.put("columnOverrides", "[{person1.first_name1, person1.name1}]"); - overridesMap.put("shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); - overridesMap.put("shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); + overridesMap.put( + "shadowTableSpannerInstanceId", shadowSpannerResourceManager.getInstanceId()); + overridesMap.put( + "shadowTableSpannerDatabaseId", shadowSpannerResourceManager.getDatabaseId()); jobInfo = launchDataflowJob( @@ -132,42 +158,70 @@ public void setUp() throws IOException { } private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate(String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + cloudOracleSysUser.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); - } @AfterClass @@ -176,7 +230,13 @@ public static void cleanUp() throws IOException { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( - spannerResourceManager, pubsubResourceManager, shadowSpannerResourceManager, gcsResourceManager, datastreamResourceManager, cloudOracleSysUser, cloudSqlResourceManager); + spannerResourceManager, + pubsubResourceManager, + shadowSpannerResourceManager, + gcsResourceManager, + datastreamResourceManager, + cloudOracleSysUser, + cloudSqlResourceManager); } @Test @@ -186,17 +246,21 @@ public void migrationTestWithRenameTableAndColumns() { List.of( new org.apache.beam.it.conditions.ConditionCheck() { boolean executed = false; + @Override protected String getDescription() { return "Insert records into Oracle"; } + @Override protected CheckResult check() { if (!executed) { - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); - cloudSqlResourceManager.runSQLUpdate("INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); + cloudSqlResourceManager.runSQLUpdate( + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); - flushOracleRedoLogs(cloudOracleSysUser); + flushOracleRedoLogs(cloudOracleSysUser); executed = true; } return new CheckResult(true, "Inserted successfully"); @@ -210,9 +274,7 @@ protected CheckResult check() { PipelineOperator.Result result = pipelineOperator() - .waitForCondition( - createConfig(jobInfo, Duration.ofMinutes(45)), - conditionCheck); + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(45)), conditionCheck); assertThatResult(result).meetsConditions(); assertHumanTableContents(); From 23c5a3061b66ecda720ffd3bab5bd9c67294f0a2 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Mon, 24 Aug 2026 16:25:35 +0000 Subject: [PATCH 03/13] Fix spotless issue in spanner-common module --- .../cloud/teleport/v2/templates/DataStreamToSpannerIT.java | 6 ++++-- .../migrations/utils/CustomTransformationImplFetcher.java | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java index 50dede9660..bcede2bfe1 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java @@ -507,7 +507,8 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager).runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager) + .runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(success, "Sent " + String.join(", ", messages) + "."); } @@ -578,7 +579,8 @@ protected CheckResult check() { // Force log file archive - needed so Datastream can see changes which are read from // archived log files. if (isOracle) { - ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager).runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); + ((org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) cloudSqlResourceManager) + .runSQLUpdate("ALTER SYSTEM SWITCH LOGFILE"); } return new CheckResult(true, "Sent " + String.join(", ", messages) + "."); } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java index 9263465ac0..b2e0448fe2 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CustomTransformationImplFetcher.java @@ -59,7 +59,8 @@ public static ISpannerMigrationTransformer getApplyTransformationImpl( URL[] classLoaderUrls = JarFileReader.saveFilesLocally(customTransformation.jarPath()); // Create a new URLClassLoader - URLClassLoader urlClassLoader = new URLClassLoader(classLoaderUrls, Thread.currentThread().getContextClassLoader()); + URLClassLoader urlClassLoader = + new URLClassLoader(classLoaderUrls, Thread.currentThread().getContextClassLoader()); // Load the target class Class customTransformationClass = From 8bd7f2c9736a93324db6e30330b30d35a03d6cc1 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Tue, 25 Aug 2026 05:39:21 +0000 Subject: [PATCH 04/13] Revert change in CloudOracleResourceManager --- .../apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java | 2 +- .../cloud/teleport/v2/templates/DataStreamToSpannerITBase.java | 2 +- .../oracle/DataStreamToSpannerOracleRetryAllDLQIT.java | 2 +- .../templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java | 2 +- ...ataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java | 2 +- .../oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java | 2 +- .../oracle/OracleDataStreamToSpannerFileOverridesIT.java | 2 +- .../v2/templates/oracle/OracleDataStreamToSpannerIT.java | 2 +- .../v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java | 2 +- .../templates/oracle/OracleDataStreamToSpannerTimezoneIT.java | 2 +- .../templates/oracle/OracleDatastreamToSpannerDataTypesIT.java | 2 +- .../OracleDatastreamToSpannerSingleDFShardedMigrationIT.java | 2 +- .../oracle/OracleSeparateShadowTableDatabaseEventsIT.java | 2 +- .../OracleSeparateShadowTableDatabaseFileOverridesIT.java | 2 +- .../oracle/OracleSeparateShadowTableDatabaseMixedIT.java | 2 +- .../oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java | 2 +- ...bleDatabaseShardedMigrationWithMigrationShardIdColumnIT.java | 2 +- ...DatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java | 2 +- ...leSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java index 391eb2fe41..1e5081a77b 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/cloudsql/CloudOracleResourceManager.java @@ -61,7 +61,7 @@ public static Builder builder(String testId) { @Override public synchronized @NonNull String getUri() { return String.format( - "jdbc:%s:thin:@//%s:%d/%s", + "jdbc:%s:thin:@%s:%d:%s", getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index 7a61cacdb2..c668103ca2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -99,7 +99,7 @@ public org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager setUpOracleRes builder.setPort(1521); builder.setUsername(System.getProperty("cloudProxyUsername", "system")); builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } return (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index c92f4f0643..00e8af02b6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -104,7 +104,7 @@ public void setUp() throws IOException, InterruptedException { builder.setPort(sysUser.getPort()); builder.setUsername(oracleUser); builder.setPassword(oraclePassword); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); jdbcResourceManager = (CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index c7d9c068ae..2839e12b80 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -88,7 +88,7 @@ private CloudOracleResourceManager setUpOracleResourceManagerForShard(String sha shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); shardBuilder.setHost(System.getProperty("hostIp")); shardBuilder.setPort(1521); - shardBuilder.setDatabaseName("XEPDB1"); + shardBuilder.setDatabaseName("/XEPDB1"); return shardBuilder.build(); } else { return CloudOracleResourceManager.builder(testName + shardName).build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index e8a0e08779..00082f361f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -113,7 +113,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(System.getProperty("cloudProxyUsername")); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } cloudOracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index a33650473a..80d422c470 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -93,7 +93,7 @@ public void setUp() throws IOException, InterruptedException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index dcdda1276d..63588e9e0a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -173,7 +173,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index 170df238bf..0148d206c3 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -233,7 +233,7 @@ private void simpleOracleToSpannerTest( CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index cadae8f1dc..7e5aad6d2a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -185,7 +185,7 @@ public void mixedMigrationTest() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index 53090d6d1a..5c7a60fa6f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -91,7 +91,7 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setHost("" + System.getProperty("hostIp") + ""); sysBuilder.setPort(1521); sysBuilder.setUsername("system"); - sysBuilder.setDatabaseName("XEPDB1"); + sysBuilder.setDatabaseName("/XEPDB1"); cloudOracleSysUser = (CloudOracleResourceManager) sysBuilder.build(); datastreamResourceManager = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index 7bbc856a82..24277db8d5 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -426,7 +426,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index 98dc215b77..58688e3872 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -108,7 +108,7 @@ public void setUp() throws IOException, InterruptedException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index b353f8ef8d..b1603b65b2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -113,7 +113,7 @@ public void setUp() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index b5a53c6adc..efb98da165 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -175,7 +175,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index b3e18a6549..12cda5d180 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -106,7 +106,7 @@ public void setUp() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index 0531765e64..9256957e15 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -177,7 +177,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("XEPDB1"); + builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 9440532331..7a47cded21 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -120,7 +120,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index 06f8261700..f380230859 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -119,7 +119,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index d789402d3c..f421e4d031 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -112,7 +112,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(System.getProperty("cloudProxyUsername", "system")) .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) - .setDatabaseName("XEPDB1") + .setDatabaseName("/XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); From ffe1bc58b73ff463402e978da57bcb0cf2935503 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Tue, 25 Aug 2026 10:30:08 +0000 Subject: [PATCH 05/13] Override uri method for CloudOracleResourceManager --- .../v2/templates/DataStreamToSpannerIT.java | 4 +- .../templates/DataStreamToSpannerITBase.java | 8 ++- .../DataStreamToSpannerOracleDDLIT.java | 56 +++++++++++-------- ...ataStreamToSpannerOracleRetryAllDLQIT.java | 18 ++++-- .../DataStreamToSpannerOracleRetryDLQIT.java | 18 ++++-- ...treamToSpannerOracleStringOverridesIT.java | 6 +- ...OracleWideRowForMaxColumnsPerTablesIT.java | 3 +- ...amToSpannerShardedOracleRetryAllDLQIT.java | 27 +++++---- ...cleDataStreamToSpannerFileOverridesIT.java | 12 ++-- .../oracle/OracleDataStreamToSpannerIT.java | 30 +++++----- .../OracleDataStreamToSpannerMixedIT.java | 2 +- .../OracleDataStreamToSpannerSessionIT.java | 21 ++++--- .../OracleDataStreamToSpannerTimezoneIT.java | 11 +++- ...rWideRowForMax16KeyTablePerDatabaseIT.java | 3 +- .../OracleDatastreamToSpannerDataTypesIT.java | 18 ++++-- ...amToSpannerSingleDFShardedMigrationIT.java | 15 ++--- ...astreamToSpannerTableAndIndexLimitsIT.java | 3 +- ...erWideRowForMax9MibTablePerDatabaseIT.java | 3 +- ...racleSeparateShadowTableDatabaseDDLIT.java | 56 +++++++++++-------- ...leSeparateShadowTableDatabaseEventsIT.java | 29 ++++++---- ...ateShadowTableDatabaseFileOverridesIT.java | 12 ++-- ...cleSeparateShadowTableDatabaseMixedIT.java | 2 +- ...eparateShadowTableDatabasePKFocusedIT.java | 27 ++++++--- ...eSeparateShadowTableDatabaseSessionIT.java | 9 ++- ...MigrationWithMigrationShardIdColumnIT.java | 2 +- ...rationWithoutMigrationShardIdColumnIT.java | 2 +- ...bleDatabaseSingleDFShardedMigrationIT.java | 2 +- ...eShadowTableDatabaseStringOverridesIT.java | 8 ++- .../oracle/SpannerOracleResourceManager.java | 38 +++++++++++++ 29 files changed, 290 insertions(+), 155 deletions(-) create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java index bcede2bfe1..828a3f6d60 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerIT.java @@ -24,6 +24,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager; import com.google.common.io.Resources; import com.google.pubsub.v1.SubscriptionName; import com.google.pubsub.v1.TopicName; @@ -225,7 +226,8 @@ private void simpleJdbcToSpannerTest( cloudSqlResourceManager = jdbcType.equals(JDBCType.MYSQL) ? CloudMySQLResourceManager.builder(testName).build() - : CloudOracleResourceManager.builder(testName).build(); + : new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) CloudOracleResourceManager.builder(testName)); // Create Spanner Resource Manager SpannerResourceManager.Builder spannerResourceManagerBuilder = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index c668103ca2..d924d0dc48 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -90,7 +90,8 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { .build(); } - public org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager setUpOracleResourceManager() { + public com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager + setUpOracleResourceManager() { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { @@ -99,9 +100,10 @@ public org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager setUpOracleRes builder.setPort(1521); builder.setUsername(System.getProperty("cloudProxyUsername", "system")); builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); } - return (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + builder.maybeUseStaticInstance(); + return new com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager(builder); } public String generateSessionFile( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java index 28ade7e925..b797ad3394 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -299,13 +299,13 @@ private void assertAllDatatypeColumnsTableCdcContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column" - + ", tinyblob_column, blob_column, mediumblob_column" - + ", longblob_column, enum_column, bool_column, other_bool_column" - + ", varbinary_column, bit_column, decimal_column, text_column, binary_column" - + ", char_column, tinytext_column, mediumtext_column, longtext_column from AllDatatypeColumns")) + "select varchar_column, tinyint_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column," + + " tinyblob_column, blob_column, mediumblob_column, longblob_column," + + " enum_column, bool_column, other_bool_column, varbinary_column, bit_column," + + " decimal_column, text_column, binary_column, char_column, tinytext_column," + + " mediumtext_column, longtext_column from AllDatatypeColumns")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -378,12 +378,13 @@ private void assertAllDatatypeColumns2TableBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" - + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -455,7 +456,11 @@ private void assertAllDatatypeTransformationTableBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, binary_column, bit_column FROM AllDatatypeTransformation")) + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column," + + " bigint_column, float_column, double_column, decimal_column," + + " datetime_column, timestamp_column, time_column, year_column, blob_column," + + " enum_column, bool_column, binary_column, bit_column FROM" + + " AllDatatypeTransformation")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -545,12 +550,13 @@ private void assertAllDatatypeColumns2TableCdcContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" - + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -581,8 +587,9 @@ private void assertDataTypeSizeConversionBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, float_column, decimal_column, char_column, bool_column" - + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsWithSizes")) + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsWithSizes")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -603,8 +610,9 @@ private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOExcept SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, float_column, decimal_column, char_column, bool_column" - + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsReducedSizes")) + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsReducedSizes")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index 00e8af02b6..7d6f5382f8 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -93,7 +93,8 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setUsername("sys as sysdba"); sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); sysBuilder.setDatabaseName(sysUser.getDatabaseName()); - CloudOracleResourceManager trueSysUser = (CloudOracleResourceManager) sysBuilder.build(); + CloudOracleResourceManager trueSysUser = + (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); String oracleUser = System.getProperty("cloudProxyUsername", "system"); String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); @@ -106,7 +107,8 @@ public void setUp() throws IOException, InterruptedException { builder.setPassword(oraclePassword); builder.setDatabaseName("/XEPDB1"); - jdbcResourceManager = (CloudOracleResourceManager) builder.build(); + jdbcResourceManager = + (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); // Create Oracle Schema using helper executeSqlScript(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE); @@ -352,13 +354,17 @@ private boolean rowExistsInSpanner(String tableName, String idColumnName, long i private void insertDataInOracle() { jdbcResourceManager.runSQLUpdate( - "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); jdbcResourceManager.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); jdbcResourceManager.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); jdbcResourceManager.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); jdbcResourceManager.runSQLUpdate( "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar2_col\") VALUES (1, 'test1')"); jdbcResourceManager.runSQLUpdate( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index 2839e12b80..8c78e788c9 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -89,9 +89,11 @@ private CloudOracleResourceManager setUpOracleResourceManagerForShard(String sha shardBuilder.setHost(System.getProperty("hostIp")); shardBuilder.setPort(1521); shardBuilder.setDatabaseName("/XEPDB1"); - return shardBuilder.build(); + return new SpannerOracleResourceManager(shardBuilder); } else { - return CloudOracleResourceManager.builder(testName + shardName).build(); + return new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName + shardName)); } } @@ -408,13 +410,17 @@ private boolean rowExistsInSpanner( private void insertDataInOracle() { LOG.info("Inserting data in Oracle Shard A"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); LOG.info("Inserting data in Oracle Shard B"); jdbcResourceManagerShardA.runSQLUpdate( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java index 330cefa4a5..5a242b5f73 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java @@ -171,9 +171,11 @@ protected String getDescription() { protected CheckResult check() { try { oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('John', 'Doe')"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('Alice', 'Johnson')"); return new CheckResult(true, "Inserted successfully"); } catch (Exception e) { return new CheckResult(false, "Failed to insert"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index 00082f361f..0126f5892e 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -116,7 +116,8 @@ public void setUp() throws IOException { builder.setDatabaseName("/XEPDB1"); } cloudOracleResourceManager = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); String sessionContent = generateBaseSchema(); sessionContent = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 80d422c470..4aed67db31 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -90,13 +90,14 @@ public void setUp() throws IOException, InterruptedException { jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("/XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521)); try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) { @@ -406,13 +407,17 @@ private String getCustomShardJarPath() { private void insertDataInOracle() { LOG.info("Inserting data in Oracle"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\", \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); + "INSERT INTO \"Customers\" (\"CustomerId\", \"CustomerName\", \"CreditLimit\"," + + " \"LoyaltyTier\") VALUES (1, 'Customer 1', 500, 'Bronze')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (3, 101, 1000, 'Website')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (3, 101, 1000, 'Website')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (2, 102, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (2, 102, 1000, 'AppStore')"); jdbcResourceManagerShardA.runSQLUpdate( - "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\") VALUES (4, 103, 1000, 'AppStore')"); + "INSERT INTO \"Orders\" (\"CustomerId\", \"OrderId\", \"OrderValue\", \"OrderSource\")" + + " VALUES (4, 103, 1000, 'AppStore')"); jdbcResourceManagerShardA.runSQLUpdate( "INSERT INTO \"AllDataTypes\" (\"id\", \"varchar_col\") VALUES (1, 'test1')"); jdbcResourceManagerShardA.runSQLUpdate( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index 63588e9e0a..7343305a4f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -158,7 +158,8 @@ public void setUp() throws IOException { sysBuilder.setDatabaseName("XE"); } oracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); @@ -176,7 +177,8 @@ public void setUp() throws IOException { builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); spannerResourceManager = setUpSpannerResourceManager(); gcsResourceManager = setUpSpannerITGcsResourceManager(); @@ -259,9 +261,11 @@ public String getDescription() { protected CheckResult check() { try { oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1," + + " 'John', 'Doe')"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2," + + " 'Alice', 'Johnson')"); try (java.sql.Connection conn = java.sql.DriverManager.getConnection( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index 0148d206c3..068b1ce298 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -216,13 +216,14 @@ private void simpleOracleToSpannerTest( cloudOracleSysUser = (CloudOracleResourceManager) - CloudOracleResourceManager.builder(testName) - .setUsername("sys as sysdba") - .setPassword(System.getProperty("cloudProxyPassword")) - .setDatabaseName("XE") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudProxyPassword")) + .setDatabaseName("XE") + .setHost(System.getProperty("hostIp")) + .setPort(1521)); String oracleUser = System.getProperty("cloudProxyUsername"); String oraclePassword = System.getProperty("cloudProxyPassword"); // setUpOracleUser(oracleUser, oraclePassword); @@ -230,13 +231,14 @@ private void simpleOracleToSpannerTest( cloudSqlResourceManager = (CloudOracleResourceManager) - CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("/XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521)); SpannerResourceManager.Builder spannerResourceManagerBuilder = SpannerResourceManager.builder(testName, PROJECT, REGION, spannerDialect) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index 7e5aad6d2a..4398f1e8ca 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -175,7 +175,7 @@ public void mixedMigrationTest() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); setUpOracleUser(oracleUser, oraclePassword); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index 710fc6e471..fb8f2ffab9 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -110,17 +110,22 @@ public void setUp() throws IOException { // Pre-insert books oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the Rings', 1)"); + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(1, 'The Lord of the" + + " Rings', 1)"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and Prejudice', 2)"); + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(2, 'Pride and" + + " Prejudice', 2)"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers Guide to the Galaxy', 3)"); + "INSERT INTO \"Books\" (\"id\", \"title\", \"author_id\") VALUES(3, 'The Hitchhikers" + + " Guide to the Galaxy', 3)"); // Pre-insert categories oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz', CURRENT_TIMESTAMP)"); + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(1, 'xyz'," + + " CURRENT_TIMESTAMP)"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc', CURRENT_TIMESTAMP)"); + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc'," + + " CURRENT_TIMESTAMP)"); flushOracleLogs(); @@ -218,9 +223,11 @@ public void migrationTestWithRenameAndDrops() { assertCategoryTableBackfillContents(); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def', CURRENT_TIMESTAMP)"); + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(3, 'def'," + + " CURRENT_TIMESTAMP)"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi', CURRENT_TIMESTAMP)"); + "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(4, 'ghi'," + + " CURRENT_TIMESTAMP)"); oracleResourceManager.runSQLUpdate( "UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); oracleResourceManager.runSQLUpdate("DELETE FROM \"Category\" WHERE \"category_id\"=1"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index 5c7a60fa6f..6fb03dadb6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -92,7 +92,8 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setPort(1521); sysBuilder.setUsername("system"); sysBuilder.setDatabaseName("/XEPDB1"); - cloudOracleSysUser = (CloudOracleResourceManager) sysBuilder.build(); + cloudOracleSysUser = + (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) @@ -174,9 +175,13 @@ protected String getDescription() { protected CheckResult check() { try { oracleResourceManager.runSQLUpdate( - "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02 20:00:00.000000')"); + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\")" + + " VALUES (1, TIMESTAMP '2024-02-02 10:00:00.000000', TIMESTAMP '2024-02-02" + + " 20:00:00.000000')"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\") VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03 06:00:00.000000')"); + "INSERT INTO \"DateData\" (\"id\", \"timestamp_column\", \"datetime_column\")" + + " VALUES (2, TIMESTAMP '2024-02-02 20:00:00.000000', TIMESTAMP '2024-02-03" + + " 06:00:00.000000')"); flushOracleRedoLogs(cloudOracleSysUser); return new CheckResult(true, "Data inserted successfully"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java index 69c9a20b85..470529c6de 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMax16KeyTablePerDatabaseIT.java @@ -369,7 +369,8 @@ private ConditionCheck writeJdbcData(Map>> cdcE sysBuilder.setDatabaseName("XE"); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager cloudOracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); flushOracleRedoLogs(cloudOracleSysUser); cloudOracleSysUser.cleanupAll(); } catch (Throwable e) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index 24277db8d5..02dd57b01b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -144,10 +144,12 @@ public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITB "CREATE TABLE T_SPRG_R8_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_8 NATIONAL CHAR(255))", "ALTER TABLE T_SPRG_R8_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R8_NATIONAL_C (ID, COL_8) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER VARYING(255))", + "CREATE TABLE T_SPRG_R9_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_9 NATIONAL CHARACTER" + + " VARYING(255))", "ALTER TABLE T_SPRG_R9_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R9_NATIONAL_C (ID, COL_9) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR VARYING(255))", + "CREATE TABLE T_SPRG_R10_NATIONAL_C (ID NUMBER PRIMARY KEY, COL_10 NATIONAL CHAR" + + " VARYING(255))", "ALTER TABLE T_SPRG_R10_NATIONAL_C ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R10_NATIONAL_C (ID, COL_10) VALUES (1, NULL)", "CREATE TABLE T_SPRG_R11_NUMBER (ID NUMBER PRIMARY KEY, COL_11 NUMBER)", @@ -192,10 +194,12 @@ public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITB "CREATE TABLE T_SPRG_R24_TIMESTAMP (ID NUMBER PRIMARY KEY, COL_24 TIMESTAMP)", "ALTER TABLE T_SPRG_R24_TIMESTAMP ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R24_TIMESTAMP (ID, COL_24) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME ZONE)", + "CREATE TABLE T_SPRG_R25_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_25 TIMESTAMP WITH TIME" + + " ZONE)", "ALTER TABLE T_SPRG_R25_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R25_TIMESTAMP_ (ID, COL_25) VALUES (1, NULL)", - "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL TIME ZONE)", + "CREATE TABLE T_SPRG_R26_TIMESTAMP_ (ID NUMBER PRIMARY KEY, COL_26 TIMESTAMP WITH LOCAL" + + " TIME ZONE)", "ALTER TABLE T_SPRG_R26_TIMESTAMP_ ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "INSERT INTO T_SPRG_R26_TIMESTAMP_ (ID, COL_26) VALUES (1, NULL)", "CREATE TABLE T_SPRG_R27_INTERVAL_Y (ID NUMBER PRIMARY KEY, DUMMY VARCHAR2(10))", @@ -410,7 +414,8 @@ public void setUp() throws IOException { sysBuilder.setDatabaseName("XE"); } oracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); @@ -429,7 +434,8 @@ public void setUp() throws IOException { builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); LOG.info("Setting up Spanner resource manager..."); spannerResourceManager = setUpSpannerResourceManager(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index 58688e3872..32efe60a46 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -105,13 +105,14 @@ public void setUp() throws IOException, InterruptedException { jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) - org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") - .setHost(System.getProperty("hostIp")) - .setPort(1521) - .build(); + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(testName) + .setUsername(oracleUser) + .setPassword(oraclePassword) + .setDatabaseName("/XEPDB1") + .setHost(System.getProperty("hostIp")) + .setPort(1521)); try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java index 4ba158d4c0..6dd2980bd2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -147,7 +147,8 @@ public void setUp() throws IOException { oracleResourceManager.getPassword()); PreparedStatement pstmt = conn.prepareStatement( - "INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\", \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { + "INSERT INTO \"LargeCell\" (\"id\", \"max_string_col_to_bytes\"," + + " \"max_string_col_to_str\") VALUES (1, ?, ?)"); ) { byte[] bytes = new byte[4000]; Arrays.fill(bytes, (byte) 'b'); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java index 352b3363ff..ae051c723e 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java @@ -322,7 +322,8 @@ protected CheckResult check() { conn.prepareStatement( "INSERT INTO " + tableName - + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?, ?, ?, ?)")) { + + " (ROW_ID, NAME, AGE, MEMBER, ENTRY_ADDED, LARGE_BLOB) VALUES (?, ?, ?," + + " ?, ?, ?)")) { for (Map values : rows) { pstmt.setLong(1, ((Number) values.get(COLUMNS.get(0))).longValue()); pstmt.setString(2, (String) values.get(COLUMNS.get(1))); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java index 0253ebef33..ecac812fad 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -317,13 +317,13 @@ private void assertAllDatatypeColumnsTableCdcContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column" - + ", tinyblob_column, blob_column, mediumblob_column" - + ", longblob_column, enum_column, bool_column, other_bool_column" - + ", varbinary_column, bit_column, decimal_column, text_column, binary_column" - + ", char_column, tinytext_column, mediumtext_column, longtext_column from AllDatatypeColumns")) + "select varchar_column, tinyint_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column," + + " tinyblob_column, blob_column, mediumblob_column, longblob_column," + + " enum_column, bool_column, other_bool_column, varbinary_column, bit_column," + + " decimal_column, text_column, binary_column, char_column, tinytext_column," + + " mediumtext_column, longtext_column from AllDatatypeColumns")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -396,12 +396,13 @@ private void assertAllDatatypeColumns2TableBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" - + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -473,7 +474,11 @@ private void assertAllDatatypeTransformationTableBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, binary_column, bit_column FROM AllDatatypeTransformation")) + "SELECT varchar_column, tinyint_column, text_column, date_column, int_column," + + " bigint_column, float_column, double_column, decimal_column," + + " datetime_column, timestamp_column, time_column, year_column, blob_column," + + " enum_column, bool_column, binary_column, bit_column FROM" + + " AllDatatypeTransformation")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -563,12 +568,13 @@ private void assertAllDatatypeColumns2TableCdcContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column" - + ", longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column, decimal_column from AllDatatypeColumns2")) + "select varchar_column, tinyint_column, text_column, date_column, smallint_column," + + " mediumint_column, int_column, bigint_column, float_column, double_column," + + " datetime_column, timestamp_column, time_column, year_column, char_column," + + " tinyblob_column, tinytext_column, blob_column, mediumblob_column," + + " mediumtext_column, longblob_column, longtext_column, enum_column," + + " bool_column, binary_column, varbinary_column, bit_column, decimal_column" + + " from AllDatatypeColumns2")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -599,8 +605,9 @@ private void assertDataTypeSizeConversionBackfillContents() { SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, float_column, decimal_column, char_column, bool_column" - + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsWithSizes")) + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsWithSizes")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -621,8 +628,9 @@ private void assertDatatypeColumnsReducedSizesBackfillContents() throws IOExcept SpannerAsserts.assertThatStructs( spannerResourceManager.runQuery( - "select varchar_column, float_column, decimal_column, char_column, bool_column" - + ", binary_column, varbinary_column, bit_column, from DatatypeColumnsReducedSizes")) + "select varchar_column, float_column, decimal_column, char_column, bool_column," + + " binary_column, varbinary_column, bit_column, from" + + " DatatypeColumnsReducedSizes")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index b1603b65b2..a9a91d5d8f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -102,7 +102,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); @@ -343,9 +343,11 @@ protected String getDescription() { protected CheckResult check() { if (!executed) { cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (1, 'Tester Kumar', 30, 0, 'A', TO_DATE('2023-01-01', 'YYYY-MM-DD'))"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (3, 'Tester Gupta', 50, 0, 'Z', TO_DATE('2023-06-07', 'YYYY-MM-DD'))"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -367,7 +369,8 @@ protected String getDescription() { protected CheckResult check() { if (!executed) { cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\") VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); + "INSERT INTO \"Users\"(\"id\",\"name\",\"age\",\"subscribed\",\"plan\",\"startDate\")" + + " VALUES (4, 'Tester', 38, 1, 'D', TO_DATE('2023-09-10', 'YYYY-MM-DD'))"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -389,9 +392,11 @@ protected String getDescription() { protected CheckResult check() { if (!executed) { cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1', TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (1, 'movie1'," + + " TO_TIMESTAMP('2023-01-01 12:12:12', 'YYYY-MM-DD HH24:MI:SS'), 12345.09876)"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2', TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); + "INSERT INTO \"Movie\"(\"id\",\"name\",\"startTime\",\"actor\") VALUES (2, 'movie2'," + + " TO_TIMESTAMP('2023-11-25 17:10:12', 'YYYY-MM-DD HH24:MI:SS'), 931.5123)"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); executed = true; } @@ -422,13 +427,17 @@ protected CheckResult check() { "INSERT INTO \"Authors\"(\"author_id\",\"name\") VALUES (4, 'a4')"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (1, 'Article001', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (2, 'Article002', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 1)"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (3, 'Article004', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 4)"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); + "INSERT INTO \"Articles\"(\"id\",\"name\",\"published_date\",\"author_id\") VALUES" + + " (4, 'Article005', TO_DATE('2024-01-01', 'YYYY-MM-DD'), 3)"); cloudSqlResourceManager.runSQLUpdate( "INSERT INTO \"Books\"(\"id\",\"title\",\"author_id\") VALUES (1, 'Book005', 3)"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index efb98da165..48daffde86 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -160,7 +160,8 @@ public void setUp() throws IOException { sysBuilder.setDatabaseName("XE"); } oracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); @@ -178,7 +179,8 @@ public void setUp() throws IOException { builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); @@ -265,9 +267,11 @@ public String getDescription() { protected CheckResult check() { try { oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1, 'John', 'Doe')"); + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (1," + + " 'John', 'Doe')"); oracleResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2, 'Alice', 'Johnson')"); + "INSERT INTO \"person1\" (\"ID\", \"first_name1\", \"last_name1\") VALUES (2," + + " 'Alice', 'Johnson')"); try (java.sql.Connection conn = java.sql.DriverManager.getConnection( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index 12cda5d180..d0feaf59b6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -95,7 +95,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index 9256957e15..e5158a5246 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -162,7 +162,8 @@ public void setUp() throws IOException { sysBuilder.setDatabaseName("XE"); } oracleSysUser = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) sysBuilder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(sysBuilder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); @@ -180,7 +181,8 @@ public void setUp() throws IOException { builder.setDatabaseName("/XEPDB1"); } oracleResourceManager = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) builder.build(); + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + new SpannerOracleResourceManager(builder); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); spannerResourceManager = setUpSpannerResourceManager(); @@ -420,16 +422,25 @@ protected CheckResult check() { try { // 1 oracleResourceManager.runSQLUpdate( - "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " - + "VALUES ('true', 1, 3.14, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 10)"); + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 1, 3.14, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 10)"); // 2 oracleResourceManager.runSQLUpdate( - "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " - + "VALUES ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 20)"); + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 2, 3.1415, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 20)"); // 3 oracleResourceManager.runSQLUpdate( - "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD, BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) " - + "VALUES ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768', TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20', 'YYYY-MM-DD'), 12345.1234, 30)"); + "INSERT INTO ALLTYPES (BOOL_FIELD, INT64_FIELD, FLOAT64_FIELD, STRING_FIELD," + + " BYTES_FIELD, TIMESTAMP_FIELD, DATE_FIELD, NUMERIC_FIELD, VAL) VALUES" + + " ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768'," + + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + + " 'YYYY-MM-DD'), 12345.1234, 30)"); flushOracleLogs(); executed = true; } catch (Exception e) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java index 232f039432..9d42d9ece1 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -375,15 +375,18 @@ protected CheckResult check() { try { cloudOracleResourceManager.runSQLUpdate( String.format( - "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of the Rings', 1)", + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (1, 'The Lord of" + + " the Rings', 1)", TABLE2)); cloudOracleResourceManager.runSQLUpdate( String.format( - "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and Prejudice', 2)", + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (2, 'Pride and" + + " Prejudice', 2)", TABLE2)); cloudOracleResourceManager.runSQLUpdate( String.format( - "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The Hitchhikers Guide to the Galaxy', 3)", + "INSERT INTO \"%s\" (\"id\", \"title\", \"author_id\") VALUES (3, 'The" + + " Hitchhikers Guide to the Galaxy', 3)", TABLE2)); cloudOracleResourceManager.runSQLUpdate("COMMIT"); flushOracleRedoLogs(cloudOracleSysUser); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 7a47cded21..97282a8913 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -109,7 +109,7 @@ public void setUp() throws Exception { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index f380230859..c649a19eaf 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -108,7 +108,7 @@ public void setUp() throws Exception { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); String oracleUser = "C##U" + RandomStringUtils.randomAlphanumeric(10).toUpperCase(); String oraclePassword = "A" + RandomStringUtils.randomAlphanumeric(10); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index f421e4d031..1970d6c24b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setDatabaseName("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); jdbcResourceManagerShardA = (CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index 148ee060a9..7fe0896a30 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -97,7 +97,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setDatabaseName("XE"); - cloudOracleSysUser = (CloudOracleResourceManager) builder.build(); + cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); cloudSqlResourceManager = (CloudOracleResourceManager) @@ -256,9 +256,11 @@ protected String getDescription() { protected CheckResult check() { if (!executed) { cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('John', 'Doe')"); + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('John', 'Doe')"); cloudSqlResourceManager.runSQLUpdate( - "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES ('Alice', 'Johnson')"); + "INSERT INTO \"person1\" (\"first_name1\", \"last_name1\") VALUES" + + " ('Alice', 'Johnson')"); cloudSqlResourceManager.runSQLUpdate("COMMIT"); flushOracleRedoLogs(cloudOracleSysUser); executed = true; diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java new file mode 100644 index 0000000000..5d62627edd --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.oracle; + +import org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager; +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * Custom class for Oracle implementations that require Service Name format (//host:port/Service) + * instead of the legacy SID format. Specifically built for Spanner integration tests hitting + * Pluggable Databases (PDBs) through GCP Datastream. + */ +public class SpannerOracleResourceManager extends CloudOracleResourceManager { + + public SpannerOracleResourceManager(CloudOracleResourceManager.Builder builder) { + super(builder); + } + + @Override + public synchronized @NonNull String getUri() { + return String.format( + "jdbc:%s:thin:@//%s:%d/%s", + getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); + } +} From f7726b0497c57a9d81aa473cdcf6a0054c8f1214 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Tue, 25 Aug 2026 13:35:42 +0000 Subject: [PATCH 06/13] Fix connection issue --- .../oracle/DataStreamToSpannerOracleRetryAllDLQIT.java | 4 ++-- .../oracle/DataStreamToSpannerOracleRetryDLQIT.java | 2 +- ...eamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java | 6 +++--- .../DataStreamToSpannerShardedOracleRetryAllDLQIT.java | 2 +- .../oracle/OracleDataStreamToSpannerFileOverridesIT.java | 4 ++-- .../v2/templates/oracle/OracleDataStreamToSpannerIT.java | 8 ++++---- .../oracle/OracleDataStreamToSpannerMixedIT.java | 2 +- .../oracle/OracleDataStreamToSpannerSessionIT.java | 4 ++-- .../oracle/OracleDataStreamToSpannerTimezoneIT.java | 2 +- .../oracle/OracleDatastreamToSpannerDataTypesIT.java | 4 ++-- ...acleDatastreamToSpannerSingleDFShardedMigrationIT.java | 2 +- .../OracleDatastreamToSpannerTableAndIndexLimitsIT.java | 2 +- .../oracle/OracleSeparateShadowTableDatabaseEventsIT.java | 4 ++-- .../OracleSeparateShadowTableDatabaseFileOverridesIT.java | 4 ++-- .../oracle/OracleSeparateShadowTableDatabaseMixedIT.java | 4 ++-- .../OracleSeparateShadowTableDatabasePKFocusedIT.java | 6 +++--- ...abaseShardedMigrationWithMigrationShardIdColumnIT.java | 4 ++-- ...seShardedMigrationWithoutMigrationShardIdColumnIT.java | 4 ++-- ...rateShadowTableDatabaseSingleDFShardedMigrationIT.java | 4 ++-- ...racleSeparateShadowTableDatabaseStringOverridesIT.java | 2 +- 20 files changed, 37 insertions(+), 37 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index 7d6f5382f8..d5d818680b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -91,7 +91,7 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setHost(sysUser.getHost()); sysBuilder.setPort(sysUser.getPort()); sysBuilder.setUsername("sys as sysdba"); - sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); sysBuilder.setDatabaseName(sysUser.getDatabaseName()); CloudOracleResourceManager trueSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); @@ -105,7 +105,7 @@ public void setUp() throws IOException, InterruptedException { builder.setPort(sysUser.getPort()); builder.setUsername(oracleUser); builder.setPassword(oraclePassword); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); jdbcResourceManager = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index 8c78e788c9..62af2599ae 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -88,7 +88,7 @@ private CloudOracleResourceManager setUpOracleResourceManagerForShard(String sha shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); shardBuilder.setHost(System.getProperty("hostIp")); shardBuilder.setPort(1521); - shardBuilder.setDatabaseName("/XEPDB1"); + shardBuilder.setDatabaseName("XEPDB1"); return new SpannerOracleResourceManager(shardBuilder); } else { return new SpannerOracleResourceManager( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index 0126f5892e..ca39bdc392 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -109,11 +109,11 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); - builder.setUsername(System.getProperty("cloudProxyUsername")); - builder.setDatabaseName("/XEPDB1"); + builder.setUsername(System.getProperty("cloudProxyUsername", "system")); + builder.setDatabaseName("XEPDB1"); } cloudOracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 4aed67db31..525c18a87c 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -95,7 +95,7 @@ public void setUp() throws IOException, InterruptedException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521)); try { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index 7343305a4f..5d534242a8 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -151,7 +151,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("hostIp")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); @@ -174,7 +174,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index 068b1ce298..b380488059 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -220,12 +220,12 @@ private void simpleOracleToSpannerTest( (CloudOracleResourceManager.Builder) CloudOracleResourceManager.builder(testName) .setUsername("sys as sysdba") - .setPassword(System.getProperty("cloudProxyPassword")) + .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) .setDatabaseName("XE") .setHost(System.getProperty("hostIp")) .setPort(1521)); - String oracleUser = System.getProperty("cloudProxyUsername"); - String oraclePassword = System.getProperty("cloudProxyPassword"); + String oracleUser = System.getProperty("cloudProxyUsername", "system"); + String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); // setUpOracleUser(oracleUser, oraclePassword); // setUpOracleUser(oracleUser, oraclePassword); @@ -236,7 +236,7 @@ private void simpleOracleToSpannerTest( CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521)); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index 4398f1e8ca..817628de70 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -185,7 +185,7 @@ public void mixedMigrationTest() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index fb8f2ffab9..7d08a83c02 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -138,8 +138,8 @@ public void setUp() throws IOException { JDBCSource jdbcSource = OracleSource.builder( System.getProperty("hostIp"), - System.getProperty("cloudProxyUsername"), - System.getProperty("cloudProxyPassword"), + System.getProperty("cloudProxyUsername", "system"), + System.getProperty("cloudProxyPassword", "TestPassword123"), 1521, oracleResourceManager.getDatabaseName()) .setAllowedTables( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index 6fb03dadb6..79b3652a6a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -91,7 +91,7 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setHost("" + System.getProperty("hostIp") + ""); sysBuilder.setPort(1521); sysBuilder.setUsername("system"); - sysBuilder.setDatabaseName("/XEPDB1"); + sysBuilder.setDatabaseName("XEPDB1"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index 02dd57b01b..409307fa7f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -407,7 +407,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("hostIp")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); @@ -431,7 +431,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index 32efe60a46..325f71c07c 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -110,7 +110,7 @@ public void setUp() throws IOException, InterruptedException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521)); try { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java index 6dd2980bd2..05a37071c9 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -167,7 +167,7 @@ public void setUp() throws IOException { + oracleResourceManager.getHost() + ":" + oracleResourceManager.getPort() - + "/XEPDB1", + + "XEPDB1", "system", "TestPassword123"); Statement stmt = conn.createStatement(); ) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index a9a91d5d8f..0c232367c5 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -98,7 +98,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); @@ -113,7 +113,7 @@ public void setUp() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index 48daffde86..4095849ce3 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -153,7 +153,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("hostIp")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); @@ -176,7 +176,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index d0feaf59b6..6a2c3e1ad6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -91,7 +91,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); @@ -106,7 +106,7 @@ public void setUp() throws IOException { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index e5158a5246..a8f977c50d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -155,7 +155,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("hostIp") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword")); + sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("hostIp")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); @@ -178,7 +178,7 @@ public void setUp() throws IOException { builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setUsername(oracleUser); - builder.setDatabaseName("/XEPDB1"); + builder.setDatabaseName("XEPDB1"); } oracleResourceManager = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) @@ -258,7 +258,7 @@ private void flushOracleLogs() { java.sql.DriverManager.getConnection( "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XE", "system", - System.getProperty("cloudProxyPassword")); + System.getProperty("cloudProxyPassword", "TestPassword123")); java.sql.Statement stmt = conn.createStatement()) { flushOracleRedoLogs(null); } catch (Exception e) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 97282a8913..4c539d30de 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); @@ -120,7 +120,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index c649a19eaf..65cd6761e2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -104,7 +104,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setSystemIdentifier("XE"); @@ -119,7 +119,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(oracleUser) .setPassword(oraclePassword) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index 1970d6c24b..0bd20b9a54 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -101,7 +101,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setDatabaseName("XE"); @@ -112,7 +112,7 @@ public void setUp() throws Exception { CloudOracleResourceManager.builder(testName) .setUsername(System.getProperty("cloudProxyUsername", "system")) .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) - .setDatabaseName("/XEPDB1") + .setDatabaseName("XEPDB1") .setHost(System.getProperty("hostIp")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index 7fe0896a30..bf5ce79aca 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -93,7 +93,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword")); + builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); builder.setHost(System.getProperty("hostIp")); builder.setPort(1521); builder.setDatabaseName("XE"); From 90868464e68e542f2b525eb66a005b8fd24b62f3 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 26 Aug 2026 05:33:48 +0000 Subject: [PATCH 07/13] Fix host env variable --- .../v2/templates/DataStreamToSpannerITBase.java | 8 ++++---- .../oracle/DataStreamToSpannerOracleDDLIT.java | 2 +- .../oracle/DataStreamToSpannerOracleRetryDLQIT.java | 4 ++-- ...ToSpannerOracleWideRowForMaxColumnsPerTablesIT.java | 4 ++-- .../DataStreamToSpannerShardedOracleRetryAllDLQIT.java | 4 ++-- .../DatastreamToSpannerReservedKeywordsOracleIT.java | 2 +- .../OracleDataStreamToSpannerFileOverridesIT.java | 10 +++++----- .../templates/oracle/OracleDataStreamToSpannerIT.java | 4 ++-- .../oracle/OracleDataStreamToSpannerMixedIT.java | 4 ++-- .../oracle/OracleDataStreamToSpannerSessionIT.java | 4 ++-- .../oracle/OracleDataStreamToSpannerTimezoneIT.java | 2 +- .../oracle/OracleDatastreamToSpannerDataTypesIT.java | 8 ++++---- ...eDatastreamToSpannerSingleDFShardedMigrationIT.java | 8 ++++---- .../oracle/OracleSeparateShadowTableDatabaseDDLIT.java | 2 +- .../OracleSeparateShadowTableDatabaseEventsIT.java | 4 ++-- ...acleSeparateShadowTableDatabaseFileOverridesIT.java | 10 +++++----- .../OracleSeparateShadowTableDatabaseMixedIT.java | 4 ++-- .../OracleSeparateShadowTableDatabasePKFocusedIT.java | 10 +++++----- ...seShardedMigrationWithMigrationShardIdColumnIT.java | 6 +++--- ...hardedMigrationWithoutMigrationShardIdColumnIT.java | 6 +++--- ...eShadowTableDatabaseSingleDFShardedMigrationIT.java | 6 +++--- ...leSeparateShadowTableDatabaseStringOverridesIT.java | 4 ++-- 22 files changed, 58 insertions(+), 58 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index d924d0dc48..937f431933 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -94,9 +94,9 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { setUpOracleResourceManager() { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(System.getProperty("cloudProxyUsername", "system")); builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); @@ -648,8 +648,8 @@ public void flushOracleRedoLogs( } } - if (!success && System.getProperty("hostIp") != null) { - String url = "jdbc:oracle:thin:@//" + System.getProperty("hostIp") + ":1521/XE"; + if (!success && System.getProperty("cloudOracleHost") != null) { + String url = "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost") + ":1521/XE"; String user = System.getProperty("cloudProxyUsername", "system"); String pass = System.getProperty("cloudProxyPassword", "TestPassword123"); try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java index b797ad3394..5d6a601efc 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -89,7 +89,7 @@ private void flushOracleLogs() { Class.forName("oracle.jdbc.OracleDriver"); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", + "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index 62af2599ae..e9281449bf 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -81,12 +81,12 @@ public class DataStreamToSpannerOracleRetryDLQIT extends DataStreamToSpannerITBa private static String streamNameA; private CloudOracleResourceManager setUpOracleResourceManagerForShard(String shardName) { - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { CloudOracleResourceManager.Builder shardBuilder = CloudOracleResourceManager.builder(testName + shardName); shardBuilder.setUsername(System.getProperty("cloudProxyUsername", "system")); shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - shardBuilder.setHost(System.getProperty("hostIp")); + shardBuilder.setHost(System.getProperty("cloudOracleHost")); shardBuilder.setPort(1521); shardBuilder.setDatabaseName("XEPDB1"); return new SpannerOracleResourceManager(shardBuilder); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index ca39bdc392..6491becb7b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -108,9 +108,9 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(System.getProperty("cloudProxyUsername", "system")); builder.setDatabaseName("XEPDB1"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 525c18a87c..157bed2ce0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -96,7 +96,7 @@ public void setUp() throws IOException, InterruptedException { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); @@ -129,7 +129,7 @@ public void setUp() throws IOException, InterruptedException { .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("hostIp"), oracleUser, oraclePassword, 1521, "XEPDB1") + System.getProperty("cloudOracleHost"), oracleUser, oraclePassword, 1521, "XEPDB1") .setAllowedTables( java.util.Map.of( oracleUser, java.util.List.of("Customers", "Orders", "AllDataTypes"))) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java index b2fbfeea0d..9b6c4c089f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -116,7 +116,7 @@ private void flushOracleLogs() { LOG.info("Flushing Oracle logs via raw JDBC..."); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index 5d534242a8..29a2711326 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -150,9 +150,9 @@ public void setUp() throws IOException { LOG.info("Setting up Oracle sys resource manager..."); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); @@ -169,9 +169,9 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(oraclePassword); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); @@ -270,7 +270,7 @@ protected CheckResult check() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( "jdbc:oracle:thin:@" - + System.getProperty("hostIp", "localhost") + + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index b380488059..a04c15de02 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -222,7 +222,7 @@ private void simpleOracleToSpannerTest( .setUsername("sys as sysdba") .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) .setDatabaseName("XE") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); String oracleUser = System.getProperty("cloudProxyUsername", "system"); String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); @@ -237,7 +237,7 @@ private void simpleOracleToSpannerTest( .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); SpannerResourceManager.Builder spannerResourceManagerBuilder = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index 817628de70..b919809370 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -172,7 +172,7 @@ public void mixedMigrationTest() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -186,7 +186,7 @@ public void mixedMigrationTest() throws IOException { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index 7d08a83c02..70ecb53d64 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -137,7 +137,7 @@ public void setUp() throws IOException { JDBCSource jdbcSource = OracleSource.builder( - System.getProperty("hostIp"), + System.getProperty("cloudOracleHost"), System.getProperty("cloudProxyUsername", "system"), System.getProperty("cloudProxyPassword", "TestPassword123"), 1521, @@ -191,7 +191,7 @@ public static void cleanUp() throws IOException { private void flushOracleLogs() { try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/FREE", + "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/FREE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index 79b3652a6a..fabdeed0e6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -88,7 +88,7 @@ public void setUp() throws IOException, InterruptedException { CloudOracleResourceManager.Builder sysBuilder = CloudOracleResourceManager.builder(testName); sysBuilder.setPassword("TestPassword123"); - sysBuilder.setHost("" + System.getProperty("hostIp") + ""); + sysBuilder.setHost("" + System.getProperty("cloudOracleHost") + ""); sysBuilder.setPort(1521); sysBuilder.setUsername("system"); sysBuilder.setDatabaseName("XEPDB1"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index 409307fa7f..a1a3615007 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -406,9 +406,9 @@ public void setUp() throws IOException { LOG.info("Setting up Oracle sys resource manager..."); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); @@ -426,9 +426,9 @@ public void setUp() throws IOException { // Build isolated RM org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(oraclePassword); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index 325f71c07c..e9b02187d0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -111,7 +111,7 @@ public void setUp() throws IOException, InterruptedException { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); try { jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); @@ -131,7 +131,7 @@ public void setUp() throws IOException, InterruptedException { .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("hostIp"), oracleUser, oraclePassword, 1521, "XEPDB1") + System.getProperty("cloudOracleHost"), oracleUser, oraclePassword, 1521, "XEPDB1") .setAllowedTables(java.util.Map.of(oracleUser, java.util.List.of("Users"))) .build(); @@ -277,11 +277,11 @@ private void insertDataInOracle() { // Force log file archive - needed so Datastream can see changes which are read from archived // log files. // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + - // System.getProperty("hostIp") + ":1521 (User: system, Pass: TestPassword123) calling ALTER + // System.getProperty("cloudOracleHost") + ":1521 (User: system, Pass: TestPassword123) calling ALTER // SYSTEM SWITCH LOGFILE. try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost") + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java index ecac812fad..8eee2892d2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -91,7 +91,7 @@ private void flushOracleLogs() { Class.forName("oracle.jdbc.OracleDriver"); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("hostIp", "localhost") + ":1521/XE", + "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index 0c232367c5..0e2d228555 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -99,7 +99,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -114,7 +114,7 @@ public void setUp() throws IOException { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index 4095849ce3..91f97ec2c0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -152,9 +152,9 @@ public void setUp() throws IOException { LOG.info("Setting up Oracle sys resource manager..."); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); @@ -171,9 +171,9 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(oraclePassword); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); @@ -276,7 +276,7 @@ protected CheckResult check() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( "jdbc:oracle:thin:@" - + System.getProperty("hostIp", "localhost") + + System.getProperty("cloudOracleHost", "localhost") + ":1521/XE", "system", "TestPassword123"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index 6a2c3e1ad6..6527eb07ac 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -92,7 +92,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -107,7 +107,7 @@ public void setUp() throws IOException { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index a8f977c50d..1558d87245 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -154,9 +154,9 @@ public void setUp() throws IOException { LOG.info("Setting up Oracle sys resource manager..."); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - sysBuilder.setHost(System.getProperty("hostIp")); + sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); sysBuilder.setDatabaseName("XE"); @@ -173,9 +173,9 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); - if (System.getProperty("hostIp") != null) { + if (System.getProperty("cloudOracleHost") != null) { builder.setPassword(oraclePassword); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setUsername(oracleUser); builder.setDatabaseName("XEPDB1"); @@ -256,7 +256,7 @@ public static void cleanUp() throws IOException { private void flushOracleLogs() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp") + ":1521/XE", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost") + ":1521/XE", "system", System.getProperty("cloudProxyPassword", "TestPassword123")); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 4c539d30de..557e675686 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -106,7 +106,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -121,7 +121,7 @@ public void setUp() throws Exception { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); @@ -367,7 +367,7 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index 65cd6761e2..b6d494a163 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -120,7 +120,7 @@ public void setUp() throws Exception { .setUsername(oracleUser) .setPassword(oraclePassword) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); @@ -361,7 +361,7 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index 0bd20b9a54..486b9c9e03 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -102,7 +102,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setDatabaseName("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -113,7 +113,7 @@ public void setUp() throws Exception { .setUsername(System.getProperty("cloudProxyUsername", "system")) .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) .setDatabaseName("XEPDB1") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); @@ -353,7 +353,7 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("hostIp", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index bf5ce79aca..734fbb0d51 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -94,7 +94,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); - builder.setHost(System.getProperty("hostIp")); + builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setDatabaseName("XE"); cloudOracleSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(builder); @@ -105,7 +105,7 @@ public void setUp() throws IOException { .setUsername(System.getProperty("cloudProxyUsername", "system")) .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) .setDatabaseName("XE") - .setHost(System.getProperty("hostIp")) + .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) .build(); From 1dc499d0e20e3689b62fe28070b5ebad90ad3a15 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 26 Aug 2026 06:05:33 +0000 Subject: [PATCH 08/13] Fix spotless --- .../templates/oracle/DataStreamToSpannerOracleDDLIT.java | 4 +++- .../DataStreamToSpannerShardedOracleRetryAllDLQIT.java | 6 +++++- .../DatastreamToSpannerReservedKeywordsOracleIT.java | 4 +++- .../oracle/OracleDataStreamToSpannerSessionIT.java | 4 +++- ...cleDatastreamToSpannerSingleDFShardedMigrationIT.java | 9 +++++++-- .../oracle/OracleSeparateShadowTableDatabaseDDLIT.java | 4 +++- ...baseShardedMigrationWithMigrationShardIdColumnIT.java | 4 +++- ...eShardedMigrationWithoutMigrationShardIdColumnIT.java | 4 +++- ...ateShadowTableDatabaseSingleDFShardedMigrationIT.java | 4 +++- 9 files changed, 33 insertions(+), 10 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java index 5d6a601efc..f66406f2bd 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -89,7 +89,9 @@ private void flushOracleLogs() { Class.forName("oracle.jdbc.OracleDriver"); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XE", + "jdbc:oracle:thin:@//" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 157bed2ce0..aade7141b5 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -129,7 +129,11 @@ public void setUp() throws IOException, InterruptedException { .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("cloudOracleHost"), oracleUser, oraclePassword, 1521, "XEPDB1") + System.getProperty("cloudOracleHost"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") .setAllowedTables( java.util.Map.of( oracleUser, java.util.List.of("Customers", "Orders", "AllDataTypes"))) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java index 9b6c4c089f..115db03fda 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -116,7 +116,9 @@ private void flushOracleLogs() { LOG.info("Flushing Oracle logs via raw JDBC..."); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index 70ecb53d64..566dc6262d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -191,7 +191,9 @@ public static void cleanUp() throws IOException { private void flushOracleLogs() { try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/FREE", + "jdbc:oracle:thin:@//" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/FREE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index e9b02187d0..efbf3f4f1f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -131,7 +131,11 @@ public void setUp() throws IOException, InterruptedException { .build(); org.apache.beam.it.gcp.datastream.OracleSource jdbcSource = org.apache.beam.it.gcp.datastream.OracleSource.builder( - System.getProperty("cloudOracleHost"), oracleUser, oraclePassword, 1521, "XEPDB1") + System.getProperty("cloudOracleHost"), + oracleUser, + oraclePassword, + 1521, + "XEPDB1") .setAllowedTables(java.util.Map.of(oracleUser, java.util.List.of("Users"))) .build(); @@ -277,7 +281,8 @@ private void insertDataInOracle() { // Force log file archive - needed so Datastream can see changes which are read from archived // log files. // Explicit constraint: Hard-boot raw JDBC strictly mapping to FREE at " + - // System.getProperty("cloudOracleHost") + ":1521 (User: system, Pass: TestPassword123) calling ALTER + // System.getProperty("cloudOracleHost") + ":1521 (User: system, Pass: TestPassword123) calling + // ALTER // SYSTEM SWITCH LOGFILE. try (java.sql.Connection conn = java.sql.DriverManager.getConnection( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java index 8eee2892d2..7b944f57fe 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -91,7 +91,9 @@ private void flushOracleLogs() { Class.forName("oracle.jdbc.OracleDriver"); try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XE", + "jdbc:oracle:thin:@//" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XE", "system", "TestPassword123"); Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 557e675686..75a374edd5 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -367,7 +367,9 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index b6d494a163..5875f63bf6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -361,7 +361,9 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index 486b9c9e03..c0565aef5f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -353,7 +353,9 @@ private void insertDataInOracle() { try (java.sql.Connection conn = java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost", "localhost") + ":1521/XEPDB1", + "jdbc:oracle:thin:@" + + System.getProperty("cloudOracleHost", "localhost") + + ":1521/XEPDB1", "system", "TestPassword123"); java.sql.Statement stmt = conn.createStatement()) { From 7abdfb1c9228fb6d6687552b2de273b48376646e Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 26 Aug 2026 07:42:12 +0000 Subject: [PATCH 09/13] Fix credential issue --- .../v2/templates/DataStreamToSpannerITBase.java | 10 +++++----- .../oracle/DataStreamToSpannerOracleRetryAllDLQIT.java | 6 +++--- .../oracle/DataStreamToSpannerOracleRetryDLQIT.java | 4 ++-- ...ToSpannerOracleWideRowForMaxColumnsPerTablesIT.java | 4 ++-- .../DataStreamToSpannerShardedOracleRetryAllDLQIT.java | 4 ++-- .../OracleDataStreamToSpannerFileOverridesIT.java | 2 +- .../templates/oracle/OracleDataStreamToSpannerIT.java | 6 +++--- .../oracle/OracleDataStreamToSpannerMixedIT.java | 2 +- .../oracle/OracleDataStreamToSpannerSessionIT.java | 4 ++-- .../oracle/OracleDatastreamToSpannerDataTypesIT.java | 2 +- ...eDatastreamToSpannerSingleDFShardedMigrationIT.java | 4 ++-- .../OracleSeparateShadowTableDatabaseEventsIT.java | 2 +- ...acleSeparateShadowTableDatabaseFileOverridesIT.java | 2 +- .../OracleSeparateShadowTableDatabaseMixedIT.java | 2 +- .../OracleSeparateShadowTableDatabasePKFocusedIT.java | 4 ++-- ...seShardedMigrationWithMigrationShardIdColumnIT.java | 2 +- ...hardedMigrationWithoutMigrationShardIdColumnIT.java | 2 +- ...eShadowTableDatabaseSingleDFShardedMigrationIT.java | 6 +++--- ...leSeparateShadowTableDatabaseStringOverridesIT.java | 6 +++--- 19 files changed, 37 insertions(+), 37 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index 937f431933..f1f83098f6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -94,15 +94,15 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { setUpOracleResourceManager() { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); + builder.maybeUseStaticInstance(); if (System.getProperty("cloudOracleHost") != null) { - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); - builder.setUsername(System.getProperty("cloudProxyUsername", "system")); + builder.setUsername(System.getProperty("cloudOracleUsername", "system")); builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); builder.setDatabaseName("XEPDB1"); } - builder.maybeUseStaticInstance(); return new com.google.cloud.teleport.v2.templates.oracle.SpannerOracleResourceManager(builder); } @@ -650,8 +650,8 @@ public void flushOracleRedoLogs( if (!success && System.getProperty("cloudOracleHost") != null) { String url = "jdbc:oracle:thin:@//" + System.getProperty("cloudOracleHost") + ":1521/XE"; - String user = System.getProperty("cloudProxyUsername", "system"); - String pass = System.getProperty("cloudProxyPassword", "TestPassword123"); + String user = System.getProperty("cloudOracleUsername", "system"); + String pass = System.getProperty("cloudOraclePassword", "TestPassword123"); try (java.sql.Connection conn = java.sql.DriverManager.getConnection(url, user, pass); java.sql.Statement stmt = conn.createStatement()) { stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index d5d818680b..8442636ac2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -91,13 +91,13 @@ public void setUp() throws IOException, InterruptedException { sysBuilder.setHost(sysUser.getHost()); sysBuilder.setPort(sysUser.getPort()); sysBuilder.setUsername("sys as sysdba"); - sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); sysBuilder.setDatabaseName(sysUser.getDatabaseName()); CloudOracleResourceManager trueSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); - String oracleUser = System.getProperty("cloudProxyUsername", "system"); - String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = CloudOracleResourceManager.builder(testName); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index e9281449bf..043f03ade7 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -84,8 +84,8 @@ private CloudOracleResourceManager setUpOracleResourceManagerForShard(String sha if (System.getProperty("cloudOracleHost") != null) { CloudOracleResourceManager.Builder shardBuilder = CloudOracleResourceManager.builder(testName + shardName); - shardBuilder.setUsername(System.getProperty("cloudProxyUsername", "system")); - shardBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + shardBuilder.setUsername(System.getProperty("cloudOracleUsername", "system")); + shardBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); shardBuilder.setHost(System.getProperty("cloudOracleHost")); shardBuilder.setPort(1521); shardBuilder.setDatabaseName("XEPDB1"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index 6491becb7b..85a778446b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -109,10 +109,10 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("cloudOracleHost") != null) { - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); - builder.setUsername(System.getProperty("cloudProxyUsername", "system")); + builder.setUsername(System.getProperty("cloudOracleUsername", "system")); builder.setDatabaseName("XEPDB1"); } cloudOracleResourceManager = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index aade7141b5..6a83421914 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -85,8 +85,8 @@ public void setUp() throws IOException, InterruptedException { spannerResourceManager = setUpSpannerResourceManager(); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - String oracleUser = System.getProperty("cloudProxyUsername", "system"); - String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index 29a2711326..aeafa9b6ab 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -151,7 +151,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("cloudOracleHost") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index a04c15de02..a50e8f599d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -220,12 +220,12 @@ private void simpleOracleToSpannerTest( (CloudOracleResourceManager.Builder) CloudOracleResourceManager.builder(testName) .setUsername("sys as sysdba") - .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) .setDatabaseName("XE") .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); - String oracleUser = System.getProperty("cloudProxyUsername", "system"); - String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); // setUpOracleUser(oracleUser, oraclePassword); // setUpOracleUser(oracleUser, oraclePassword); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index b919809370..d58109df0b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -171,7 +171,7 @@ public void mixedMigrationTest() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index 566dc6262d..f70d944f60 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -138,8 +138,8 @@ public void setUp() throws IOException { JDBCSource jdbcSource = OracleSource.builder( System.getProperty("cloudOracleHost"), - System.getProperty("cloudProxyUsername", "system"), - System.getProperty("cloudProxyPassword", "TestPassword123"), + System.getProperty("cloudOracleUsername", "system"), + System.getProperty("cloudOraclePassword", "TestPassword123"), 1521, oracleResourceManager.getDatabaseName()) .setAllowedTables( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index a1a3615007..13a3bcd18f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -407,7 +407,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("cloudOracleHost") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index efbf3f4f1f..f089c6a827 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -100,8 +100,8 @@ public void setUp() throws IOException, InterruptedException { if (jobInfo == null) { - String oracleUser = System.getProperty("cloudProxyUsername", "system"); - String oraclePassword = System.getProperty("cloudProxyPassword", "TestPassword123"); + String oracleUser = System.getProperty("cloudOracleUsername", "system"); + String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); jdbcResourceManagerShardA = (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index 0e2d228555..f63abb2e1c 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -98,7 +98,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index 91f97ec2c0..b5e7078aca 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -153,7 +153,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("cloudOracleHost") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index 6527eb07ac..fe907e3807 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -91,7 +91,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index 1558d87245..7906fb5022 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -155,7 +155,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder sysBuilder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); if (System.getProperty("cloudOracleHost") != null) { - sysBuilder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + sysBuilder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); sysBuilder.setHost(System.getProperty("cloudOracleHost")); sysBuilder.setPort(1521); sysBuilder.setUsername("sys as sysdba"); @@ -258,7 +258,7 @@ private void flushOracleLogs() { java.sql.DriverManager.getConnection( "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost") + ":1521/XE", "system", - System.getProperty("cloudProxyPassword", "TestPassword123")); + System.getProperty("cloudOraclePassword", "TestPassword123")); java.sql.Statement stmt = conn.createStatement()) { flushOracleRedoLogs(null); } catch (Exception e) { diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 75a374edd5..c988d13fd4 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index 5875f63bf6..64db3ce3d3 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -104,7 +104,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setSystemIdentifier("XE"); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index c0565aef5f..d6602fea9d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -101,7 +101,7 @@ public void setUp() throws Exception { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setDatabaseName("XE"); @@ -110,8 +110,8 @@ public void setUp() throws Exception { jdbcResourceManagerShardA = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(System.getProperty("cloudProxyUsername", "system")) - .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setUsername(System.getProperty("cloudOracleUsername", "system")) + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) .setDatabaseName("XEPDB1") .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index 734fbb0d51..ea30593086 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -93,7 +93,7 @@ public void setUp() throws IOException { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.setUsername("sys as sysdba"); - builder.setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")); + builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); builder.setDatabaseName("XE"); @@ -102,8 +102,8 @@ public void setUp() throws IOException { cloudSqlResourceManager = (CloudOracleResourceManager) CloudOracleResourceManager.builder(testName) - .setUsername(System.getProperty("cloudProxyUsername", "system")) - .setPassword(System.getProperty("cloudProxyPassword", "TestPassword123")) + .setUsername(System.getProperty("cloudOracleUsername", "system")) + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) .setDatabaseName("XE") .setHost(System.getProperty("cloudOracleHost")) .setPort(1521) From 38d4bbb930a9b7f9e4f199420d1677922a28f6a4 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Tue, 1 Sep 2026 11:20:35 +0000 Subject: [PATCH 10/13] Change parallalism to 1 for oracle ITs --- v2/datastream-to-spanner/pom.xml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/v2/datastream-to-spanner/pom.xml b/v2/datastream-to-spanner/pom.xml index 86d5ab887d..8791ceee55 100644 --- a/v2/datastream-to-spanner/pom.xml +++ b/v2/datastream-to-spanner/pom.xml @@ -144,5 +144,49 @@ + + templatesIntegrationTests + + false + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + default-test + + + **/*Oracle*IT.java + **/KafkaToKafkaIT.java + **/BigQueryAnomalyDetectionIT.java + + + + + oracle-integration-tests + + test + + test + + + **/*Oracle*IT.java + + + none + 1 + false + + + + + + + + From f53a7e0514dcf314ba36329dcb375f3b887d581b Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 2 Sep 2026 17:19:20 +0000 Subject: [PATCH 11/13] Revert "Change parallalism to 1 for oracle ITs" This reverts commit 38d4bbb930a9b7f9e4f199420d1677922a28f6a4. --- v2/datastream-to-spanner/pom.xml | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/v2/datastream-to-spanner/pom.xml b/v2/datastream-to-spanner/pom.xml index 8791ceee55..86d5ab887d 100644 --- a/v2/datastream-to-spanner/pom.xml +++ b/v2/datastream-to-spanner/pom.xml @@ -144,49 +144,5 @@ - - templatesIntegrationTests - - false - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - default-test - - - **/*Oracle*IT.java - **/KafkaToKafkaIT.java - **/BigQueryAnomalyDetectionIT.java - - - - - oracle-integration-tests - - test - - test - - - **/*Oracle*IT.java - - - none - 1 - false - - - - - - - - From 7fb169eba302fe5ea03c5bd1c27c02cc4a9745ca Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Fri, 4 Sep 2026 07:15:27 +0000 Subject: [PATCH 12/13] Removed alter commands for enabling all column logs --- .../DataStreamToSpannerOracleRetryAllDLQIT.java | 8 +------- .../oracle/DataStreamToSpannerOracleRetryDLQIT.java | 6 ------ ...SpannerOracleWideRowForMaxColumnsPerTablesIT.java | 2 -- ...ataStreamToSpannerShardedOracleRetryAllDLQIT.java | 7 ------- .../DatastreamToSpannerReservedKeywordsOracleIT.java | 3 +-- .../oracle/OracleDataStreamToSpannerIT.java | 2 -- .../oracle/OracleDataStreamToSpannerMixedIT.java | 6 ------ ...nerWideRowForMaxTableNameWithMaxColumnNameIT.java | 6 ------ ...atastreamToSpannerSingleDFShardedMigrationIT.java | 3 --- ...acleDatastreamToSpannerTableAndIndexLimitsIT.java | 12 ------------ ...ToSpannerWideRowForMax9MibTablePerDatabaseIT.java | 2 -- .../OracleSeparateShadowTableDatabaseEventsIT.java | 2 +- .../OracleSeparateShadowTableDatabaseMixedIT.java | 1 - .../OracleSeparateShadowTableDatabaseSessionIT.java | 4 ---- ...ShardedMigrationWithMigrationShardIdColumnIT.java | 6 ------ ...rdedMigrationWithoutMigrationShardIdColumnIT.java | 1 - ...hadowTableDatabaseSingleDFShardedMigrationIT.java | 1 - ...SeparateShadowTableDatabaseStringOverridesIT.java | 1 - 18 files changed, 3 insertions(+), 70 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index 8442636ac2..f10ff7c88a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -114,13 +114,7 @@ public void setUp() throws IOException, InterruptedException { executeSqlScript(jdbcResourceManager, ORACLE_SCHEMA_FILE_RESOURCE); // Add Supplemental Log Data required by Datastream CDC - List.of("Customers", "Orders", "AllDataTypes") - .forEach( - tableName -> { - jdbcResourceManager.runSQLUpdate( - String.format( - "ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); - }); + List.of("Customers", "Orders", "AllDataTypes").forEach(tableName -> {}); gcsResourceManager = setUpSpannerITGcsResourceManager(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index 043f03ade7..d88dcfc82d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -120,12 +120,6 @@ public void setUp() throws IOException, InterruptedException { } catch (Exception e) { } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); gcsResourceManager = setUpSpannerITGcsResourceManager(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java index 85a778446b..104f480a15 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleWideRowForMaxColumnsPerTablesIT.java @@ -177,8 +177,6 @@ private void setupSchema() { TABLE_NAMES.forEach( tableName -> { cloudOracleResourceManager.runSQLUpdate(getJDBCSchema(tableName)); - cloudOracleResourceManager.runSQLUpdate( - String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); }); createSpannerTables(); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 6a83421914..0fc32c5d8b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -112,13 +112,6 @@ public void setUp() throws IOException, InterruptedException { } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Customers\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Orders\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"AllDataTypes\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - gcsResourceManager = setUpSpannerITGcsResourceManager(); datastreamResourceManager = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java index 115db03fda..e79fbad723 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -138,8 +138,7 @@ public void testOracleReservedKeywords() throws Exception { LOG.info("Table true does not exist or could not be dropped: " + e.getMessage()); } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - oracleResourceManager.runSQLUpdate( - "ALTER TABLE \"true\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); + flushOracleLogs(); LOG.info("Creating Spanner DDL..."); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index a50e8f599d..7a8e1f4cd6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -255,8 +255,6 @@ private void simpleOracleToSpannerTest( tableNames.forEach( tableName -> { cloudSqlResourceManager.createTable(tableName, createJdbcSchema()); - cloudSqlResourceManager.runSQLUpdate( - String.format("ALTER TABLE %s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", tableName)); }); OracleSource jdbcSource = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index d58109df0b..206d4a78de 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -192,12 +192,6 @@ public void mixedMigrationTest() throws IOException { executeSqlScript( cloudSqlResourceManager, "oracle/OracleDataStreamToSpannerMixedIT/oracle-schema.sql"); - cloudSqlResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Authors")); - cloudSqlResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Books")); - cloudSqlResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", "Genre")); OracleSource jdbcSource = OracleSource.builder( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java index b5bc08bf27..5cf0b1c867 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java @@ -163,12 +163,6 @@ private void setupSchema() { TABLE_NAMES.forEach( tableName -> { cloudSqlResourceManager.runSQLUpdate(getJDBCSchema(tableName)); - cloudSqlResourceManager.runSQLUpdate( - "ALTER TABLE \"" - + oracleSchema - + "\".\"" - + tableName - + "\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); }); createSpannerTables(); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index f089c6a827..dd78ec9328 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -120,9 +120,6 @@ public void setUp() throws IOException, InterruptedException { executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - datastreamResourceManager = org.apache.beam.it.gcp.datastream.DatastreamResourceManager.builder( testName, PROJECT, REGION) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java index 05a37071c9..43122e94a8 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -124,18 +124,6 @@ public void setUp() throws IOException { LOG.info("Executing Oracle DDL script..."); executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - LOG.info("Adding supplemental logging to tables..."); - try { - oracleResourceManager.runSQLUpdate( - "ALTER TABLE \"LargeKey\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - } catch (Exception e) { - } - try { - oracleResourceManager.runSQLUpdate( - "ALTER TABLE \"LargeCell\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - } catch (Exception e) { - } - LOG.info("Creating Spanner DDL..."); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java index ae051c723e..981793cdd8 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerWideRowForMax9MibTablePerDatabaseIT.java @@ -161,8 +161,6 @@ private void setupSchema() { TABLE_NAMES.forEach( tableName -> { oracleResourceManager.createTable(tableName, createJdbcSchema()); - oracleResourceManager.runSQLUpdate( - "ALTER TABLE " + tableName + " ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); }); createSpannerTables(); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index f63abb2e1c..6bcdc90cd2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -231,7 +231,7 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); // Supplement logging requirement - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); + } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index fe907e3807..9e05f3db86 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -222,7 +222,6 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java index 9d42d9ece1..1c631ad5cf 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -106,10 +106,6 @@ public void setUp() throws IOException { } catch (Exception e) { } executeSqlScript(cloudOracleResourceManager, ORACLE_DDL_RESOURCE); - cloudOracleResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE1)); - cloudOracleResourceManager.runSQLUpdate( - String.format("ALTER TABLE \"%s\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS", TABLE2)); spannerResourceManager = setUpSpannerResourceManager(); shadowSpannerResourceManager = setUpShadowSpannerResourceManager(); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index c988d13fd4..8891a41729 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -128,11 +128,6 @@ public void setUp() throws Exception { executeSqlScript( jdbcResourceManagerShardA, "oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT/oracle-schema.sql"); - try { - jdbcResourceManagerShardA.runSQLUpdate( - "ALTER TABLE \"Users\" ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS"); - } catch (Exception e) { - } OracleSource jdbcSource = OracleSource.builder( @@ -279,7 +274,6 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index 64db3ce3d3..414c464c06 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -273,7 +273,6 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index d6602fea9d..17f978bace 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -266,7 +266,6 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index ea30593086..e889c85960 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -221,7 +221,6 @@ private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate("ALTER DATABASE ADD SUPPLEMENTAL LOG DATA"); } @AfterClass From cedad265605b644576d047524cd67e36052e88d7 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Fri, 4 Sep 2026 12:06:16 +0000 Subject: [PATCH 13/13] Removed few GRANT and cleaned some code --- .../templates/DataStreamToSpannerITBase.java | 27 ++++++- .../DataStreamToSpannerOracleDDLIT.java | 27 +------ ...ataStreamToSpannerOracleRetryAllDLQIT.java | 5 +- .../DataStreamToSpannerOracleRetryDLQIT.java | 3 - ...treamToSpannerOracleStringOverridesIT.java | 3 - ...amToSpannerShardedOracleRetryAllDLQIT.java | 19 +---- ...reamToSpannerReservedKeywordsOracleIT.java | 23 +----- ...cleDataStreamToSpannerFileOverridesIT.java | 55 +------------ .../oracle/OracleDataStreamToSpannerIT.java | 60 +------------- .../OracleDataStreamToSpannerIT.java.rej | 33 -------- .../OracleDataStreamToSpannerMixedIT.java | 60 +------------- .../OracleDataStreamToSpannerSessionIT.java | 25 +----- .../OracleDataStreamToSpannerTimezoneIT.java | 1 - ...RowForMaxTableNameWithMaxColumnNameIT.java | 5 +- .../OracleDatastreamToSpannerDataTypesIT.java | 55 +------------ ...amToSpannerSingleDFShardedMigrationIT.java | 1 - ...astreamToSpannerTableAndIndexLimitsIT.java | 2 - ...racleSeparateShadowTableDatabaseDDLIT.java | 27 +------ ...leSeparateShadowTableDatabaseEventsIT.java | 60 +------------- ...ateShadowTableDatabaseFileOverridesIT.java | 55 +------------ ...cleSeparateShadowTableDatabaseMixedIT.java | 60 +------------- ...eparateShadowTableDatabasePKFocusedIT.java | 80 ++----------------- ...eSeparateShadowTableDatabaseSessionIT.java | 46 +---------- ...MigrationWithMigrationShardIdColumnIT.java | 60 +------------- ...rationWithoutMigrationShardIdColumnIT.java | 60 +------------- ...bleDatabaseSingleDFShardedMigrationIT.java | 62 +------------- ...eShadowTableDatabaseStringOverridesIT.java | 61 +------------- .../oracle/SpannerOracleResourceManager.java | 42 ++++++++++ 28 files changed, 115 insertions(+), 902 deletions(-) delete mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index f1f83098f6..9fb272eaaa 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -96,10 +96,33 @@ public SpannerResourceManager setUpPGDialectSpannerResourceManager() { org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName); builder.maybeUseStaticInstance(); if (System.getProperty("cloudOracleHost") != null) { - builder.setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")); + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager sysdba = + (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) + org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.builder(testName) + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost")) + .setPort(1521) + .build(); + + String isoUser = + "U_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toUpperCase(); + String isoPassword = + "P_" + org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric(5).toUpperCase(); + + sysdba.runSQLUpdate( + String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", isoUser, isoPassword)); + sysdba.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", isoUser)); + sysdba.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", isoUser)); + sysdba.runSQLUpdate( + String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", isoUser)); + + builder.setPassword(isoPassword); builder.setHost(System.getProperty("cloudOracleHost")); builder.setPort(1521); - builder.setUsername(System.getProperty("cloudOracleUsername", "system")); + builder.setUsername(isoUser); builder.setSystemIdentifier(System.getProperty("cloudOracleSid", "XE")); builder.setDatabaseName("XEPDB1"); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java index f66406f2bd..93a8cccedd 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleDDLIT.java @@ -23,9 +23,6 @@ import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -84,24 +81,6 @@ public class DataStreamToSpannerOracleDDLIT extends DataStreamToSpannerITBase { public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; - private void flushOracleLogs() { - try { - Class.forName("oracle.jdbc.OracleDriver"); - try (Connection conn = - DriverManager.getConnection( - "jdbc:oracle:thin:@//" - + System.getProperty("cloudOracleHost", "localhost") - + ":1521/XE", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - } - } catch (Exception e) { - throw new RuntimeException("Failed to switch logfile", e); - } - } - @Before public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; @@ -140,7 +119,6 @@ public void setUp() throws IOException, InterruptedException { }; for (String table : tables) { try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); } catch (Exception e) { // Ignore if doesn't exist } @@ -148,12 +126,11 @@ public void setUp() throws IOException, InterruptedException { // Also drop sequences try { - oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); } catch (Exception e) { } executeSqlScript( oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-schema.sql"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); jobInfo = launchDataflowJob( @@ -221,7 +198,7 @@ protected CheckResult check() { executeSqlScript( oracleResourceManager, "oracle/DataStreamToSpannerOracleDDLIT/oracle-inserts.sql"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); return new CheckResult(true, "Success"); } catch (Exception e) { return new CheckResult(false, e.getMessage()); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java index f10ff7c88a..a379303394 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryAllDLQIT.java @@ -96,9 +96,8 @@ public void setUp() throws IOException, InterruptedException { CloudOracleResourceManager trueSysUser = (CloudOracleResourceManager) new SpannerOracleResourceManager(sysBuilder); - String oracleUser = System.getProperty("cloudOracleUsername", "system"); - String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); - + String oracleUser = sysUser.getUsername(); + String oraclePassword = sysUser.getPassword(); org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager.Builder builder = CloudOracleResourceManager.builder(testName); builder.setHost(sysUser.getHost()); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java index d88dcfc82d..77804088f9 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleRetryDLQIT.java @@ -108,15 +108,12 @@ public void setUp() throws IOException, InterruptedException { jdbcResourceManagerShardA = setUpOracleResourceManagerForShard("A"); try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) { } try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) { } try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) { } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java index 5a242b5f73..3d7de24578 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerOracleStringOverridesIT.java @@ -94,14 +94,11 @@ public void setUp() throws IOException { .replace("gs://" + gcsResourceManager.getBucket(), ""); try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch (Exception e) { } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); try { - oracleResourceManager.runSQLUpdate( - "GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL"); } catch (Exception e) { // Might exist or fail if we are not SYS, ignore wrapper } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java index 0fc32c5d8b..7571d073b2 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DataStreamToSpannerShardedOracleRetryAllDLQIT.java @@ -85,29 +85,16 @@ public void setUp() throws IOException, InterruptedException { spannerResourceManager = setUpSpannerResourceManager(); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); - String oracleUser = System.getProperty("cloudOracleUsername", "system"); - String oraclePassword = System.getProperty("cloudOraclePassword", "TestPassword123"); - - jdbcResourceManagerShardA = - (org.apache.beam.it.gcp.cloudsql.CloudOracleResourceManager) - new SpannerOracleResourceManager( - (CloudOracleResourceManager.Builder) - CloudOracleResourceManager.builder(testName) - .setUsername(oracleUser) - .setPassword(oraclePassword) - .setDatabaseName("XEPDB1") - .setHost(System.getProperty("cloudOracleHost")) - .setPort(1521)); + jdbcResourceManagerShardA = setUpOracleResourceManager(); + String oracleUser = jdbcResourceManagerShardA.getUsername(); + String oraclePassword = jdbcResourceManagerShardA.getPassword(); try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Customers\""); } catch (Exception e) { } try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Orders\""); } catch (Exception e) { } try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"AllDataTypes\""); } catch (Exception e) { } executeSqlScript(jdbcResourceManagerShardA, ORACLE_SCHEMA_FILE_RESOURCE); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java index e79fbad723..67c3a4c35d 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/DatastreamToSpannerReservedKeywordsOracleIT.java @@ -25,9 +25,6 @@ import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -112,34 +109,16 @@ public void cleanUp() { datastreamResourceManager); } - private void flushOracleLogs() { - LOG.info("Flushing Oracle logs via raw JDBC..."); - try (Connection conn = - DriverManager.getConnection( - "jdbc:oracle:thin:@" - + System.getProperty("cloudOracleHost", "localhost") - + ":1521/XEPDB1", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - LOG.info("Successfully switched Oracle log file."); - } catch (Exception e) { - LOG.error("Failed to switch Oracle logfile", e); - } - } - @Test public void testOracleReservedKeywords() throws Exception { LOG.info("Executing Oracle DDL script..."); try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"true\" CASCADE CONSTRAINTS PURGE"); } catch (Exception e) { LOG.info("Table true does not exist or could not be dropped: " + e.getMessage()); } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); LOG.info("Creating Spanner DDL..."); createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java index aeafa9b6ab..ed6b1478f0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerFileOverridesIT.java @@ -83,62 +83,11 @@ public class OracleDataStreamToSpannerFileOverridesIT extends DataStreamToSpanne private void setUpOracleUser(String user, String password) { oracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Before diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java index 7a8e1f4cd6..39b4a7a5ca 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java @@ -145,67 +145,11 @@ public void testDataStreamOracleToSpannerJson() throws IOException { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } private void simpleOracleToSpannerTest( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej deleted file mode 100644 index 3416446f98..0000000000 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java.rej +++ /dev/null @@ -1,33 +0,0 @@ ---- v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java -+++ v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerIT.java -@@ -134,16 +134,19 @@ - - private void setUpOracleUser(String user, String password) { - cloudOracleSysUser.runSQLUpdate( -- String.format("CREATE USER %s IDENTIFIED BY %s", user, password)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s", user)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s", user)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s", user)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s", user)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s", user)); -- cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s", user)); -- cloudOracleSysUser.runSQLUpdate( -- String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s", user)); -- cloudOracleSysUser.runSQLUpdate( -- String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s", user)); -+ String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); -+ cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - } - diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java index 206d4a78de..0f7426d0b0 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerMixedIT.java @@ -98,67 +98,11 @@ public void cleanUp() { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Test diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java index f70d944f60..4e8d6271e8 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerSessionIT.java @@ -23,10 +23,6 @@ import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -99,11 +95,9 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"Category\""); } catch (Exception e) { } try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"Books\""); } catch (Exception e) { } executeSqlScript(oracleResourceManager, ORACLE_DDL_RESOURCE); @@ -127,7 +121,7 @@ public void setUp() throws IOException { "INSERT INTO \"Category\" (\"category_id\", \"name\", \"last_update\") VALUES(2, 'abc'," + " CURRENT_TIMESTAMP)"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); datastreamResourceManager = DatastreamResourceManager.builder(testName, PROJECT, REGION) @@ -188,21 +182,6 @@ public static void cleanUp() throws IOException { oracleResourceManager); } - private void flushOracleLogs() { - try (Connection conn = - DriverManager.getConnection( - "jdbc:oracle:thin:@//" - + System.getProperty("cloudOracleHost", "localhost") - + ":1521/FREE", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement()) { - stmt.execute("ALTER SYSTEM SWITCH LOGFILE"); - } catch (SQLException e) { - throw new RuntimeException("Failed to flush Oracle logs", e); - } - } - @Test public void migrationTestWithRenameAndDrops() { ChainedConditionCheck conditionCheck = @@ -234,7 +213,7 @@ public void migrationTestWithRenameAndDrops() { "UPDATE \"Category\" SET \"name\"='abc1' WHERE \"category_id\"=2"); oracleResourceManager.runSQLUpdate("DELETE FROM \"Category\" WHERE \"category_id\"=1"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); conditionCheck = ChainedConditionCheck.builder( diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java index fabdeed0e6..c507370f36 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerTimezoneIT.java @@ -104,7 +104,6 @@ public void setUp() throws IOException, InterruptedException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"DateData\""); } catch (Exception e) { // ignore } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java index 5cf0b1c867..a8eef0d641 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDataStreamToSpannerWideRowForMaxTableNameWithMaxColumnNameIT.java @@ -106,7 +106,10 @@ public void setUp() throws IOException { .toUpperCase(); cloudSqlResourceManager.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s1A", oracleSchema, oracleSchema)); - cloudSqlResourceManager.runSQLUpdate(String.format("GRANT DBA TO %s", oracleSchema)); + cloudSqlResourceManager.runSQLUpdate( + String.format("GRANT DBA TO %s CONTAINER=ALL", oracleSchema)); + cloudSqlResourceManager.runSQLUpdate( + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", oracleSchema)); String sessionContent = generateBaseSchema(); sessionContent = sessionContent.replaceAll("SRC_DATABASE", oracleSchema); sessionContent = diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java index 13a3bcd18f..394166a219 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerDataTypesIT.java @@ -339,62 +339,11 @@ public class OracleDatastreamToSpannerDataTypesIT extends DataStreamToSpannerITB private void setUpOracleUser(String user, String password) { oracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Before diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java index dd78ec9328..71ff01c668 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerSingleDFShardedMigrationIT.java @@ -114,7 +114,6 @@ public void setUp() throws IOException, InterruptedException { .setHost(System.getProperty("cloudOracleHost")) .setPort(1521)); try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) { } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java index 43122e94a8..c9c375b245 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleDatastreamToSpannerTableAndIndexLimitsIT.java @@ -94,11 +94,9 @@ public void setUp() throws IOException { LOG.info("Oracle resource manager created with URI: {}", oracleResourceManager.getUri()); try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeKey\" CASCADE CONSTRAINTS"); } catch (Exception e) { } try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"LargeCell\" CASCADE CONSTRAINTS"); } catch (Exception e) { } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java index 7b944f57fe..5859ce3c95 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseDDLIT.java @@ -23,9 +23,6 @@ import com.google.cloud.teleport.v2.templates.DataStreamToSpanner; import com.google.cloud.teleport.v2.templates.DataStreamToSpannerITBase; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -86,24 +83,6 @@ public class OracleSeparateShadowTableDatabaseDDLIT extends DataStreamToSpannerI public static org.apache.beam.it.gcp.datastream.DatastreamResourceManager datastreamResourceManager; - private void flushOracleLogs() { - try { - Class.forName("oracle.jdbc.OracleDriver"); - try (Connection conn = - DriverManager.getConnection( - "jdbc:oracle:thin:@//" - + System.getProperty("cloudOracleHost", "localhost") - + ":1521/XE", - "system", - "TestPassword123"); - Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - } - } catch (Exception e) { - throw new RuntimeException("Failed to switch logfile", e); - } - } - @Before public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; @@ -143,7 +122,6 @@ public void setUp() throws IOException, InterruptedException { }; for (String table : tables) { try { - oracleResourceManager.runSQLUpdate("DROP TABLE \"" + table + "\" CASCADE CONSTRAINTS"); } catch (Exception e) { // Ignore if doesn't exist } @@ -151,13 +129,12 @@ public void setUp() throws IOException, InterruptedException { // Also drop sequences try { - oracleResourceManager.runSQLUpdate("DROP SEQUENCE sequence_id"); } catch (Exception e) { } executeSqlScript( oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-schema.sql"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); jobInfo = launchDataflowJob( @@ -238,7 +215,7 @@ protected CheckResult check() { oracleResourceManager, "oracle/OracleSeparateShadowTableDatabaseDDLIT/oracle-inserts.sql"); executed = true; // Repositioned ABOVE flushOracleLogs - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); } catch (Exception e) { return new CheckResult(false, e.getMessage()); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java index 6bcdc90cd2..99a36d1d5a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseEventsIT.java @@ -168,67 +168,11 @@ public void setUp() throws IOException { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); // Supplement logging requirement diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java index b5e7078aca..be4b6b0b85 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseFileOverridesIT.java @@ -85,62 +85,11 @@ public class OracleSeparateShadowTableDatabaseFileOverridesIT extends DataStream private void setUpOracleUser(String user, String password) { oracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Before diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java index 9e05f3db86..890acf04bf 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseMixedIT.java @@ -161,67 +161,11 @@ public void setUp() throws IOException { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java index 7906fb5022..ff99160cb1 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabasePKFocusedIT.java @@ -87,62 +87,11 @@ public class OracleSeparateShadowTableDatabasePKFocusedIT extends DataStreamToSp private void setUpOracleUser(String user, String password) { oracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + oracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); oracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON USERS CONTAINER=ALL", user)); - oracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @Before @@ -253,19 +202,6 @@ public static void cleanUp() throws IOException { datastreamResourceManager); } - private void flushOracleLogs() { - try (java.sql.Connection conn = - java.sql.DriverManager.getConnection( - "jdbc:oracle:thin:@" + System.getProperty("cloudOracleHost") + ":1521/XE", - "system", - System.getProperty("cloudOraclePassword", "TestPassword123")); - java.sql.Statement stmt = conn.createStatement()) { - flushOracleRedoLogs(null); - } catch (Exception e) { - LOG.warn("Log switch failed. Continuing...", e); - } - } - @Test public void migrationTestSimpleTable() throws Exception { ConditionCheck insertRecords = @@ -284,7 +220,7 @@ protected CheckResult check() { oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (1, 10)"); oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (2, 20)"); oracleResourceManager.runSQLUpdate("INSERT INTO MY_TABLE (ID, VAL) VALUES (3, 30)"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); @@ -309,7 +245,7 @@ protected CheckResult check() { try { oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET VAL = 10 WHERE ID = 2"); oracleResourceManager.runSQLUpdate("DELETE FROM MY_TABLE WHERE ID = 1"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); @@ -333,7 +269,7 @@ protected CheckResult check() { if (!executed) { try { oracleResourceManager.runSQLUpdate("UPDATE MY_TABLE SET ID = 10 WHERE ID = 3"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); @@ -441,7 +377,7 @@ protected CheckResult check() { + " ('true', 3, 3.14159, 'This is a test string for MySQL.', '564768'," + " TIMESTAMP '2024-12-20 10:30:00.00', TO_DATE('2024-12-20'," + " 'YYYY-MM-DD'), 12345.1234, 30)"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); @@ -467,7 +403,7 @@ protected CheckResult check() { oracleResourceManager.runSQLUpdate("DELETE FROM ALLTYPES WHERE INT64_FIELD = 1"); oracleResourceManager.runSQLUpdate( "UPDATE ALLTYPES SET VAL = 10 WHERE INT64_FIELD = 2"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); @@ -492,7 +428,7 @@ protected CheckResult check() { try { oracleResourceManager.runSQLUpdate( "UPDATE ALLTYPES SET INT64_FIELD = 10 WHERE INT64_FIELD = 3"); - flushOracleLogs(); + flushOracleRedoLogs(oracleResourceManager); executed = true; } catch (Exception e) { return new CheckResult(false, e.getMessage()); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java index 1c631ad5cf..ec5152212b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSessionIT.java @@ -154,54 +154,12 @@ public static void setUpOracleUser( CloudOracleResourceManager sysUser, String user, String password) { sysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); - sysUser.runSQLUpdate(String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); + sysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); sysUser.runSQLUpdate(String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - sysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); // Add c##datastream grants based on requirements - sysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO c##datastream CONTAINER=ALL")); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java index 8891a41729..af34bc2db1 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithMigrationShardIdColumnIT.java @@ -213,67 +213,11 @@ public void setUp() throws Exception { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java index 414c464c06..6db397af65 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseShardedMigrationWithoutMigrationShardIdColumnIT.java @@ -212,67 +212,11 @@ public void setUp() throws Exception { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java index 17f978bace..a780f59b45 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseSingleDFShardedMigrationIT.java @@ -118,12 +118,10 @@ public void setUp() throws Exception { .build(); try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) { } try { - jdbcResourceManagerShardA.runSQLUpdate("DROP TABLE \"Users\""); } catch (Exception e) { } executeSqlScript( @@ -205,67 +203,11 @@ public void setUp() throws Exception { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java index e889c85960..b308d14d51 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/OracleSeparateShadowTableDatabaseStringOverridesIT.java @@ -110,7 +110,6 @@ public void setUp() throws IOException { .build(); try { - cloudSqlResourceManager.runSQLUpdate("DROP TABLE \"person1\""); } catch (Exception e) { } executeSqlScript( @@ -160,67 +159,11 @@ public void setUp() throws IOException { private void setUpOracleUser(String user, String password) { cloudOracleSysUser.runSQLUpdate( String.format("CREATE USER %s IDENTIFIED BY %s CONTAINER=ALL", user, password)); + cloudOracleSysUser.runSQLUpdate(String.format("GRANT DBA TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE_CATALOG_ROLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CONNECT TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE SESSION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$DATABASE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PDBS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_SUPPLEMENTAL_LOGGING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_CONTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOG TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGFILE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$THREAD TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$PARAMETER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$NLS_PARAMETERS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TIMEZONE_NAMES TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$LOGMNR_LOGS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$ARCHIVE_DEST_STATUS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.V_$TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.DBA_REGISTRY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.OBJ$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON SYS.ENC$ TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT CREATE TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT UNLIMITED TABLESPACE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY DICTIONARY TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT SET CONTAINER TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT LOGMINING TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT EXECUTE ON DBMS_LOGMNR_D TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TRANSACTION TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ANY TABLE TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT SELECT ON DBA_EXTENTS TO %s CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate( - String.format("GRANT CREATE ANY TABLE TO %s CONTAINER=ALL", user)); + String.format("GRANT EXECUTE ON SYS.DBMS_LOGMNR TO %s CONTAINER=ALL", user)); cloudOracleSysUser.runSQLUpdate( String.format("ALTER USER %s QUOTA 50m ON SYSTEM CONTAINER=ALL", user)); - cloudOracleSysUser.runSQLUpdate(String.format("GRANT ALTER SYSTEM TO %s CONTAINER=ALL", user)); } @AfterClass diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java index 5d62627edd..7abb5030e7 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/oracle/SpannerOracleResourceManager.java @@ -25,6 +25,9 @@ */ public class SpannerOracleResourceManager extends CloudOracleResourceManager { + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(SpannerOracleResourceManager.class); + public SpannerOracleResourceManager(CloudOracleResourceManager.Builder builder) { super(builder); } @@ -35,4 +38,43 @@ public SpannerOracleResourceManager(CloudOracleResourceManager.Builder builder) "jdbc:%s:thin:@//%s:%d/%s", getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); } + + @Override + public void cleanupAll() { + super.cleanupAll(); + + String userToDrop = this.getUsername(); + if (userToDrop != null + && !userToDrop.equalsIgnoreCase("system") + && !userToDrop.contains("sysdba")) { + LOG.info("Attempting to dynamically drop isolated Oracle user: {}", userToDrop); + try { + CloudOracleResourceManager sysdba = + (CloudOracleResourceManager) + CloudOracleResourceManager.builder(this.getDatabaseName() + "_cleanup") + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost", this.getHost())) + .setPort(1521) + .build(); + + // Oracle Spanner implementation requires specific URI + sysdba = + new SpannerOracleResourceManager( + (CloudOracleResourceManager.Builder) + CloudOracleResourceManager.builder(this.getDatabaseName() + "_cleanup") + .setUsername("sys as sysdba") + .setPassword(System.getProperty("cloudOraclePassword", "TestPassword123")) + .setDatabaseName("XEPDB1") + .setHost(System.getProperty("cloudOracleHost", this.getHost())) + .setPort(1521)); + + sysdba.runSQLUpdate("DROP USER " + userToDrop + " CASCADE"); + LOG.info("Successfully dropped Oracle schema: {}", userToDrop); + } catch (Exception e) { + LOG.warn("Failed to drop schema: " + userToDrop, e); + } + } + } }