-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Add Oracle Reverse ITs #4177
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Add Oracle Reverse ITs #4177
Changes from all commits
b8f81b5
89a565e
86a6aaa
0db9f6a
5926099
8bd93bd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,6 +54,9 @@ | |
| public abstract class SpannerToSourceDbITBase extends TemplateTestBase { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(SpannerToSourceDbITBase.class); | ||
| protected String testUsername; | ||
| protected String testUsernameShardA; | ||
| protected String testUsernameShardB; | ||
|
|
||
| protected SpannerResourceManager setUpSpannerResourceManager() { | ||
| return SpannerResourceManager.builder("rr-main-" + testName, PROJECT, REGION) | ||
|
|
@@ -166,6 +169,21 @@ private Shard createShardConfig(JDBCResourceManager jdbcResourceManager, String | |
| shard.setLogicalShardId(shardId); | ||
| shard.setUser(jdbcResourceManager.getUsername()); | ||
| shard.setPassword(jdbcResourceManager.getPassword()); | ||
| if (shardId.equals("Shard1") && testUsername != null) { | ||
| shard.setNamespace(testUsername); | ||
| shard.setUser(testUsername); | ||
| shard.setPassword("password"); | ||
| } else if (shardId.equals("shardA") && testUsernameShardA != null) { | ||
| shard.setNamespace(testUsernameShardA); | ||
| shard.setUser(testUsernameShardA); | ||
| shard.setPassword("password"); | ||
| } else if (shardId.equals("shardB") && testUsernameShardB != null) { | ||
| shard.setNamespace(testUsernameShardB); | ||
| shard.setUser(testUsernameShardB); | ||
| shard.setPassword("password"); | ||
| } else if (jdbcResourceManager instanceof org.apache.beam.it.jdbc.OracleResourceManager) { | ||
| shard.setNamespace(jdbcResourceManager.getUsername().toUpperCase()); | ||
| } | ||
| if (jdbcResourceManager instanceof org.apache.beam.it.jdbc.PostgresResourceManager pgRm) { | ||
| shard.setHost(pgRm.getHost()); | ||
| shard.setPort(String.valueOf(pgRm.getPort())); | ||
|
|
@@ -174,7 +192,14 @@ private Shard createShardConfig(JDBCResourceManager jdbcResourceManager, String | |
| shard.setHost(mySqlRm.getHost()); | ||
| shard.setPort(String.valueOf(mySqlRm.getPort())); | ||
| shard.setDbName(mySqlRm.getDatabaseName()); | ||
|
|
||
| } else if (jdbcResourceManager | ||
| instanceof org.apache.beam.it.jdbc.OracleResourceManager oracleRm) { | ||
| shard.setHost(oracleRm.getHost()); | ||
| shard.setPort(String.valueOf(oracleRm.getPort())); | ||
| shard.setDbName(oracleRm.getDatabaseName()); | ||
| } else { | ||
|
|
||
| throw new IllegalArgumentException("Unsupported JDBC resource manager type"); | ||
| } | ||
| return shard; | ||
|
|
@@ -299,7 +324,8 @@ public PipelineLauncher.LaunchInfo launchDataflowJob( | |
| && !Objects.equals( | ||
| sourceType, | ||
| com.google.cloud.teleport.v2.templates.constants.Constants | ||
| .SOURCE_POSTGRESQL)) | ||
| .SOURCE_POSTGRESQL) | ||
| && !Objects.equals(sourceType, "oracle")) | ||
| ? "input/cassandra-config.conf" | ||
| : "input/shard.json", | ||
| gcsResourceManager)); | ||
|
|
@@ -561,4 +587,104 @@ protected void createMySQLTableWithNColumns( | |
| throw new RuntimeException("Error executing DDL statement: " + ddl, e); | ||
| } | ||
| } | ||
|
|
||
| protected static String setupOracleIsolatedUser( | ||
| org.apache.beam.it.jdbc.JDBCResourceManager jdbcResourceManager) { | ||
| String username = | ||
| "REV_" | ||
| + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8).toUpperCase(); | ||
| LOG.info("Creating isolated Oracle user: {}", username); | ||
| jdbcResourceManager.runSQLUpdate("CREATE USER " + username + " IDENTIFIED BY password"); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This SQL statement is constructed using string concatenation with the |
||
| jdbcResourceManager.runSQLUpdate("GRANT ALL PRIVILEGES TO " + username); | ||
| jdbcResourceManager.runSQLUpdate("GRANT UNLIMITED TABLESPACE TO " + username); | ||
| jdbcResourceManager.runSQLUpdate("GRANT DBA TO " + username); | ||
| return username; | ||
| } | ||
|
|
||
| public static long runIsolatedGetRowCount( | ||
| org.apache.beam.it.jdbc.JDBCResourceManager manager, String testUsername, String tableName) { | ||
| String fullTableName = testUsername + "." + tableName; | ||
| return manager.getRowCount(fullTableName); | ||
| } | ||
|
|
||
| public static java.util.List<java.util.Map<String, Object>> runIsolatedReadTable( | ||
| org.apache.beam.it.jdbc.JDBCResourceManager manager, String testUsername, String tableName) { | ||
| String fullTableName = testUsername + "." + tableName; | ||
| return manager.readTable(fullTableName); | ||
| } | ||
|
|
||
| public static java.util.List<java.util.Map<String, Object>> runIsolatedSQLQuery( | ||
| org.apache.beam.it.jdbc.JDBCResourceManager jdbcResourceManager, | ||
| String testUsername, | ||
| String query) { | ||
| try (java.sql.Connection connection = | ||
| java.sql.DriverManager.getConnection( | ||
| jdbcResourceManager.getUri(), testUsername, "password"); | ||
| java.sql.Statement stmt = connection.createStatement()) { | ||
| if (!"SYSTEM".equalsIgnoreCase(testUsername) | ||
| && jdbcResourceManager instanceof org.apache.beam.it.jdbc.OracleResourceManager) { | ||
| stmt.execute("ALTER SESSION SET CURRENT_SCHEMA = " + testUsername); | ||
| } | ||
| java.util.List<java.util.Map<String, Object>> result = new java.util.ArrayList<>(); | ||
| try (java.sql.ResultSet rs = stmt.executeQuery(query)) { | ||
| java.sql.ResultSetMetaData md = rs.getMetaData(); | ||
| int columns = md.getColumnCount(); | ||
| while (rs.next()) { | ||
| java.util.Map<String, Object> row = new java.util.HashMap<>(columns); | ||
| for (int i = 1; i <= columns; ++i) { | ||
| row.put(md.getColumnName(i).toLowerCase(), rs.getObject(i)); | ||
| } | ||
| result.add(row); | ||
| } | ||
| } | ||
| return result; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("Error running isolated query", e); | ||
| } | ||
| } | ||
|
|
||
| protected void createOracleSchema( | ||
| org.apache.beam.it.jdbc.OracleResourceManager jdbcResourceManager, | ||
| String mySqlSchemaFile, | ||
| String targetUsername) | ||
| throws java.io.IOException { | ||
| String ddl = | ||
| String.join( | ||
| " ", | ||
| com.google.common.io.Resources.readLines( | ||
| com.google.common.io.Resources.getResource(mySqlSchemaFile), | ||
| java.nio.charset.StandardCharsets.UTF_8)); | ||
| ddl = ddl.replaceAll("\r\n", " ").replaceAll("\n", " "); | ||
| String[] ddls = ddl.split(";"); | ||
| try (java.sql.Connection connection = | ||
| java.sql.DriverManager.getConnection( | ||
| jdbcResourceManager.getUri(), targetUsername, "password"); | ||
| java.sql.Statement stmt = connection.createStatement()) { | ||
| if (!"SYSTEM".equalsIgnoreCase(targetUsername)) { | ||
| stmt.execute("ALTER SESSION SET CURRENT_SCHEMA = " + targetUsername); | ||
| } | ||
| for (String d : ddls) { | ||
| if (!d.trim().isEmpty() && !d.trim().toUpperCase().startsWith("SELECT")) { | ||
| try { | ||
| stmt.executeUpdate(d); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("Failed to execute schema DDL: " + d, e); | ||
| } | ||
| } | ||
| } | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("failed creating isolated oracle schema", e); | ||
| } | ||
| } | ||
|
|
||
| protected void createOracleTableWithNColumns( | ||
| org.apache.beam.it.jdbc.OracleResourceManager jdbcResourceManager, | ||
| String arg1, | ||
| int arg2, | ||
| String arg3) {} | ||
|
Comment on lines
+680
to
+684
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
|
|
||
| @org.junit.After | ||
| public void clearIsolatedUser() { | ||
| testUsername = null; | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The magic string "oracle" is used here. For consistency and maintainability, please use the existing constant
ORACLE_SOURCE_TYPEfromcom.google.cloud.teleport.v2.spanner.migrations.constants.Constants.