diff --git a/nifi-system-tests/nifi-system-test-suite/env/arenadata/.env b/nifi-system-tests/nifi-system-test-suite/env/arenadata/.env index 2483c97a0bd0..5f4e90c23920 100644 --- a/nifi-system-tests/nifi-system-test-suite/env/arenadata/.env +++ b/nifi-system-tests/nifi-system-test-suite/env/arenadata/.env @@ -13,4 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -ADB_IMAGE=hub.adsw.io/java/gpdb6-java-ubuntu:v1.2 \ No newline at end of file +ADB_IMAGE=hub.adsw.io/java/gpdb6-java-ubuntu:v1.4 +NIFI_VERSION= +NIFI_IMAGE_VERSION= \ No newline at end of file diff --git a/nifi-system-tests/nifi-system-test-suite/env/arenadata/docker-compose.yaml b/nifi-system-tests/nifi-system-test-suite/env/arenadata/docker-compose.yaml index 67ef0940c7cb..ebfae19d973f 100644 --- a/nifi-system-tests/nifi-system-test-suite/env/arenadata/docker-compose.yaml +++ b/nifi-system-tests/nifi-system-test-suite/env/arenadata/docker-compose.yaml @@ -175,7 +175,7 @@ services: - nifinet nifi01: - image: apache/nifi:${NIFI_VERSION:-2.5.0} + image: apache/nifi:${NIFI_IMAGE_VERSION} ports: - 9091:8080 - 8001:8000 #debug port @@ -200,12 +200,12 @@ services: volumes: - "nifi_node3_data:/data" - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar" - - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar" + - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar" - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh" - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh" nifi02: - image: apache/nifi:${NIFI_VERSION:-2.5.0} + image: apache/nifi:${NIFI_IMAGE_VERSION} ports: - 9092:8080 - 8002:8000 @@ -230,12 +230,12 @@ services: volumes: - "nifi_node3_data:/data" - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar" - - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar" + - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar" - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh" - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh" nifi03: - image: apache/nifi:${NIFI_VERSION:-2.5.0} + image: apache/nifi:${NIFI_IMAGE_VERSION} ports: - 9093:8080 - 8003:8000 @@ -260,7 +260,7 @@ services: volumes: - "nifi_node3_data:/data" - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar" - - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar" + - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar" - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh" - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh" diff --git a/nifi-system-tests/nifi-system-test-suite/pom.xml b/nifi-system-tests/nifi-system-test-suite/pom.xml index ae3d6c2e08ff..635d69efe26a 100644 --- a/nifi-system-tests/nifi-system-test-suite/pom.xml +++ b/nifi-system-tests/nifi-system-test-suite/pom.xml @@ -286,6 +286,34 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + update-env-nifi-image-version + process-resources + + + + + + + + run + + + + diff --git a/nifi-system-tests/nifi-system-test-suite/src/main/java/org/apache/nifi/tests/system/arenadata/model/TestConfig.java b/nifi-system-tests/nifi-system-test-suite/src/main/java/org/apache/nifi/tests/system/arenadata/model/TestConfig.java index b85cc5af590f..72b40c81bc69 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/main/java/org/apache/nifi/tests/system/arenadata/model/TestConfig.java +++ b/nifi-system-tests/nifi-system-test-suite/src/main/java/org/apache/nifi/tests/system/arenadata/model/TestConfig.java @@ -27,6 +27,7 @@ public class TestConfig { private DataSourceProperties adb; private DataSourceProperties postgres; private String gpfdistPort; + private int clientApiPort; private int generalTimeout; private int pollInterval; private String dockerHostIp; diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/GetGreengageRecordIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/GetGreengageRecordIT.java index 581b3ac09f59..e9f3d4bd457d 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/GetGreengageRecordIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/GetGreengageRecordIT.java @@ -305,7 +305,7 @@ private ControllerServiceEntity configureDbcpService(DataSourceProperties dsProp Map dbcpServiceProperties = new HashMap<>(); dbcpServiceProperties.put("Database Connection URL", dsProperties.getContainerAddress()); dbcpServiceProperties.put("Database Driver Class Name", dsProperties.getDriverName()); - dbcpServiceProperties.put("database-driver-locations", dsProperties.getDriverLocation()); + dbcpServiceProperties.put("Database Driver Locations", dsProperties.getDriverLocation()); dbcpServiceProperties.put("Database User", dsProperties.getUsername()); if (dsProperties.getPassword() != null) { dbcpServiceProperties.put("Password", dsProperties.getPassword()); @@ -380,15 +380,15 @@ private ProcessorEntity configurePutDatabaseRecordProcessor(ControllerServiceEnt ProcessorEntity processor = getClientUtil().createProcessor(PUT_DATABASE_RECORD_PROCESSOR_CLASS_NAME, NIFI_GROUP_ID, STANDARD_NAR_ARTIFACT, getNiFiVersion()); Map properties = new HashMap<>(); - properties.put("put-db-record-dcbp-service", pgDbcpService.getId()); - properties.put("put-db-record-record-reader", avroReaderService.getId()); - properties.put("put-db-record-statement-type", "INSERT"); - properties.put("put-db-record-table-name", PG_TABLE_NAME); - properties.put("put-db-record-schema-name", GG_SCHEMA_NAME); - properties.put("put-db-record-unmatched-column-behavior", "Ignore Unmatched Columns"); + properties.put("Database Connection Pooling Service", pgDbcpService.getId()); + properties.put("Record Reader", avroReaderService.getId()); + properties.put("Statement Type", "INSERT"); + properties.put("Table Name", PG_TABLE_NAME); + properties.put("Schema Name", GG_SCHEMA_NAME); + properties.put("Unmatched Column Behavior", "Ignore Unmatched Columns"); getClientUtil().updateProcessorProperties(processor, properties); processor = getClientUtil().setAutoTerminatedRelationships(processor, Set.of(RELATION_SUCCESS, RELATION_FAILURE, RELATION_RETRY)); return processor; } -} \ No newline at end of file +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/NifiSystemContainerizedIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/NifiSystemContainerizedIT.java index 61518508da1c..dac27049ca5d 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/NifiSystemContainerizedIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/NifiSystemContainerizedIT.java @@ -19,7 +19,6 @@ import io.qameta.allure.Step; import lombok.SneakyThrows; import lombok.val; -import org.apache.nifi.tests.system.NiFiClientUtil; import org.apache.nifi.tests.system.NiFiSystemIT; import org.apache.nifi.tests.system.arenadata.model.Component; import org.apache.nifi.tests.system.arenadata.service.DockerComposeService; @@ -38,8 +37,6 @@ import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.util.List; import java.util.Map; @@ -72,21 +69,17 @@ public static void setup() { @Override @BeforeEach - public void setup(final TestInfo testInfo) throws IOException, NoSuchAlgorithmException { + public void setup(final TestInfo testInfo) { super.testInfo = testInfo; final String testClassName = testInfo.getTestClass().map(Class::getSimpleName).orElse(""); final String friendlyTestName = testClassName + ":" + testInfo.getDisplayName(); logger.info("Beginning Test {}", friendlyTestName); Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); - setupClient(9091); - } - - protected void setupClient(final int apiPort) { - nifiClient = createClient(apiPort); - clientUtil = new NiFiClientUtil(nifiClient, getNiFiVersion(), getTestName()); + setupClient(); } + @Override protected NiFiClient createClient(final int port) { final NiFiClientConfig.Builder clientConfigBuilder = new NiFiClientConfig.Builder() .baseUrl("http://localhost:" + port) @@ -98,6 +91,11 @@ protected NiFiClient createClient(final int port) { .build(); } + @Override + protected int getClientApiPort() { + return getTestConfig().getClientApiPort(); + } + @SneakyThrows protected void enableControllerServiceAndWait(ControllerServiceEntity controllerServiceEntity) { getClientUtil().enableControllerService(controllerServiceEntity); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/PutGreengageRecordIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/PutGreengageRecordIT.java index bd511400e43a..1135ecabfaae 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/PutGreengageRecordIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/PutGreengageRecordIT.java @@ -349,7 +349,7 @@ private ControllerServiceEntity configureDbcpService(DataSourceProperties dsProp Map dbcpServiceProperties = new HashMap<>(); dbcpServiceProperties.put("Database Connection URL", dsProperties.getContainerAddress()); dbcpServiceProperties.put("Database Driver Class Name", dsProperties.getDriverName()); - dbcpServiceProperties.put("database-driver-locations", dsProperties.getDriverLocation()); + dbcpServiceProperties.put("Database Driver Locations", dsProperties.getDriverLocation()); dbcpServiceProperties.put("Database User", dsProperties.getUsername()); if (dsProperties.getPassword() != null) { dbcpServiceProperties.put("Password", dsProperties.getPassword()); @@ -391,7 +391,7 @@ private ProcessorEntity configureQueryDbTableProcessor(ControllerServiceEntity d NIFI_GROUP_ID, QUERY_DB_TABLE_PROCESSOR_NAR_ARTIFACT, getNiFiVersion()); Map queryDbTableProperties = new HashMap<>(); queryDbTableProperties.put("Database Connection Pooling Service", dbcpService.getId()); - queryDbTableProperties.put("db-fetch-db-type", "PostgreSQL"); + queryDbTableProperties.put("Database Type", "PostgreSQL"); queryDbTableProperties.put("Table Name", PG_TABLE_NAME); queryDbTableProperties.put("Columns to Return", getFieldNamesString(fieldMap)); if (fieldMap.containsKey(ID_COLUMN)) { diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/arenadata/it-config.yaml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/arenadata/it-config.yaml index fc1a125a77be..16d2e98bc0ff 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/arenadata/it-config.yaml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/arenadata/it-config.yaml @@ -16,7 +16,7 @@ adb: url: "jdbc:postgresql://localhost:6000/adb" driverName: "org.postgresql.Driver" - driverLocation: "/opt/postgresql-42.6.1.jar" + driverLocation: "file:/opt/postgresql-42.6.1.jar" username: gpadmin password: gpadmin containerAddress: "jdbc:postgresql://mdw:6000/adb" @@ -24,11 +24,12 @@ adb: postgres: url: "jdbc:postgresql://localhost:5432/postgres" driverName: "org.postgresql.Driver" - driverLocation: "/opt/postgresql-42.6.1.jar" + driverLocation: "file:/opt/postgresql-42.6.1.jar" username: nifi password: containerAddress: "jdbc:postgresql://postgres:5432/postgres" gpfdistPort: 8099 +clientApiPort: 9091 generalTimeout: 120 pollInterval: 3 \ No newline at end of file