ADS-3567: Check 2.10 autotests

nifi-system-tests/nifi-system-test-suite/env/arenadata/.env

@@ -13,4 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ADB_IMAGE=hub.adsw.io/java/gpdb6-java-ubuntu:v1.2
+ADB_IMAGE=hub.adsw.io/java/gpdb6-java-ubuntu:v1.4
+NIFI_VERSION=
+NIFI_IMAGE_VERSION=

nifi-system-tests/nifi-system-test-suite/env/arenadata/docker-compose.yaml

@@ -175,7 +175,7 @@ services:
         - nifinet
 
     nifi01:
-        image: apache/nifi:${NIFI_VERSION:-2.5.0}
+        image: apache/nifi:${NIFI_IMAGE_VERSION}
         ports:
         - 9091:8080
         - 8001:8000 #debug port
@@ -200,12 +200,12 @@ services:
         volumes:
           - "nifi_node3_data:/data"
           - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar"
-          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar"
+          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar"
           - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh"
           - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh"
 
     nifi02:
-        image: apache/nifi:${NIFI_VERSION:-2.5.0}
+        image: apache/nifi:${NIFI_IMAGE_VERSION}
         ports:
           - 9092:8080
           - 8002:8000
@@ -230,12 +230,12 @@ services:
         volumes:
           - "nifi_node3_data:/data"
           - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar"
-          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar"
+          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar"
           - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh"
           - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh"
 
     nifi03:
-        image: apache/nifi:${NIFI_VERSION:-2.5.0}
+        image: apache/nifi:${NIFI_IMAGE_VERSION}
         ports:
           - 9093:8080
           - 8003:8000
@@ -260,7 +260,7 @@ services:
         volumes:
           - "nifi_node3_data:/data"
           - "../../target/postgresql-42.6.1.jar:/opt/postgresql-42.6.1.jar"
-          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION:-2.5.0}.nar"
+          - "../../target/nifi-greengage-service-nar-${NIFI_VERSION}.nar:/opt/nifi/nifi-current/lib/nifi-greengage-service-nar-${NIFI_VERSION}.nar"
           - "./nifi-cluster/conf/start.sh:/opt/nifi/scripts/start.sh"
           - "./prepare_nifi_configs.sh:/opt/nifi/nifi-current/prepare_nifi_configs.sh"
 

nifi-system-tests/nifi-system-test-suite/pom.xml

@@ -286,6 +286,34 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <id>update-env-nifi-image-version</id>
+                        <phase>process-resources</phase>
+                        <configuration>
+                            <target>
+                                <replaceregexp
+                                        file="${project.basedir}/env/arenadata/.env"
+                                        match="^NIFI_VERSION=.*$"
+                                        replace="NIFI_VERSION=${project.version}"
+                                        byline="true"/>
+                                <replaceregexp
+                                        file="${project.basedir}/env/arenadata/.env"
+                                        match="^NIFI_IMAGE_VERSION=.*$"
+                                        replace="NIFI_IMAGE_VERSION=${project.version}-maven"
+                                        byline="true"/>
+                            </target>
+                        </configuration>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     <dependencyManagement>

nifi-system-tests/nifi-system-test-suite/src/main/java/org/apache/nifi/tests/system/arenadata/model/TestConfig.java

@@ -27,6 +27,7 @@ public class TestConfig {
     private DataSourceProperties adb;
     private DataSourceProperties postgres;
     private String gpfdistPort;
+    private int clientApiPort;
     private int generalTimeout;
     private int pollInterval;
     private String dockerHostIp;

nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/GetGreengageRecordIT.java

@@ -305,7 +305,7 @@ private ControllerServiceEntity configureDbcpService(DataSourceProperties dsProp
         Map<String, String> dbcpServiceProperties = new HashMap<>();
         dbcpServiceProperties.put("Database Connection URL", dsProperties.getContainerAddress());
         dbcpServiceProperties.put("Database Driver Class Name", dsProperties.getDriverName());
-        dbcpServiceProperties.put("database-driver-locations", dsProperties.getDriverLocation());
+        dbcpServiceProperties.put("Database Driver Locations", dsProperties.getDriverLocation());
         dbcpServiceProperties.put("Database User", dsProperties.getUsername());
         if (dsProperties.getPassword() != null) {
             dbcpServiceProperties.put("Password", dsProperties.getPassword());
@@ -380,15 +380,15 @@ private ProcessorEntity configurePutDatabaseRecordProcessor(ControllerServiceEnt
         ProcessorEntity processor = getClientUtil().createProcessor(PUT_DATABASE_RECORD_PROCESSOR_CLASS_NAME,
                 NIFI_GROUP_ID, STANDARD_NAR_ARTIFACT, getNiFiVersion());
         Map<String, String> properties = new HashMap<>();
-        properties.put("put-db-record-dcbp-service", pgDbcpService.getId());
-        properties.put("put-db-record-record-reader", avroReaderService.getId());
-        properties.put("put-db-record-statement-type", "INSERT");
-        properties.put("put-db-record-table-name", PG_TABLE_NAME);
-        properties.put("put-db-record-schema-name", GG_SCHEMA_NAME);
-        properties.put("put-db-record-unmatched-column-behavior", "Ignore Unmatched Columns");
+        properties.put("Database Connection Pooling Service", pgDbcpService.getId());
+        properties.put("Record Reader", avroReaderService.getId());
+        properties.put("Statement Type", "INSERT");
+        properties.put("Table Name", PG_TABLE_NAME);
+        properties.put("Schema Name", GG_SCHEMA_NAME);
+        properties.put("Unmatched Column Behavior", "Ignore Unmatched Columns");
         getClientUtil().updateProcessorProperties(processor, properties);
         processor = getClientUtil().setAutoTerminatedRelationships(processor,
                 Set.of(RELATION_SUCCESS, RELATION_FAILURE, RELATION_RETRY));
         return processor;
     }
-}
+}

nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/NifiSystemContainerizedIT.java

@@ -19,7 +19,6 @@
 import io.qameta.allure.Step;
 import lombok.SneakyThrows;
 import lombok.val;
-import org.apache.nifi.tests.system.NiFiClientUtil;
 import org.apache.nifi.tests.system.NiFiSystemIT;
 import org.apache.nifi.tests.system.arenadata.model.Component;
 import org.apache.nifi.tests.system.arenadata.service.DockerComposeService;
@@ -38,8 +37,6 @@
 import org.slf4j.LoggerFactory;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
 
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
@@ -72,21 +69,17 @@ public static void setup() {
 
     @Override
     @BeforeEach
-    public void setup(final TestInfo testInfo) throws IOException, NoSuchAlgorithmException {
+    public void setup(final TestInfo testInfo) {
         super.testInfo = testInfo;
         final String testClassName = testInfo.getTestClass().map(Class::getSimpleName).orElse("<Unknown Test Class>");
         final String friendlyTestName = testClassName + ":" + testInfo.getDisplayName();
         logger.info("Beginning Test {}", friendlyTestName);
 
         Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader());
-        setupClient(9091);
-    }
-
-    protected void setupClient(final int apiPort) {
-        nifiClient = createClient(apiPort);
-        clientUtil = new NiFiClientUtil(nifiClient, getNiFiVersion(), getTestName());
+        setupClient();
     }
 
+    @Override
     protected NiFiClient createClient(final int port) {
         final NiFiClientConfig.Builder clientConfigBuilder = new NiFiClientConfig.Builder()
                 .baseUrl("http://localhost:" + port)
@@ -98,6 +91,11 @@ protected NiFiClient createClient(final int port) {
                 .build();
     }
 
+    @Override
+    protected int getClientApiPort() {
+        return getTestConfig().getClientApiPort();
+    }
+
     @SneakyThrows
     protected void enableControllerServiceAndWait(ControllerServiceEntity controllerServiceEntity) {
         getClientUtil().enableControllerService(controllerServiceEntity);

nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/arenadata/PutGreengageRecordIT.java

@@ -349,7 +349,7 @@ private ControllerServiceEntity configureDbcpService(DataSourceProperties dsProp
         Map<String, String> dbcpServiceProperties = new HashMap<>();
         dbcpServiceProperties.put("Database Connection URL", dsProperties.getContainerAddress());
         dbcpServiceProperties.put("Database Driver Class Name", dsProperties.getDriverName());
-        dbcpServiceProperties.put("database-driver-locations", dsProperties.getDriverLocation());
+        dbcpServiceProperties.put("Database Driver Locations", dsProperties.getDriverLocation());
         dbcpServiceProperties.put("Database User", dsProperties.getUsername());
         if (dsProperties.getPassword() != null) {
             dbcpServiceProperties.put("Password", dsProperties.getPassword());
@@ -391,7 +391,7 @@ private ProcessorEntity configureQueryDbTableProcessor(ControllerServiceEntity d
                 NIFI_GROUP_ID, QUERY_DB_TABLE_PROCESSOR_NAR_ARTIFACT, getNiFiVersion());
         Map<String, String> queryDbTableProperties = new HashMap<>();
         queryDbTableProperties.put("Database Connection Pooling Service", dbcpService.getId());
-        queryDbTableProperties.put("db-fetch-db-type", "PostgreSQL");
+        queryDbTableProperties.put("Database Type", "PostgreSQL");
         queryDbTableProperties.put("Table Name", PG_TABLE_NAME);
         queryDbTableProperties.put("Columns to Return", getFieldNamesString(fieldMap));
         if (fieldMap.containsKey(ID_COLUMN)) {

nifi-system-tests/nifi-system-test-suite/src/test/resources/arenadata/it-config.yaml

@@ -16,19 +16,20 @@
 adb:
   url: "jdbc:postgresql://localhost:6000/adb"
   driverName: "org.postgresql.Driver"
-  driverLocation: "/opt/postgresql-42.6.1.jar"
+  driverLocation: "file:/opt/postgresql-42.6.1.jar"
   username: gpadmin
   password: gpadmin
   containerAddress: "jdbc:postgresql://mdw:6000/adb"
 
 postgres:
   url: "jdbc:postgresql://localhost:5432/postgres"
   driverName: "org.postgresql.Driver"
-  driverLocation: "/opt/postgresql-42.6.1.jar"
+  driverLocation: "file:/opt/postgresql-42.6.1.jar"
   username: nifi
   password:
   containerAddress: "jdbc:postgresql://postgres:5432/postgres"
 
 gpfdistPort: 8099
+clientApiPort: 9091
 generalTimeout: 120
 pollInterval: 3