From a9fdf34170c9dd114efb1890d4104dddaef34b2b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 30 Oct 2025 10:14:58 +0200 Subject: [PATCH 1/3] [improve][test] Add filtering solution for fast selection of TestNG groups --- build/run_unit_group.sh | 63 +++- .../apache/pulsar/tests/TestNGTestFilter.java | 288 ++++++++++++++++++ pom.xml | 59 +++- 3 files changed, 404 insertions(+), 6 deletions(-) create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 9653a787325fa..546811c3496fc 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -56,10 +56,63 @@ function mvn_test() { else failfast_args="-DtestFailFast=false --fail-at-end" fi - echo "::group::Run tests for " "$@" - # use "verify" instead of "test" to workaround MDEP-187 issue in pulsar-functions-worker and pulsar-broker projects with the maven-dependency-plugin's copy goal - # Error message was "Artifact has not been packaged yet. When used on reactor artifact, copy should be executed after packaging: see MDEP-187" - $MVN_TEST_OPTIONS $failfast_args $clean_arg $target $coverage_arg "$@" "${COMMANDLINE_ARGS[@]}" + + local use_test_filtering=0 + local groups_arg="" + local in_pl_modules=0 + local pl_modules_args=() + local filtered_args=() + local skip_arg=0 + + for arg in "$@" "${COMMANDLINE_ARGS[@]}"; do + if [[ "$arg" =~ ^-Dgroups=(.+)$ ]]; then + use_test_filtering=1 + groups_arg="$arg" + skip_arg=1 + elif [[ "$arg" =~ ^-pl$ ]] || [[ "$arg" =~ ^--projects$ ]]; then + in_pl_modules=1 + pl_modules_args+=("$arg") + elif [[ "$arg" =~ ^-pl=(.+)$ ]] || [[ "$arg" =~ ^--projects=(.+)$ ]]; then + pl_modules_args+=("$arg") + elif [[ "$arg" =~ ^-DexcludedGroups=(.+)$ ]]; then + skip_arg=1 + elif [ $in_pl_modules -eq 1 ]; then + pl_modules_args+=("$arg") + in_pl_modules=0 + fi + if [ $skip_arg -eq 0 ]; then + filtered_args+=("$arg") + else + skip_arg=0 + fi + done + + if [ $use_test_filtering -eq 1 ]; then + echo "::group::Creating test filters for groups: $groups_value" + # First, create the test filters using the testFilterCreate profile + local create_filter_targets=() + if [[ -n "$clean_arg" ]]; then + create_filter_targets+=("$clean_arg") + fi + # handle the case where the classfiles haven't been compiled yet + if [[ target == "install" ]]; then + create_filter_targets+=("$target" "-DskipTests") + # no need to run install for test run + target="verify" + fi + create_filter_targets+=("exec:exec") + $MVN_TEST_OPTIONS -PtestFilterCreate "${pl_modules_args[@]}" "${create_filter_targets[@]}" "${groups_arg}" + echo "::endgroup::" + + echo "::group::Run tests for ${filtered_args[*]} with test filtering" + # Then run tests with testFiltering profile instead of -Dgroups + $MVN_TEST_OPTIONS -PtestFiltering $failfast_args $target $coverage_arg "${filtered_args[@]}" + else + echo "::group::Run tests for " "$@" + # use "verify" instead of "test" to workaround MDEP-187 issue in pulsar-functions-worker and pulsar-broker projects with the maven-dependency-plugin's copy goal + # Error message was "Artifact has not been packaged yet. When used on reactor artifact, copy should be executed after packaging: see MDEP-187" + $MVN_TEST_OPTIONS $failfast_args $clean_arg $target $coverage_arg "$@" "${COMMANDLINE_ARGS[@]}" + fi echo "::endgroup::" set +x "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports @@ -82,7 +135,7 @@ function test_group_broker_group_1() { } function test_group_broker_group_2() { - mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' + mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other,_default' } function test_group_broker_group_3() { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java b/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java new file mode 100644 index 0000000000000..6421f8c935593 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import net.bytebuddy.description.annotation.AnnotationDescription; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; + +/** + * Utility class to filter TestNG tests based on groups and generate + * Maven Surefire includes/excludes files. + * + * This tool parses Java test class files using ByteBuddy to extract TestNG + * @Test annotations and their groups, then generates two files: + * - includes file: Contains test classes/methods that match the specified groups + * - excludes file: Contains test methods to exclude from classes that have matching tests + * + * The tests must be compiled before running this tool. There are "testFilterCreate" and "testFiltering" profiles + * in apache/pulsar pom.xml that are used together with this tool. + * + * Creating includes/excludes files for "some-group" and "some-group2" + * mvn -PtestFilterCreate exec:exec -Dgroups=some-group,some-group2 + * + * After this, it's possible to run tests with: + * mvn -PtestFiltering test + * + * Commonly the tests are targeted to run for specific modules by passing "-pl module1[,module2,module3,...]" to both + * commands. + */ +public class TestNGTestFilter { + + private static final String TEST_ANNOTATION = "org.testng.annotations.Test"; + + /** + * Filters TestNG tests based on groups and generates include/exclude files. + * + * @param testClassDirectory Directory containing compiled test class files + * @param includedGroups Set of groups to include in the test run + * @param includesOutputFile Path to the output file for included tests + * @param excludesOutputFile Path to the output file for excluded tests + * @throws IOException if there's an error reading class files or writing output files + */ + public static void generateTestFilters( + String testClassDirectory, + Set includedGroups, + String includesOutputFile, + String excludesOutputFile) throws IOException { + + Path classDir = Paths.get(testClassDirectory); + + if (!Files.exists(classDir) || !Files.isDirectory(classDir)) { + throw new IllegalArgumentException( + "Invalid test class directory: " + testClassDirectory + " absolute path: " + + classDir.toAbsolutePath()); + } + + List includes = new ArrayList<>(); + List excludes = new ArrayList<>(); + + // Find all .class files recursively + try (Stream paths = Files.walk(classDir)) { + List classFiles = paths + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".class")) + .collect(Collectors.toList()); + + // Create TypePool for class file analysis + TypePool typePool = TypePool.Default.ofSystemLoader(); + + for (Path classFile : classFiles) { + processClassFile(classFile, classDir, typePool, includedGroups, includes, excludes); + } + } + + // Write output files + if (includes.isEmpty()) { + includes.add(""); + excludes.add("**/*"); + } + writeOutputFile(includesOutputFile, includes); + writeOutputFile(excludesOutputFile, excludes); + } + + /** + * Process a single class file to determine which tests to include/exclude. + */ + private static void processClassFile( + Path classFile, + Path baseDir, + TypePool typePool, + Set includedGroups, + List includes, + List excludes) { + + try { + // Convert file path to fully qualified class name + String relativePath = baseDir.relativize(classFile).toString(); + String className = relativePath + .replace(File.separatorChar, '.') + .replace(".class", ""); + + TypeDescription typeDescription = typePool.describe(className).resolve(); + + // Get class-level @Test annotation groups + Set classLevelGroups = getClassLevelGroups(typeDescription); + + // Track methods that match and don't match + List matchingMethods = new ArrayList<>(); + List nonMatchingMethods = new ArrayList<>(); + + Set handledMethods = new HashSet<>(); + // Check each method for @Test annotation + TypeDescription currentType = typeDescription; + while (currentType != null) { + for (MethodDescription.InDefinedShape method : currentType.getDeclaredMethods()) { + if (isTestMethod(method) && !handledMethods.contains(method.getName())) { + Set methodGroups = getTestGroups(method.getDeclaredAnnotations()); + + // Merge class-level and method-level groups + Set effectiveGroups = new HashSet<>(); + if (!methodGroups.isEmpty()) { + effectiveGroups.addAll(methodGroups); + } else if (!classLevelGroups.isEmpty()) { + effectiveGroups.addAll(classLevelGroups); + } else { + effectiveGroups.add("_default"); + } + + // Check if any group matches included groups + boolean matches = effectiveGroups.stream().anyMatch(includedGroups::contains); + + if (matches) { + matchingMethods.add(method.getName()); + } else { + nonMatchingMethods.add(method.getName()); + } + handledMethods.add(method.getName()); + } + } + currentType = currentType.getSuperClass() != null ? currentType.getSuperClass().asErasure() : null; + } + + // Determine output based on matching methods + if (!matchingMethods.isEmpty()) { + if (classLevelGroups.stream().anyMatch(includedGroups::contains) + || classLevelGroups.isEmpty() && includedGroups.contains("_default")) { + // Include entire class + includes.add(className); + // Exclude specific methods + for (String method : nonMatchingMethods) { + excludes.add(className + "#" + method); + } + } else { + // Some methods match - include specific methods + for (String method : matchingMethods) { + includes.add(className + "#" + method); + } + } + } + + } catch (Exception e) { + System.err.println("Warning: Could not process class file " + classFile + ": " + e.getMessage()); + } + } + + private static Set getClassLevelGroups(TypeDescription typeDescription) { + Set classLevelGroups = getTestGroups(typeDescription.getDeclaredAnnotations()); + TypeDescription.Generic superClass = typeDescription.getSuperClass(); + while (classLevelGroups.isEmpty() && superClass != null) { + classLevelGroups = getTestGroups(superClass.asErasure().getDeclaredAnnotations()); + superClass = superClass.getSuperClass(); + } + return classLevelGroups; + } + + /** + * Check if a method has @Test annotation. + */ + private static boolean isTestMethod(MethodDescription method) { + for (AnnotationDescription annotation : method.getDeclaredAnnotations()) { + if (annotation.getAnnotationType().getName().equals(TEST_ANNOTATION)) { + return true; + } + } + return false; + } + + /** + * Extract groups from @Test annotation. + */ + private static Set getTestGroups(Iterable annotations) { + Set groups = new HashSet<>(); + + for (AnnotationDescription annotation : annotations) { + if (annotation.getAnnotationType().getName().equals(TEST_ANNOTATION)) { + // Get the groups attribute + Object groupsValue = annotation.getValue("groups").resolve(); + if (groupsValue instanceof String[]) { + groups.addAll(Arrays.asList((String[]) groupsValue)); + } else if (groupsValue instanceof String) { + groups.add((String) groupsValue); + } + } + } + + return groups; + } + + /** + * Write lines to output file. + */ + private static void writeOutputFile(String filePath, List lines) throws IOException { + Path path = Paths.get(filePath); + + // Create parent directories if they don't exist + if (path.getParent() != null) { + Files.createDirectories(path.getParent()); + } + + Files.write(path, lines); + } + + /** + * Example usage of the utility. + */ + public static void main(String[] args) { + Set includedGroups; + String testClassDirectory = "target/test-classes"; + String includesOutputFile = "target/surefire-includes.txt"; + String excludesOutputFile = "target/surefire-excludes.txt"; + + if (args.length == 0) { + System.out.println("Usage: TestNGTestFilter "); + System.exit(1); + } + + includedGroups = new HashSet<>(Arrays.asList(args[0].split(","))); + if (args.length > 1) { + testClassDirectory = args[1]; + } + if (args.length > 2) { + includesOutputFile = args[2]; + } + if (args.length > 3) { + excludesOutputFile = args[3]; + } + + try { + generateTestFilters(testClassDirectory, includedGroups, includesOutputFile, excludesOutputFile); + System.out.println("Test filter files generated successfully:"); + System.out.println(" Includes: " + includesOutputFile); + System.out.println(" Excludes: " + excludesOutputFile); + } catch (IOException e) { + System.err.println("Error generating test filters: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1efa8760f7e0e..793469255edac 100644 --- a/pom.xml +++ b/pom.xml @@ -336,7 +336,7 @@ flexible messaging model and an intuitive client API. 0.6.1 - 3.0.0 + 3.6.2 5.0.0 1.0 3.3.0 @@ -3110,6 +3110,63 @@ flexible messaging model and an intuitive client API. false + + testFiltering + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + ${project.build.directory}/surefire-includes.txt + ${project.build.directory}/surefire-excludes.txt + + + + + + + + testFilterCreate + + + + + org.codehaus.mojo + exec-maven-plugin + + test + ${project.basedir} + java + + -classpath + + org.apache.pulsar.tests.TestNGTestFilter + ${groups} + + + + + + From b443f792d4abf57a478a9834fe027e9fecf23025 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 30 Oct 2025 13:21:35 +0200 Subject: [PATCH 2/3] Fix checkstyle --- .../main/java/org/apache/pulsar/tests/TestNGTestFilter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java b/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java index 6421f8c935593..1a0842604a10e 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/TestNGTestFilter.java @@ -40,7 +40,7 @@ * Maven Surefire includes/excludes files. * * This tool parses Java test class files using ByteBuddy to extract TestNG - * @Test annotations and their groups, then generates two files: + * Test annotations and their groups, then generates two files: * - includes file: Contains test classes/methods that match the specified groups * - excludes file: Contains test methods to exclude from classes that have matching tests * @@ -259,7 +259,9 @@ public static void main(String[] args) { String excludesOutputFile = "target/surefire-excludes.txt"; if (args.length == 0) { - System.out.println("Usage: TestNGTestFilter "); + System.out.println( + "Usage: TestNGTestFilter " + + ""); System.exit(1); } From b8413d0742880045eb1ed08bb364275b40e9ff64 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 30 Oct 2025 13:43:02 +0200 Subject: [PATCH 3/3] Add test-compile to targets in filter creation --- build/run_unit_group.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 546811c3496fc..06fc1e5ac4bbe 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -99,6 +99,8 @@ function mvn_test() { create_filter_targets+=("$target" "-DskipTests") # no need to run install for test run target="verify" + else + create_filter_targets+=("test-compile") fi create_filter_targets+=("exec:exec") $MVN_TEST_OPTIONS -PtestFilterCreate "${pl_modules_args[@]}" "${create_filter_targets[@]}" "${groups_arg}"