[#13529] improvement(test): Run core database lanes with bounded adaptive forks

.github/workflows/backend-integration-test-action.yml

@@ -20,12 +20,16 @@ on:
         required: true
         description: 'run on embedded or deploy mode'
         type: string
+      shard:
+        required: true
+        description: 'Test shard defined in dev/ci/test-shards.sh'
+        type: string
 
 jobs:
   start-runner:
-    name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
+    name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}-${{ inputs.shard }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 120
+    timeout-minutes: 90
     env:
       PLATFORM: ${{ inputs.architecture }}
     steps:
@@ -58,15 +62,18 @@ jobs:
           wget https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
           sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb  
 
-      - name: Backend Integration Test (JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }})
+      - name: Backend Integration Test (JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}-${{ inputs.shard }})
         id: integrationTest
         run: |
           EXCLUDE_CONTRIB_TESTS=""
           for module in $(find catalogs-contrib -mindepth 1 -maxdepth 1 -type d -name 'catalog-*' -exec basename {} \; | sort); do
             EXCLUDE_CONTRIB_TESTS="$EXCLUDE_CONTRIB_TESTS -x :catalogs-contrib:$module:test"
           done
 
-          ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false -PskipWeb=true \
+          shard_args_text="$(dev/ci/test-shards.sh backend-it "${{ inputs.shard }}")"
+          mapfile -t shard_args <<< "${shard_args_text}"
+
+          ./gradlew "${shard_args[@]}" -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false -PskipWeb=true \
             -x :web:web:test -x :web:integration-test:test -x :web-v2:web:test -x :web-v2:integration-test:test -x :clients:client-python:test \
             -x :flink-connector:flink-common:test \
             -x :flink-connector:flink-1.18:test -x :flink-connector:flink-runtime-1.18:test \
@@ -85,7 +92,7 @@ jobs:
         uses: actions/upload-artifact@v7
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
-          name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
+          name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}-${{ inputs.shard }}
           path: |
             build/reports
             iceberg/iceberg-rest-server/build/*.log

.github/workflows/backend-integration-test.yml

@@ -50,41 +50,25 @@ jobs:
               - gradle.properties
               - gradlew
               - settings.gradle.kts
+      - name: List backend integration test shards
+        id: shards
+        run: echo "backend_it_shards=$(dev/ci/test-shards.sh backend-it --list)" >> "${GITHUB_OUTPUT}"
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
+      backend_it_shards: ${{ steps.shards.outputs.backend_it_shards }}
 
-  BackendIT-on-push:
+  BackendIT:
     needs: changes
-    if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true')
-    strategy:
-      matrix:
-        architecture: [linux/amd64]
-        java-version: [ 17 ]
-        backend: [ h2, mysql, postgresql ]
-        test-mode: [ embedded, deploy ]
-        exclude:
-          - test-mode: 'embedded'
-            backend: 'mysql'
-          - test-mode: 'embedded'
-            backend: 'postgresql'
-          - test-mode: 'deploy'
-            backend: 'h2'
-    uses: ./.github/workflows/backend-integration-test-action.yml
-    with:
-      architecture: ${{ matrix.architecture }}
-      java-version: ${{ matrix.java-version }}
-      backend: ${{ matrix.backend }}
-      test-mode: ${{ matrix.test-mode }}
-
-  BackendIT-on-pr:
-    needs: changes
-    if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
+    if: needs.changes.outputs.source_changes == 'true'
     strategy:
+      fail-fast: false
       matrix:
         architecture: [ linux/amd64 ]
         java-version: [ 17 ]
         backend: [ h2, mysql, postgresql ]
         test-mode: [ embedded, deploy ]
+        # Shards are defined in dev/ci/test-shards.sh.
+        shard: ${{ fromJSON(needs.changes.outputs.backend_it_shards) }}
         exclude:
           - test-mode: 'embedded'
             backend: 'mysql'
@@ -98,3 +82,4 @@ jobs:
       java-version: ${{ matrix.java-version }}
       backend: ${{ matrix.backend }}
       test-mode: ${{ matrix.test-mode }}
+      shard: ${{ matrix.shard }}