[Draft]Add antreanodeconfig crd agent

.github/workflows/kind.yml

@@ -606,6 +606,76 @@ jobs:
           path: log.tar.gz
           retention-days: 30
 
+  test-multicluster-e2e:
+    name: Multi-cluster e2e tests on Kind clusters
+    needs: [check-changes]
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
+    runs-on: [ubuntu-latest-8-cores]
+    timeout-minutes: 150
+    env:
+      GIT_COMMIT: ${{ github.sha }}
+      WORKSPACE: ${{ github.workspace }}
+    steps:
+    - name: Free disk space (extended)
+      # Multi-cluster e2e builds Antrea and the multi-cluster controller, then runs three Kind clusters.
+      run: |
+        sudo apt-get clean
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        sudo rm -rf "/usr/local/lib/android"
+        df -h
+    - uses: actions/checkout@v6
+      with:
+        show-progress: false
+    - uses: actions/setup-go@v6
+      with:
+        go-version-file: '.go-version'
+    - uses: ./.github/actions/setup-docker-classic
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+      with:
+        driver: docker
+    - uses: ./.github/actions/setup-kind
+    - name: Run multi-cluster e2e tests
+      run: |
+        mkdir -p "$HOME/.kube"
+        ./ci/jenkins/test-mc.sh \
+          --testcase e2e \
+          --registry "$(head -n1 ci/docker-registry)" \
+          --mc-gateway \
+          --coverage \
+          --kind \
+          --use-system-go \
+          --workdir "$GITHUB_WORKSPACE" \
+          --kubeconfigs-path "$HOME/.kube"
+    - name: Tar coverage files
+      run: tar -czf mc-e2e-coverage.tar.gz mc-e2e-coverage
+    - name: Upload coverage for test-multicluster-e2e
+      uses: actions/upload-artifact@v7
+      with:
+        name: test-multicluster-e2e-coverage
+        path: mc-e2e-coverage.tar.gz
+        retention-days: 30
+    - name: Codecov
+      uses: codecov/codecov-action@v6
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: '**/*.cov.out'
+        disable_search: true
+        flags: kind-e2e-tests
+        name: codecov-test-multicluster-e2e
+        directory: mc-e2e-coverage
+        fail_ci_if_error: ${{ github.event_name == 'push' }}
+    - name: Upload test log
+      uses: actions/upload-artifact@v7
+      if: ${{ failure() }}
+      with:
+        name: multicluster-e2e-kind.tar.gz
+        path: antrea-test-logs.tar.gz
+        retention-days: 30
+
   test-network-policy-conformance-encap:
     name: NetworkPolicy conformance tests on a Kind cluster on Linux
     needs: [build-antrea-coverage-image]
@@ -867,6 +937,7 @@ jobs:
     - test-network-policy-conformance-encap
     - test-secondary-network
     - test-e2e-conformance
+    - test-multicluster-e2e
     - run-installation-checks
     runs-on: [ubuntu-latest]
     steps:

.github/workflows/trivy_scan.yml

@@ -41,7 +41,7 @@ jobs:
         docker pull antrea/antrea-controller-ubuntu:latest
         docker pull antrea/antrea-controller-ubuntu:${{ steps.find-antrea-greatest-version.outputs.antrea_version }}
     - name: Install Trivy
-      uses: aquasecurity/setup-trivy@3fb12ec12f41e471780db15c232d5dd185dcb514  # v0.2.6
+      uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567  # v0.3.1
     - name: Get current UTC date
       id: date
       run: echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT

.github/workflows/trivy_scan_before_release.yml

@@ -23,7 +23,7 @@ jobs:
       run: |
         ./hack/build-antrea-linux-all.sh --pull
     - name: Install Trivy
-      uses: aquasecurity/setup-trivy@3fb12ec12f41e471780db15c232d5dd185dcb514  # v0.2.6
+      uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567  # v0.3.1
     - name: Download Trivy DB
       # Always download the latest DB for releases, don't use a cached version.
       # Try downloading the vulnerability DB up to 5 times, to account for TOOMANYREQUESTS errors.

build/charts/antrea/conf/antrea-agent.conf

@@ -92,6 +92,10 @@ featureGates:
 # - AntreaProxy (proxyAll)
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "NFTablesHostNetworkMode" "default" false) }}
 
+# Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+# of Antrea agent settings via nodeSelector-based policies.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AntreaNodeConfig" "default" true) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

build/yamls/antrea-aks.yml

@@ -4471,6 +4471,10 @@ data:
     # - AntreaProxy (proxyAll)
     #  NFTablesHostNetworkMode: false
 
+    # Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+    # of Antrea agent settings via nodeSelector-based policies.
+    #  AntreaNodeConfig: true
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -5899,7 +5903,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 8210b1e66f6269e7d5a9c85f070a2b1e76459aefa850d7a5066b7b824f00c292
+        checksum/config: fc224133cdae1f19d3343015183b27b51dae6251e252a6fbdb30fb7df1e73539
       labels:
         app: antrea
         component: antrea-agent
@@ -6147,7 +6151,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 8210b1e66f6269e7d5a9c85f070a2b1e76459aefa850d7a5066b7b824f00c292
+        checksum/config: fc224133cdae1f19d3343015183b27b51dae6251e252a6fbdb30fb7df1e73539
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-eks.yml

@@ -4467,6 +4467,10 @@ data:
     # - AntreaProxy (proxyAll)
     #  NFTablesHostNetworkMode: false
 
+    # Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+    # of Antrea agent settings via nodeSelector-based policies.
+    #  AntreaNodeConfig: true
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -5895,7 +5899,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 8210b1e66f6269e7d5a9c85f070a2b1e76459aefa850d7a5066b7b824f00c292
+        checksum/config: fc224133cdae1f19d3343015183b27b51dae6251e252a6fbdb30fb7df1e73539
       labels:
         app: antrea
         component: antrea-agent
@@ -6144,7 +6148,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 8210b1e66f6269e7d5a9c85f070a2b1e76459aefa850d7a5066b7b824f00c292
+        checksum/config: fc224133cdae1f19d3343015183b27b51dae6251e252a6fbdb30fb7df1e73539
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-gke.yml

@@ -4467,6 +4467,10 @@ data:
     # - AntreaProxy (proxyAll)
     #  NFTablesHostNetworkMode: false
 
+    # Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+    # of Antrea agent settings via nodeSelector-based policies.
+    #  AntreaNodeConfig: true
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -5886,7 +5890,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: b09c8311a5c77d881bef59af4c7ea0c75d4c0b78dc7b5c548d77f2048c1be2b5
+        checksum/config: b1844ac9a453a56e0aa1f485b8f3bdbf162af8758d49b52a5d1f8e0c6ed37186
       labels:
         app: antrea
         component: antrea-agent
@@ -6132,7 +6136,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: b09c8311a5c77d881bef59af4c7ea0c75d4c0b78dc7b5c548d77f2048c1be2b5
+        checksum/config: b1844ac9a453a56e0aa1f485b8f3bdbf162af8758d49b52a5d1f8e0c6ed37186
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-ipsec.yml

@@ -4480,6 +4480,10 @@ data:
     # - AntreaProxy (proxyAll)
     #  NFTablesHostNetworkMode: false
 
+    # Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+    # of Antrea agent settings via nodeSelector-based policies.
+    #  AntreaNodeConfig: true
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -5899,7 +5903,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a9782c402bb6dbad085d0abb61931d4a21a9398bdd12794942ee8c73ef21c193
+        checksum/config: cca55471c59e9cee5f5b7d82d2d1f5d94e75ae07cd51c6b9ac678946213bdc9b
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -6191,7 +6195,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a9782c402bb6dbad085d0abb61931d4a21a9398bdd12794942ee8c73ef21c193
+        checksum/config: cca55471c59e9cee5f5b7d82d2d1f5d94e75ae07cd51c6b9ac678946213bdc9b
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea.yml

@@ -4467,6 +4467,10 @@ data:
     # - AntreaProxy (proxyAll)
     #  NFTablesHostNetworkMode: false
 
+    # Enable support for AntreaNodeConfig CRD, which allows per-Node configuration
+    # of Antrea agent settings via nodeSelector-based policies.
+    #  AntreaNodeConfig: true
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -5886,7 +5890,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: ca5121f404625ca074a0c3bce23b781e7a2fb88bceb5083cc9d4883f81ce6a67
+        checksum/config: 95884506388aff1c6aca5b8418d18be3f7b890fe14384d60026055f890b9f869
       labels:
         app: antrea
         component: antrea-agent
@@ -6132,7 +6136,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: ca5121f404625ca074a0c3bce23b781e7a2fb88bceb5083cc9d4883f81ce6a67
+        checksum/config: 95884506388aff1c6aca5b8418d18be3f7b890fe14384d60026055f890b9f869
       labels:
         app: antrea
         component: antrea-controller

ci/jenkins/test-mc.sh

@@ -33,6 +33,11 @@ CODECOV_TOKEN=""
 COVERAGE=false
 KIND=false
 DEBUG=false
+USE_SYSTEM_GO=false
+GOLANG_RELEASE_DIR=${WORKDIR}/golang-releases
+
+multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
+membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
 
 CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h"
 PRINT_DOCKER_STATUS="docker system df -v"
@@ -41,7 +46,7 @@ CLEAN_STALE_IMAGES_CONTAINERD="crictl rmi --prune"
 PRINT_CONTAINERD_STATUS="crictl ps --state Exited"
 
 _usage="Usage: $0 [--kubeconfigs-path <KubeconfigSavePath>] [--workdir <HomePath>]
-                  [--testcase <e2e>] [--mc-gateway] [--codecov-token] [--coverage] [--kind] [--debug]
+                  [--testcase <e2e>] [--mc-gateway] [--codecov-token] [--coverage] [--kind] [--use-system-go] [--debug]
 
 Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set.
 
@@ -53,6 +58,7 @@ Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set.
         --codecov-token               Token used to upload coverage report(s) to Codecov.
         --coverage                    Run e2e with coverage.
         --kind                        Run e2e on Kind clusters.
+        --use-system-go               Use the Go toolchain already available in PATH.
         --debug                       Do not clean up Kind clusters when --kind is set."
 
 function print_usage {
@@ -97,6 +103,10 @@ case $key in
     KIND=true
     shift
     ;;
+    --use-system-go)
+    USE_SYSTEM_GO=true
+    shift
+    ;;
     --debug)
     DEBUG=true
     shift
@@ -474,7 +484,9 @@ function collect_coverage {
 
 trap clean_multicluster EXIT
 source $WORKSPACE/ci/jenkins/utils.sh
-check_and_upgrade_golang
+if [[ ${USE_SYSTEM_GO} != "true" ]]; then
+    check_and_upgrade_golang
+fi
 clean_tmp
 clean_images
 
@@ -508,9 +520,11 @@ set -e
 if [[ ${TESTCASE} =~ "e2e" ]]; then
     export GO111MODULE=on
     export GOPATH=${WORKDIR}/go
-    export GOROOT=${GOLANG_RELEASE_DIR}/go
     export GOCACHE=${WORKDIR}/.cache/go-build
-    export PATH=$GOROOT/bin:$PATH
+    if [[ ${USE_SYSTEM_GO} != "true" ]]; then
+        export GOROOT=${GOLANG_RELEASE_DIR}/go
+        export PATH=$GOROOT/bin:$PATH
+    fi
 
     deliver_antrea_multicluster
     modify_config
@@ -522,9 +536,13 @@ if [[ ${TESTCASE} =~ "e2e" ]]; then
       mkdir -p mc-e2e-coverage
       collect_coverage ${CURRENT_DIR}/mc-e2e-coverage
       # Backup coverage files for later analysis
-      set +e;find ${DEFAULT_WORKDIR}/mc-e2e-coverage -maxdepth 1 -mtime +1 -type f | xargs -n 1 rm;set -e; # Clean up backup files older than one day.
-      cp -r mc-e2e-coverage ${DEFAULT_WORKDIR}
-      run_codecov "e2e-tests" "*antrea-mc*" "${CURRENT_DIR}/mc-e2e-coverage"
+      if [[ -d ${DEFAULT_WORKDIR} && -w ${DEFAULT_WORKDIR} ]]; then
+        set +e;find ${DEFAULT_WORKDIR}/mc-e2e-coverage -maxdepth 1 -mtime +1 -type f | xargs -n 1 rm;set -e; # Clean up backup files older than one day.
+        cp -r mc-e2e-coverage ${DEFAULT_WORKDIR}
+      fi
+      if [[ -n ${CODECOV_TOKEN} ]]; then
+        run_codecov "e2e-tests" "*antrea-mc*" "${CURRENT_DIR}/mc-e2e-coverage"
+      fi
     fi
 fi
 

ci/kind/kind-setup.sh

@@ -310,6 +310,14 @@ function configure_extra_networks {
   fi
   echo "Configuring extra networks"
 
+  # Extra Docker networks must not become the container default gateway (Docker picks the
+  # highest --gw-priority; the Kind cluster network stays at implicit 0 on eth0). Without this,
+  # names like antrea-<cluster>-0 can win over network "kind" and default via eth1.
+  local -a extra_gw_priority=()
+  if [[ -n "${docker_version:-}" ]] && version_ge "$docker_version" "28.0.0"; then
+    extra_gw_priority=(--gw-priority -1000)
+  fi
+
   # create new bridge networks
   i=0
   networks=()
@@ -324,12 +332,12 @@ function configure_extra_networks {
   nodes="$(kind get nodes --name $cluster_name)"
   for node in $nodes; do
     i=1
-    for network in $networks; do
+    for network in "${networks[@]}"; do
       ifname="eth$i"
-      docker network connect --driver-opt=com.docker.network.endpoint.ifname=$ifname $network $node
+      docker network connect "${extra_gw_priority[@]}" --driver-opt=com.docker.network.endpoint.ifname=$ifname "$network" "$node"
       echo "connected worker $node to network $network"
+      i=$((i+1))
     done
-    i=$((i+1))
   done
 }