From 4e21ed6bee5ab36b0a3365e99f78922a4c4b8e5e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 20 Aug 2026 13:49:33 +0200 Subject: [PATCH 1/3] Revert "ci: revert new system-tests gitlab pipeline (#19784)" This reverts commit 9e9c838f2445459b5cb39d1d9d48f31ce8bd46bd. --- .gitlab/system-tests.yml | 72 ++++++++++++++++++++++++++ scripts/update-system-tests-version.py | 21 ++++++++ 2 files changed, 93 insertions(+) diff --git a/.gitlab/system-tests.yml b/.gitlab/system-tests.yml index a35d69a84db..18744f276b8 100644 --- a/.gitlab/system-tests.yml +++ b/.gitlab/system-tests.yml @@ -90,3 +90,75 @@ system-tests parametric: when: always paths: - system-tests/logs* + +include: + - project: 'DataDog/system-tests' + ref: '1f3f07e9f21f049b63abb0f3ed667583395fc2b6' + file: 'utils/ci/gitlab/main.yml' + inputs: + stage: "tests" + ref: '1f3f07e9f21f049b63abb0f3ed667583395fc2b6' + condition: '$CI_COMMIT_BRANCH == "main"' + +"build lambda layer": + image: registry.ddbuild.io/images/docker:27.3.1 + tags: ["docker-in-docker:amd64"] + stage: package + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + needs: + - job: "build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64]" + artifacts: true + script: + - | + echo -e "\e[0Ksection_start:`date +%s`:checkout_lambda\r\e[0KChecking out datadog-lambda-python" + git clone --depth 1 https://github.com/DataDog/datadog-lambda-python.git ${CI_PROJECT_DIR}/datadog-lambda-python + echo -e "\e[0Ksection_end:`date +%s`:checkout_lambda\r\e[0K" + - | + echo -e "\e[0Ksection_start:`date +%s`:build_layer\r\e[0KBuilding datadog_lambda layer" + wheel_path=$(find "${CI_PROJECT_DIR}/pywheels" -name "*cp313*manylinux*.whl" -print -quit) + if [ -z "${wheel_path}" ]; then + echo "ERROR: No cp313 manylinux wheel found in ${CI_PROJECT_DIR}/pywheels" >&2 + exit 1 + fi + # AIDEV-NOTE: The wheel must be inside this repository because build_layers.sh + # uses the repository root as its Docker build context. + mkdir -p "${CI_PROJECT_DIR}/datadog-lambda-python/artifacts" + cp "${wheel_path}" "${CI_PROJECT_DIR}/datadog-lambda-python/artifacts/" + wheel_path="artifacts/$(basename "${wheel_path}")" + cd ${CI_PROJECT_DIR}/datadog-lambda-python + ARCH=amd64 PYTHON_VERSION=3.13 DD_TRACE_WHEEL="${wheel_path}" ./scripts/build_layers.sh + echo -e "\e[0Ksection_end:`date +%s`:build_layer\r\e[0K" + - | + echo -e "\e[0Ksection_start:`date +%s`:copy_layer\r\e[0KCopying layer into pywheels/" + mkdir -p ${CI_PROJECT_DIR}/pywheels + cp ${CI_PROJECT_DIR}/datadog-lambda-python/.layers/datadog_lambda_py-amd64-3.13.zip ${CI_PROJECT_DIR}/pywheels/ + ls -la ${CI_PROJECT_DIR}/pywheels/ + echo -e "\e[0Ksection_end:`date +%s`:copy_layer\r\e[0K" + artifacts: + paths: + - pywheels/*.zip + +system_tests_param: + extends: .system_tests_param_base + stage: tests + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + needs: + - job: "build linux: [amd64, cp311-cp311, v113741238-d2b8243-manylinux2014_x86_64]" + artifacts: false + - job: "build linux: [amd64, cp312-cp312, v113741238-d2b8243-manylinux2014_x86_64]" + artifacts: false + - job: "build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64]" + artifacts: false + - job: "build linux: [amd64, cp314-cp314, v113741238-d2b8243-manylinux2014_x86_64]" + artifacts: false + - job: "build lambda layer" + artifacts: false + variables: + LIBRARIES: "python python_lambda" + SCENARIO_GROUPS: tracer_release + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "true" + SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION: "true" + BINARIES_ARTIFACT_PATH: pywheels + BINARIES_ARTIFACTS: "build linux: [amd64, cp311-cp311, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp312-cp312, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp314-cp314, v113741238-d2b8243-manylinux2014_x86_64];build lambda layer" diff --git a/scripts/update-system-tests-version.py b/scripts/update-system-tests-version.py index 2edb12a8246..b8d748f147c 100755 --- a/scripts/update-system-tests-version.py +++ b/scripts/update-system-tests-version.py @@ -8,6 +8,7 @@ system_tests_repo = "https://github.com/DataDog/system-tests.git" system_tests_workflows_path = ".github/workflows/system-tests.yml" +system_tests_gitlab_e2e_path = ".gitlab/system-tests.yml" gitlab_ci_path = ".gitlab-ci.yml" @@ -69,6 +70,26 @@ def update_system_tests_version(latest_version: str) -> None: with open(gitlab_ci_path, "w") as file: file.write("\n".join(lines)) + # Update Gitlab e2e workflow file + with open(system_tests_gitlab_e2e_path, "r") as file: + content = file.read() + + lines = content.splitlines() + update_ref_count = 0 + for i in range(len(lines)): + # Only update the ref if the repository is DataDog/system-tests + if "- project: 'DataDog/system-tests'" in lines[i]: + update_ref_count = 2 + if update_ref_count and lines[i].strip().startswith("ref:"): + pre, _, _ = lines[i].partition(":") + lines[i] = f"{pre}: '{latest_version}'" + update_ref_count -= 1 + + if lines and lines[-1]: + lines.append("") + with open(system_tests_gitlab_e2e_path, "w") as file: + file.write("\n".join(lines)) + if __name__ == "__main__": current_version = get_current_system_tests_version() From 4e00c7bd1c4d4ebe696999469c9383627d354bc4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 20 Aug 2026 13:52:16 +0200 Subject: [PATCH 2/3] test --- .gitlab/system-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/system-tests.yml b/.gitlab/system-tests.yml index 18744f276b8..a04fe509888 100644 --- a/.gitlab/system-tests.yml +++ b/.gitlab/system-tests.yml @@ -98,14 +98,14 @@ include: inputs: stage: "tests" ref: '1f3f07e9f21f049b63abb0f3ed667583395fc2b6' - condition: '$CI_COMMIT_BRANCH == "main"' + # condition: '$CI_COMMIT_BRANCH == "main"' "build lambda layer": image: registry.ddbuild.io/images/docker:27.3.1 tags: ["docker-in-docker:amd64"] stage: package - rules: - - if: '$CI_COMMIT_BRANCH == "main"' + # rules: + # - if: '$CI_COMMIT_BRANCH == "main"' needs: - job: "build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64]" artifacts: true @@ -142,8 +142,8 @@ include: system_tests_param: extends: .system_tests_param_base stage: tests - rules: - - if: '$CI_COMMIT_BRANCH == "main"' + # rules: + # - if: '$CI_COMMIT_BRANCH == "main"' needs: - job: "build linux: [amd64, cp311-cp311, v113741238-d2b8243-manylinux2014_x86_64]" artifacts: false From dc4b0ee06e1f0eded9e074289034f3fdaaa1f67c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 21 Aug 2026 13:41:07 +0200 Subject: [PATCH 3/3] Allow failure --- .gitlab/system-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/system-tests.yml b/.gitlab/system-tests.yml index a04fe509888..015e7bb4410 100644 --- a/.gitlab/system-tests.yml +++ b/.gitlab/system-tests.yml @@ -93,11 +93,12 @@ system-tests parametric: include: - project: 'DataDog/system-tests' - ref: '1f3f07e9f21f049b63abb0f3ed667583395fc2b6' + ref: 'ceeb84419c5ebe6aa0860f07f1fa7d6bc55bf481' file: 'utils/ci/gitlab/main.yml' inputs: stage: "tests" - ref: '1f3f07e9f21f049b63abb0f3ed667583395fc2b6' + ref: 'ceeb84419c5ebe6aa0860f07f1fa7d6bc55bf481' + allow_failure: true # condition: '$CI_COMMIT_BRANCH == "main"' "build lambda layer":