ci: new system-tests gitlab pipeline

.gitlab/system-tests.yml

@@ -90,3 +90,76 @@ system-tests parametric:
     when: always
     paths:
       - system-tests/logs*
+
+include:
+  - project: 'DataDog/system-tests'
+    ref: 'ceeb84419c5ebe6aa0860f07f1fa7d6bc55bf481'
+    file: 'utils/ci/gitlab/main.yml'
+    inputs:
+      stage: "tests"
+      ref: 'ceeb84419c5ebe6aa0860f07f1fa7d6bc55bf481'
+      allow_failure: true
+      # condition: '$CI_COMMIT_BRANCH == "main"'
+
+"build lambda layer":
+  image: registry.ddbuild.io/images/docker:27.3.1
+  tags: ["docker-in-docker:amd64"]
+  stage: package
+  # rules:
+  #   - if: '$CI_COMMIT_BRANCH == "main"'
+  needs:
+    - job: "build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64]"
+      artifacts: true
+  script:
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:checkout_lambda\r\e[0KChecking out datadog-lambda-python"
+      git clone --depth 1 https://github.com/DataDog/datadog-lambda-python.git ${CI_PROJECT_DIR}/datadog-lambda-python
+      echo -e "\e[0Ksection_end:`date +%s`:checkout_lambda\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:build_layer\r\e[0KBuilding datadog_lambda layer"
+      wheel_path=$(find "${CI_PROJECT_DIR}/pywheels" -name "*cp313*manylinux*.whl" -print -quit)
+      if [ -z "${wheel_path}" ]; then
+        echo "ERROR: No cp313 manylinux wheel found in ${CI_PROJECT_DIR}/pywheels" >&2
+        exit 1
+      fi
+      # AIDEV-NOTE: The wheel must be inside this repository because build_layers.sh
+      # uses the repository root as its Docker build context.
+      mkdir -p "${CI_PROJECT_DIR}/datadog-lambda-python/artifacts"
+      cp "${wheel_path}" "${CI_PROJECT_DIR}/datadog-lambda-python/artifacts/"
+      wheel_path="artifacts/$(basename "${wheel_path}")"
+      cd ${CI_PROJECT_DIR}/datadog-lambda-python
+      ARCH=amd64 PYTHON_VERSION=3.13 DD_TRACE_WHEEL="${wheel_path}" ./scripts/build_layers.sh
+      echo -e "\e[0Ksection_end:`date +%s`:build_layer\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:copy_layer\r\e[0KCopying layer into pywheels/"
+      mkdir -p ${CI_PROJECT_DIR}/pywheels
+      cp ${CI_PROJECT_DIR}/datadog-lambda-python/.layers/datadog_lambda_py-amd64-3.13.zip ${CI_PROJECT_DIR}/pywheels/
+      ls -la ${CI_PROJECT_DIR}/pywheels/
+      echo -e "\e[0Ksection_end:`date +%s`:copy_layer\r\e[0K"
+  artifacts:
+    paths:
+      - pywheels/*.zip
+
+system_tests_param:
+  extends: .system_tests_param_base
+  stage: tests
+  # rules:
+  #   - if: '$CI_COMMIT_BRANCH == "main"'
+  needs:
+    - job: "build linux: [amd64, cp311-cp311, v113741238-d2b8243-manylinux2014_x86_64]"
+      artifacts: false
+    - job: "build linux: [amd64, cp312-cp312, v113741238-d2b8243-manylinux2014_x86_64]"
+      artifacts: false
+    - job: "build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64]"
+      artifacts: false
+    - job: "build linux: [amd64, cp314-cp314, v113741238-d2b8243-manylinux2014_x86_64]"
+      artifacts: false
+    - job: "build lambda layer"
+      artifacts: false
+  variables:
+    LIBRARIES: "python python_lambda"
+    SCENARIO_GROUPS: tracer_release
+    SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "true"
+    SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION: "true"
+    BINARIES_ARTIFACT_PATH: pywheels
+    BINARIES_ARTIFACTS: "build linux: [amd64, cp311-cp311, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp312-cp312, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp313-cp313, v113741238-d2b8243-manylinux2014_x86_64];build linux: [amd64, cp314-cp314, v113741238-d2b8243-manylinux2014_x86_64];build lambda layer"

scripts/update-system-tests-version.py

@@ -8,6 +8,7 @@
 
 system_tests_repo = "https://github.com/DataDog/system-tests.git"
 system_tests_workflows_path = ".github/workflows/system-tests.yml"
+system_tests_gitlab_e2e_path = ".gitlab/system-tests.yml"
 gitlab_ci_path = ".gitlab-ci.yml"
 
 
@@ -69,6 +70,26 @@ def update_system_tests_version(latest_version: str) -> None:
     with open(gitlab_ci_path, "w") as file:
         file.write("\n".join(lines))
 
+    # Update Gitlab e2e workflow file
+    with open(system_tests_gitlab_e2e_path, "r") as file:
+        content = file.read()
+
+    lines = content.splitlines()
+    update_ref_count = 0
+    for i in range(len(lines)):
+        # Only update the ref if the repository is DataDog/system-tests
+        if "- project: 'DataDog/system-tests'" in lines[i]:
+            update_ref_count = 2
+        if update_ref_count and lines[i].strip().startswith("ref:"):
+            pre, _, _ = lines[i].partition(":")
+            lines[i] = f"{pre}: '{latest_version}'"
+            update_ref_count -= 1
+
+    if lines and lines[-1]:
+        lines.append("")
+    with open(system_tests_gitlab_e2e_path, "w") as file:
+        file.write("\n".join(lines))
+
 
 if __name__ == "__main__":
     current_version = get_current_system_tests_version()