From d9bcea5b857d623fde1633f27b68f5e775526c3f Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 5 Jun 2026 13:26:20 -0400 Subject: [PATCH 1/5] feat(profiling): add INSTALL_SUBDIR keyword to dd_wrapper_add_test Allows version-specific native test binaries to install into a subdir of the shared test/ directory (e.g. INSTALL_SUBDIR py315 -> test/py315/). build_base_venvs runs in parallel across all Python versions and GitLab merges all artifacts into a single directory for downstream jobs. Without isolation a binary compiled for pyX.Y (RPATH -> libpythonX.Y) lands in the shared test/ directory and crashes when the pytest gtest plugin tries to run it against a different Python runtime. Callers that do not pass INSTALL_SUBDIR are unaffected. Co-Authored-By: Claude Sonnet 4.6 --- .../datadog/profiling/stack/test/CMakeLists.txt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt index 173db9145f2..790cd418e39 100644 --- a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt @@ -36,7 +36,12 @@ if(DO_VALGRIND) endif() function(dd_wrapper_add_test name) - add_executable(${name} ${ARGN}) + # Optional keyword argument: INSTALL_SUBDIR Installs the test binary to test// instead of test/. Use for + # version-specific binaries (e.g. INSTALL_SUBDIR py315) to prevent CI artifact collisions when build_base_venvs runs + # in parallel across Python versions and GitLab merges all artifacts into a shared directory. + cmake_parse_arguments(_ARG "" "INSTALL_SUBDIR" "" ${ARGN}) + set(_SOURCES ${_ARG_UNPARSED_ARGUMENTS}) + add_executable(${name} ${_SOURCES}) target_include_directories(${name} PRIVATE ../include) # this has to refer to the stack extension name to properly link against target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME}) @@ -72,7 +77,11 @@ function(dd_wrapper_add_test name) endif() if(LIB_INSTALL_DIR) - install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) + if(_ARG_INSTALL_SUBDIR) + install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test/${_ARG_INSTALL_SUBDIR}) + else() + install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) + endif() endif() endfunction() From e6e1f72babc2e11cdf30db666e57a7863ecc3748 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 8 Jun 2026 08:28:37 -0400 Subject: [PATCH 2/5] chore(profiling): native C++/Rust py3.15 ABI support --- .../profiling/cmake/FindLibNative.cmake | 8 +- .../profiling/cmake/NativeHeaders.cmake | 28 +++ .../profiling/dd_wrapper/src/sample.cpp | 10 +- .../datadog/profiling/ddup/CMakeLists.txt | 13 +- .../datadog/profiling/stack/CMakeLists.txt | 10 +- .../stack/echion/echion/cpython/tasks.h | 53 ++++- .../profiling/stack/fuzz/CMakeLists.txt | 6 +- .../profiling/stack/src/echion/frame.cc | 39 ++-- .../profiling/stack/test/CMakeLists.txt | 33 ++- .../test/test_cpython_layout_contracts.cpp | 201 ++++++++++++++++++ .../stack/test/test_frame_state_315.cpp | 146 +++++++++++++ 11 files changed, 505 insertions(+), 42 deletions(-) create mode 100644 ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp diff --git a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake index edb6f9bd833..a8d3ded915a 100644 --- a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake @@ -21,10 +21,10 @@ endif() message(WARNING "SOURCE_LIB_DIR: ${SOURCE_LIB_DIR}") message(WARNING "LIBRARY_NAME: ${LIBRARY_NAME}") -# We expect the native extension to be built and installed the headers in the following directory. It is configured in -# setup.py by setting CARGO_TARGET_DIR environment variable. -set(SOURCE_INCLUDE_DIR - ${CMAKE_SOURCE_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See NativeHeaders.cmake +# for the resolution policy. +include(NativeHeaders) +set(SOURCE_INCLUDE_DIR "${NATIVE_HEADERS_DIR}") set(DEST_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(DEST_INCLUDE_DIR ${DEST_LIB_DIR}/include) diff --git a/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake new file mode 100644 index 00000000000..cf28ae21dd5 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake @@ -0,0 +1,28 @@ +# Resolves NATIVE_HEADERS_DIR — the absolute path to libdatadog's generated C headers (produced by the Rust crate under +# src/native/ and written to target./include). +# +# Primary source: setup.py passes -DRUST_GENERATED_HEADERS_DIR= to every CMake invocation via +# _get_common_cmake_args. Whenever that variable is set, we trust it. +# +# Fallback: build_standalone.sh does NOT pass RUST_GENERATED_HEADERS_DIR, so we compute a path relative to this module's +# own location. Callers must have already invoked find_package(Python3) so that Python3_VERSION_MAJOR/_MINOR are +# defined; the fallback uses those to pick the right per-minor target directory (matching setup.py's CARGO_TARGET_DIR +# layout). +# +# Consumers must have "${CMAKE_CURRENT_SOURCE_DIR}/../cmake" on CMAKE_MODULE_PATH before calling include(NativeHeaders). + +if(DEFINED RUST_GENERATED_HEADERS_DIR) + set(NATIVE_HEADERS_DIR "${RUST_GENERATED_HEADERS_DIR}") +else() + if(NOT DEFINED Python3_VERSION_MAJOR OR NOT DEFINED Python3_VERSION_MINOR) + message( + FATAL_ERROR + "NativeHeaders: RUST_GENERATED_HEADERS_DIR is not set and Python3_VERSION_MAJOR/_MINOR are undefined. " + "Call find_package(Python3) before include(NativeHeaders), or pass -DRUST_GENERATED_HEADERS_DIR " + "(as setup.py does).") + endif() + get_filename_component( + NATIVE_HEADERS_DIR + "${CMAKE_CURRENT_LIST_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include" + ABSOLUTE) +endif() diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp index 17fb356be6b..c47eda66625 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp @@ -1,10 +1,14 @@ -#include "sample.hpp" - +// TODO(py-315): Python.h must be included first, before any system or project headers. +// CPython's pyconfig.h defines _POSIX_C_SOURCE and _XOPEN_SOURCE to their current +// POSIX standard values (202405L on 3.15+). If system headers (included transitively +// via libdatadog_helpers.hpp → features.h) are pulled in first, they define older +// values (200809L), and pyconfig.h's later redefinition triggers -Werror on GCC/Clang. #define PY_SSIZE_T_CLEAN - #include #include +#include "sample.hpp" + #include "libdatadog_helpers.hpp" #include "profiler_state.hpp" #include "pymacro.hpp" diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index 3ac62b0f1f5..5f1cbce606b 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -60,7 +60,9 @@ add_library(${EXTENSION_NAME} SHARED ${DDUP_CPP_SRC}) add_ddup_config(${EXTENSION_NAME}) # Cython generates code that produces errors for the following, so relax compile options -target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address) +# -Wno-missing-field-initializers: Python 3.15 added tp_iteritem to PyTypeObject; Cython doesn't initialize it yet +target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address + -Wno-missing-field-initializers) # cmake may mutate the name of the library (e.g., lib- and -.so for dynamic libraries). This suppresses that behavior, # which is required to ensure all paths can be inferred correctly by setup.py. @@ -87,11 +89,10 @@ elseif(UNIX) endif() endif() -target_include_directories( - ${EXTENSION_NAME} - PRIVATE ../dd_wrapper/include - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/ - ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) +include(NativeHeaders) + +target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include "${NATIVE_HEADERS_DIR}" + ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) diff --git a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt index 21011f4aa43..2f8f927b2e1 100644 --- a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt @@ -94,14 +94,16 @@ add_clangtidy_target(${EXTENSION_NAME}) # Never build with native unwinding, since this is not currently used target_compile_definitions(${EXTENSION_NAME} PRIVATE UNWIND_NATIVE_DISABLE) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See +# cmake/NativeHeaders.cmake for the resolution policy. +include(NativeHeaders) + # Includes; echion and python are marked "system" to suppress warnings target_include_directories( ${EXTENSION_NAME} PRIVATE .. # include dd_wrapper from the root in order to make its paths transparent in the code include) -target_include_directories( - ${EXTENSION_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored include/util - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) +target_include_directories(${EXTENSION_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored + include/util "${NATIVE_HEADERS_DIR}") # Echion sources need to be given the current platform if(APPLE) diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h index b91e44106f9..eebb777a109 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h @@ -224,8 +224,57 @@ extern "C" #define RESUME_QUICK INSTRUMENTED_RESUME #endif -#if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14+: Use stackpointer and _PyStackRef +#if PY_VERSION_HEX >= 0x030f0000 + // Python 3.15+: FRAME_SUSPENDED_YIELD_FROM_LOCKED is a new frame state for + // generators that are locked during a yield-from in free-threaded builds. + // In GIL builds this state is unreachable, so we only check it under + // Py_GIL_DISABLED. All other logic is identical to 3.14 (stackpointer/_PyStackRef). + + inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) + { + if (gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM +#ifdef Py_GIL_DISABLED + && gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED +#endif + ) { + return nullptr; + } + + _PyInterpreterFrame frame; + if (copy_type(frame_addr, frame)) { + return nullptr; + } + + PyCodeObject code; + auto code_addr = reinterpret_cast(BITS_TO_PTR_MASKED(frame.f_executable)); + if (copy_type(code_addr, code)) { + return nullptr; + } + + uintptr_t frame_addr_uint = reinterpret_cast(frame_addr); + uintptr_t localsplus_addr = frame_addr_uint + offsetof(_PyInterpreterFrame, localsplus); + uintptr_t stackbase_addr = localsplus_addr + code.co_nlocalsplus * sizeof(_PyStackRef); + + uintptr_t stackpointer_addr = reinterpret_cast(frame.stackpointer); + if (stackpointer_addr <= stackbase_addr) { + return nullptr; + } + + int stacktop = static_cast((stackpointer_addr - stackbase_addr) / sizeof(_PyStackRef)); + if (stacktop < 1 || stacktop > MAX_STACK_SIZE) { + return nullptr; + } + + _PyStackRef top_ref; + if (copy_type(reinterpret_cast(stackpointer_addr - sizeof(_PyStackRef)), top_ref)) { + return nullptr; + } + + return BITS_TO_PTR_MASKED(top_ref); + } + +#elif PY_VERSION_HEX >= 0x030e0000 + // Python 3.14: Use stackpointer and _PyStackRef inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) { diff --git a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt index 37846a592f3..c1c4d9f8061 100644 --- a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt @@ -29,10 +29,8 @@ function(add_fuzz_target TARGET_NAME) # Include paths: ../.. is the profiling root (for "dd_wrapper/include/..." paths), ../include is for stack headers. target_include_directories(${TARGET_NAME} PRIVATE ../.. ../include) - target_include_directories( - ${TARGET_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored ../include/util - ../../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored + ../include/util "${NATIVE_HEADERS_DIR}") # Ensure echion headers take the fuzz hook in vm.h target_compile_definitions(${TARGET_NAME} PRIVATE ECHION_FUZZING) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc index 2d608e08c77..9e38e1a4aa1 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc @@ -111,22 +111,31 @@ Frame::read(EchionSampler& echion, PyObject* frame_addr, PyObject** prev_addr) frame_addr = &iframe; #if PY_VERSION_HEX >= 0x030c0000 + // _PyInterpreterFrame.owner is stored as char, not the _frameowner enum + // itself, so -Wswitch can't enforce exhaustiveness. test_cpython_layout + // _contracts static_asserts the enum values we rely on; an unknown owner + // here means CPython grew a new value and is treated as an error. + switch (frame_addr->owner) { + case FRAME_OWNED_BY_THREAD: + case FRAME_OWNED_BY_GENERATOR: + break; // valid live Python frame — proceed with frame reading + case FRAME_OWNED_BY_FRAME_OBJECT: + return ErrorKind::FrameError; // frame belongs to a PyFrameObject, not executing +#if PY_VERSION_HEX < 0x030f0000 + case FRAME_OWNED_BY_CSTACK: // C shim frame (removed in 3.15) +#endif #if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14 introduced FRAME_OWNED_BY_INTERPRETER, and frames of this - // type are also ignored by the upstream profiler. - // See - // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK || frame_addr->owner == FRAME_OWNED_BY_INTERPRETER) { -#else - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK) { -#endif // PY_VERSION_HEX >= 0x030e0000 - *prev_addr = frame_addr->previous; - // This is a C frame, we just need to ignore it - return std::ref(C_FRAME); - } - - if (frame_addr->owner != FRAME_OWNED_BY_THREAD && frame_addr->owner != FRAME_OWNED_BY_GENERATOR) { - return ErrorKind::FrameError; + case FRAME_OWNED_BY_INTERPRETER: +#endif + // C/interpreter-managed frame — skip it and follow the frame chain. + // FRAME_OWNED_BY_INTERPRETER introduced in 3.14; FRAME_OWNED_BY_CSTACK + // present in 3.12–3.14, removed in 3.15. + // See + // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 + *prev_addr = frame_addr->previous; + return std::ref(C_FRAME); + default: + return ErrorKind::FrameError; } #endif // PY_VERSION_HEX >= 0x030c0000 diff --git a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt index 790cd418e39..ea366f84c44 100644 --- a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt @@ -36,13 +36,16 @@ if(DO_VALGRIND) endif() function(dd_wrapper_add_test name) - # Optional keyword argument: INSTALL_SUBDIR Installs the test binary to test// instead of test/. Use for - # version-specific binaries (e.g. INSTALL_SUBDIR py315) to prevent CI artifact collisions when build_base_venvs runs - # in parallel across Python versions and GitLab merges all artifacts into a shared directory. cmake_parse_arguments(_ARG "" "INSTALL_SUBDIR" "" ${ARGN}) set(_SOURCES ${_ARG_UNPARSED_ARGUMENTS}) add_executable(${name} ${_SOURCES}) - target_include_directories(${name} PRIVATE ../include) + # Replicate the include dirs that ${EXTENSION_NAME} sets PRIVATE (so they don't propagate to test targets): stack's + # own headers, the profiling root (for "dd_wrapper/include/sample.hpp"), Python, echion, and the libdatadog + # Rust-generated headers (NATIVE_HEADERS_DIR, set by the parent CMakeLists). + target_include_directories(${name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_SOURCE_DIR}/../..") + target_include_directories(${name} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} "${CMAKE_CURRENT_SOURCE_DIR}/../echion" + "${NATIVE_HEADERS_DIR}") # this has to refer to the stack extension name to properly link against target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME}) @@ -71,6 +74,14 @@ function(dd_wrapper_add_test name) gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running # sanitizer-built executables during build + # Echion headers (vm.h, tasks.h) require PL_DARWIN or PL_LINUX to define proc_ref_t and copy_type. These are set + # PRIVATE on ${EXTENSION_NAME} and don't propagate to test targets, so we replicate the logic here. + if(APPLE) + target_compile_definitions(${name} PRIVATE PL_DARWIN) + else() + target_compile_definitions(${name} PRIVATE PL_LINUX) + endif() + # This is supplemental artifact so make sure to install it in the right place if(INPLACE_LIB_INSTALL_DIR) set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}") @@ -114,3 +125,17 @@ configure_stack_internal_test(test_sampling_cycle_state) dd_wrapper_add_test(test_alt_stack_ownership test_alt_stack_ownership.cpp) # ThreadAltStack lives in the vendored echion header tree. target_include_directories(test_alt_stack_ownership PRIVATE ../echion) +# test_frame_state_315 validates the PyFrameState renumbering and related frame-internals changes introduced in Python +# 3.15. All test bodies are gated on PY_VERSION_HEX >= 0x030f0000, so building on older versions would produce an empty +# test binary. +if(Python3_VERSION VERSION_GREATER_EQUAL "3.15") + # INSTALL_SUBDIR py315 keeps this binary out of the shared test/ directory. build_base_venvs runs in parallel for + # all Python versions; GitLab merges their artifacts into one directory for downstream jobs. Without isolation the + # py3.15-compiled binary (RPATH -> libpython3.15) would crash when the pytest gtest plugin tried to run it in a + # py3.10 environment. + dd_wrapper_add_test(test_frame_state_315 test_frame_state_315.cpp INSTALL_SUBDIR py315) + # Route copy_memory through echion_fuzz_copy_memory so tests can assert whether the state guard allows execution to + # reach the copy site. + target_compile_definitions(test_frame_state_315 PRIVATE ECHION_FUZZING) +endif() +dd_wrapper_add_test(test_cpython_layout_contracts test_cpython_layout_contracts.cpp) diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp new file mode 100644 index 00000000000..625b46b1e2b --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp @@ -0,0 +1,201 @@ +// Compile-time contracts for CPython internal enum values that echion depends on. +// +// Each static_assert fires at *compile time* against the actual CPython headers — +// if CPython renumbers or removes an enum value the build breaks immediately, +// before any test runner is invoked. The matching gtest TEST() wrappers surface +// the same checks as human-readable failures in CI output. +// +// Update these blocks when adding support for a new CPython minor version: +// 1. Add a new versioned block with the new values. +// 2. Adjust the upper-bound on the previous block if values changed. +// 3. Run the build against the new CPython to confirm all assertions pass. +// +// Enums covered: +// _frameowner (pycore_interpframe_structs.h, 3.12+) +// PyFrameState (pycore_frame.h, 3.11+) + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#if PY_VERSION_HEX >= 0x030e0000 +// Python 3.14+: frame internals split into separate headers; +// _frameowner is in pycore_interpframe_structs.h (new in 3.14). +#include +#include +#include +#elif PY_VERSION_HEX >= 0x030b0000 +// Python 3.11-3.13: _frameowner enum lives directly in pycore_frame.h. +// pycore_interpframe_structs.h does not exist on these versions. +#include +#endif + +// echion/vm.h defines proc_ref_t, which the stub below requires. +#include + +// Stub: echion's remote-memory callback is referenced at link time via vm.h. +// Always returns failure — no live process attached in unit tests. +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + return -1; +} + +// ───────────────────────────────────────────────────────────────────────────── +// _frameowner enum (pycore_interpframe_structs.h, introduced in 3.12) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.12 – 3.13: four members, CSTACK=3, no INTERPRETER +#if PY_VERSION_HEX >= 0x030c0000 && PY_VERSION_HEX < 0x030e0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_312_313, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 3); +} +#endif // 3.12 – 3.13 + +// 3.14: five members, INTERPRETER added (=3), CSTACK bumped to 4 +#if PY_VERSION_HEX >= 0x030e0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 4, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 4); +} +#endif // 3.14 + +// 3.15+: FRAME_OWNED_BY_CSTACK removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +// FRAME_OWNED_BY_CSTACK intentionally not listed — it was removed in 3.15. +// If this file compiles without error, CPython has not re-introduced it. + +TEST(FrameOwnerEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); +} +#endif // 3.15+ + +// ───────────────────────────────────────────────────────────────────────────── +// PyFrameState / gi_frame_state (pycore_frame.h, introduced in 3.11) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.11 – 3.12: negative-valued range, no FRAME_SUSPENDED_YIELD_FROM yet. +// FRAME_SUSPENDED_YIELD_FROM was introduced in 3.13 (CPython gh-104210), which +// also shifted FRAME_CREATED and FRAME_SUSPENDED one slot more negative. +#if PY_VERSION_HEX >= 0x030b0000 && PY_VERSION_HEX < 0x030d0000 +static_assert(FRAME_CREATED == -2, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_311_312, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -2); + EXPECT_EQ(FRAME_SUSPENDED, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.11 – 3.12 + +// 3.13 – 3.14: negative-valued range, FRAME_SUSPENDED_YIELD_FROM added (-1), +// pushing FRAME_CREATED to -3 and FRAME_SUSPENDED to -2. +#if PY_VERSION_HEX >= 0x030d0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_CREATED == -3, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -2, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == -1, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_313_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -3); + EXPECT_EQ(FRAME_SUSPENDED, -2); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.13 – 3.14 + +// 3.15+: all values renumbered, COMPLETED removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_CREATED == 0, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == 1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +// value 3 is FRAME_SUSPENDED_YIELD_FROM_LOCKED in free-threaded builds (see below) +static_assert(FRAME_EXECUTING == 4, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +static_assert(FRAME_CLEARED == 5, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); +// FRAME_COMPLETED intentionally not listed — it was removed in 3.15. + +#ifdef Py_GIL_DISABLED +static_assert( + FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, + "TODO(py-315): FRAME_SUSPENDED_YIELD_FROM_LOCKED changed value — update tasks.h PyGen_yf (Py_GIL_DISABLED)"); +#endif + +TEST(PyFrameStateEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +#ifdef Py_GIL_DISABLED +TEST(PyFrameStateEnum_315_NoGIL, LockedYieldFromValueMatchesExpected) +{ + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM_LOCKED, 3); +} +#endif + +#endif // 3.15+ diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp new file mode 100644 index 00000000000..bb356d5fc8a --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp @@ -0,0 +1,146 @@ +// Unit tests for Python 3.15 frame-state guard changes. +// +// Covered: +// 1. Static assertions on renumbered PyFrameState enum values (3.15+). +// 2. PyGen_yf returns nullptr for FRAME_SUSPENDED_YIELD_FROM_LOCKED in GIL builds (3.15+). +// 3. PyGen_yf enters the body for FRAME_SUSPENDED_YIELD_FROM even after the 3.15 guard change. +// 4. PyGen_yf returns nullptr for all non-suspended states (3.15+). +// +// Memory stub: copy_type/copy_generic call echion_fuzz_copy_memory. We define it here to +// always return failure (-1), which is the correct outcome when no real Python process is +// attached. All code paths that reach a copy_type call will return nullptr safely. + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#include +#include + +#if PY_VERSION_HEX >= 0x030e0000 +#include +#include +#include +#include +#include +#endif + +#include +#include +#include + +// Counter tracking how many times copy_memory was invoked. Reset before each +// PyGen_yf call so tests can assert whether the state guard allowed execution +// to reach the copy site (>0) or filtered it out first (0). +// Must be declared before echion headers use ECHION_FUZZING to route copy_memory +// through this stub; atomic so future parallel-test runs stay race-free. +static std::atomic g_copy_attempts{ 0 }; + +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + g_copy_attempts.fetch_add(1, std::memory_order_relaxed); + return -1; // always fail — no live process attached +} + +// ───────────────────────────────────────────────────────────────────────────── +// 1. Compile-time enum value assertions (3.15+ only) +// ───────────────────────────────────────────────────────────────────────────── + +#if PY_VERSION_HEX >= 0x030f0000 + +// PyFrameState was renumbered in 3.15. Verify our understanding matches reality so +// that any future CPython change is caught immediately at compile time. +static_assert(FRAME_CREATED == 0, "FRAME_CREATED should be 0 in Python 3.15"); +static_assert(FRAME_SUSPENDED == 1, "FRAME_SUSPENDED should be 1 in Python 3.15"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, "FRAME_SUSPENDED_YIELD_FROM should be 2 in Python 3.15"); +static_assert(FRAME_EXECUTING == 4, "FRAME_EXECUTING should be 4 in Python 3.15"); +static_assert(FRAME_CLEARED == 5, "FRAME_CLEARED should be 5 in Python 3.15"); + +#ifdef Py_GIL_DISABLED +// FRAME_SUSPENDED_YIELD_FROM_LOCKED only exists when building against a free-threaded Python. +static_assert(FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, "FRAME_SUSPENDED_YIELD_FROM_LOCKED should be 3 in Python 3.15"); +#endif // Py_GIL_DISABLED + +TEST(PyFrameState315, EnumValuesMatchExpected) +{ + // Runtime counterpart of the static_asserts above — provides a readable failure + // message in the test output if run against an unexpected Python build. + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 2. PyGen_yf state-check tests (3.15+) +// +// PyGenObject::gi_frame_state is an int (signed). We set only that field; all +// other fields are zero-initialised. We pass nullptr as frame_addr so that if the +// state check passes, copy_type will immediately fail and return nullptr — which +// means any test that expects nullptr is still correct regardless of whether the +// state check or the copy fails first. +// ───────────────────────────────────────────────────────────────────────────── + +static PyGenObject +make_fake_gen(int frame_state) +{ + PyGenObject gen{}; + gen.gi_frame_state = frame_state; + return gen; +} + +#ifndef Py_GIL_DISABLED + +TEST(PyGenYf315GilBuild, LockedStateIgnored) +{ + // FRAME_SUSPENDED_YIELD_FROM_LOCKED (value 3) must NOT be treated as a + // suspended-yield-from state in GIL builds. PyGen_yf should return nullptr + // immediately from the state guard without attempting any memory read. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(3 /* FRAME_SUSPENDED_YIELD_FROM_LOCKED value */); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "state guard must filter FRAME_SUSPENDED_YIELD_FROM_LOCKED without any copy attempt"; +} + +TEST(PyGenYf315GilBuild, SuspendedYieldFromEntersBody) +{ + // FRAME_SUSPENDED_YIELD_FROM must still be recognised as a suspended state. + // The state guard passes, execution enters the body, and copy_type(nullptr, frame) + // immediately fails — confirming the guard did NOT filter out this state. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(FRAME_SUSPENDED_YIELD_FROM); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); // copy_type fails on nullptr frame_addr + EXPECT_GT(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "FRAME_SUSPENDED_YIELD_FROM must pass the state guard and attempt a copy"; +} + +#endif // !Py_GIL_DISABLED + +// Parametrised: non-suspended states must all return nullptr immediately. +class PyGenYf315OtherStates : public ::testing::TestWithParam +{}; + +TEST_P(PyGenYf315OtherStates, ReturnsNull) +{ + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(GetParam()); + EXPECT_EQ(PyGen_yf(&gen, nullptr), nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "non-suspended states must be filtered by the state guard without any copy attempt"; +} + +INSTANTIATE_TEST_SUITE_P(NonSuspendedStates, + PyGenYf315OtherStates, + ::testing::Values(FRAME_CREATED, // 0 + FRAME_EXECUTING, // 4 + FRAME_CLEARED // 5 + )); + +#endif // PY_VERSION_HEX >= 0x030f0000 From 9d8528db4680ae5821fec691a9bc9b79e81423d3 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 21 Aug 2026 07:51:19 +0300 Subject: [PATCH 3/5] chore(profiling): update Python profiling collectors for py3.15 # Conflicts: # ddtrace/internal/monitoring.py # ddtrace/internal/wrapping/asyncs.py --- ddtrace/internal/monitoring.py | 15 +++- ddtrace/profiling/_asyncio.py | 47 ++++++------ ddtrace/profiling/collector/asyncio.py | 77 +++++++++++--------- ddtrace/profiling/collector/exception.py | 14 +++- ddtrace/profiling/collector/stack.py | 11 ++- ddtrace/profiling/collector/threading.py | 92 +++++++++++++----------- tests/profiling/test_scheduler.py | 5 +- 7 files changed, 162 insertions(+), 99 deletions(-) diff --git a/ddtrace/internal/monitoring.py b/ddtrace/internal/monitoring.py index f63994cd711..e7ae535c81a 100644 --- a/ddtrace/internal/monitoring.py +++ b/ddtrace/internal/monitoring.py @@ -142,6 +142,7 @@ def _events_for_handler(handler: MonitoringEventHandler) -> int: return events + class _Entry(NamedTuple): handler: MonitoringEventHandler events: int # pre-computed from _events_for_handler @@ -274,7 +275,19 @@ def _on_py_line(code: CodeType, line_number: int) -> Optional[object]: def _set_local_events(tool_id: int, code: CodeType, events: int) -> None: - sys.monitoring.set_local_events(tool_id, code, events) + # TODO(py-315): Pre-release Python 3.15 builds may reject PY_UNWIND + # as a local event. Fall back without it when the full set is invalid; + # PY_UNWIND is still registered as a global callback via _setup() so + # exception handling degrades gracefully rather than crashing. + try: + sys.monitoring.set_local_events(tool_id, code, events) + except ValueError: + fallback = events & ~_E.PY_UNWIND + if fallback != events: + sys.monitoring.set_local_events(tool_id, code, fallback) + else: + raise + def _rearm_local_events(tool_id: int, code: CodeType, events: int) -> None: diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 133dab9ddb5..259bae9eaeb 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -119,19 +119,21 @@ def _( @partial(wrap, sys.modules["asyncio"].tasks._GatheringFuture.__init__) def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: + f(*args, **kwargs) + children = get_argument_value(args, kwargs, 1, "children") + assert children is not None # nosec: assert is used for typing + + # TODO(py-315): current_task() raises RuntimeError on Python 3.15+ when there + # is no running event loop (e.g. asyncio.gather() called outside an async + # context to build a coroutine for later scheduling). In that case there is + # no parent task to link from, so we skip link_tasks entirely. try: - return f(*args, **kwargs) - finally: - children: list[aio.Future[typing.Any]] = typing.cast( - "list[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 1, "children") - ) - assert children is not None # nosec: assert is used for typing - - if globals()["get_running_loop"]() is not None: - parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() - if parent is not None: - for child in children: - stack.link_tasks(parent, child) + parent = globals()["current_task"]() + except RuntimeError: + return + if parent is not None: + for child in children: + stack.link_tasks(parent, child) @partial(wrap, sys.modules["asyncio"].tasks._wait) def _( @@ -139,15 +141,20 @@ def _( args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any], ) -> typing.Any: + result = f(*args, **kwargs) + futures = typing.cast(set["aio.Future[typing.Any]"], get_argument_value(args, kwargs, 0, "fs")) + + # TODO(py-315): same guard as the _GatheringFuture wrapper above — _wait may + # also be invoked outside a running loop. Skip link_tasks when current_task() + # raises. try: - return f(*args, **kwargs) - finally: - futures = typing.cast("set[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs")) - - if globals()["get_running_loop"]() is not None: - parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) - for future in futures: - stack.link_tasks(parent, future) + parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) + except RuntimeError: + return result + if parent is not None: + for future in futures: + stack.link_tasks(parent, future) + return result @partial(wrap, sys.modules["asyncio"].tasks.as_completed) def _( diff --git a/ddtrace/profiling/collector/asyncio.py b/ddtrace/profiling/collector/asyncio.py index d8afdf93aeb..47933c6bcec 100644 --- a/ddtrace/profiling/collector/asyncio.py +++ b/ddtrace/profiling/collector/asyncio.py @@ -3,52 +3,63 @@ import asyncio from types import ModuleType -from . import _lock +try: + from . import _lock -class _ProfiledAsyncioLock(_lock._ProfiledLock): - pass + class _ProfiledAsyncioLock(_lock._ProfiledLock): + pass + class _ProfiledAsyncioSemaphore(_lock._ProfiledLock): + pass -class _ProfiledAsyncioSemaphore(_lock._ProfiledLock): - pass + class _ProfiledAsyncioBoundedSemaphore(_lock._ProfiledLock): + pass + class _ProfiledAsyncioCondition(_lock._ProfiledLock): + pass -class _ProfiledAsyncioBoundedSemaphore(_lock._ProfiledLock): - pass + class AsyncioLockCollector(_lock.LockCollector): + """Record asyncio.Lock usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioLock] = _ProfiledAsyncioLock + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Lock" -class _ProfiledAsyncioCondition(_lock._ProfiledLock): - pass + class AsyncioSemaphoreCollector(_lock.LockCollector): + """Record asyncio.Semaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioSemaphore] = _ProfiledAsyncioSemaphore + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Semaphore" -class AsyncioLockCollector(_lock.LockCollector): - """Record asyncio.Lock usage.""" + class AsyncioBoundedSemaphoreCollector(_lock.LockCollector): + """Record asyncio.BoundedSemaphore usage.""" - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioLock] = _ProfiledAsyncioLock - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Lock" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioBoundedSemaphore] = _ProfiledAsyncioBoundedSemaphore + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "BoundedSemaphore" + class AsyncioConditionCollector(_lock.LockCollector): + """Record asyncio.Condition usage.""" -class AsyncioSemaphoreCollector(_lock.LockCollector): - """Record asyncio.Semaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioCondition] = _ProfiledAsyncioCondition + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Condition" - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioSemaphore] = _ProfiledAsyncioSemaphore - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Semaphore" +except ImportError: + # TODO(py-315): _lock is a Cython extension that is not compiled for all Python + # versions (e.g. Python 3.15 before the manylinux image carries it). When it + # is absent the asyncio lock collectors are unavailable. Defining stubs that + # raise CollectorUnavailable lets profiler.py discover and gracefully skip them + # rather than failing at import time. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable + class AsyncioLockCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable -class AsyncioBoundedSemaphoreCollector(_lock.LockCollector): - """Record asyncio.BoundedSemaphore usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioBoundedSemaphore] = _ProfiledAsyncioBoundedSemaphore - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "BoundedSemaphore" - - -class AsyncioConditionCollector(_lock.LockCollector): - """Record asyncio.Condition usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioCondition] = _ProfiledAsyncioCondition - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Condition" + AsyncioSemaphoreCollector = AsyncioLockCollector # type: ignore[assignment,misc] + AsyncioBoundedSemaphoreCollector = AsyncioLockCollector # type: ignore[assignment,misc] + AsyncioConditionCollector = AsyncioLockCollector # type: ignore[assignment,misc] diff --git a/ddtrace/profiling/collector/exception.py b/ddtrace/profiling/collector/exception.py index af0877d07d7..851974b84b4 100644 --- a/ddtrace/profiling/collector/exception.py +++ b/ddtrace/profiling/collector/exception.py @@ -1,4 +1,16 @@ -from ddtrace.profiling.collector._exception import ExceptionCollector +try: + from ddtrace.profiling.collector._exception import ExceptionCollector +except ImportError: + # TODO(py-315): _exception is a Cython extension not compiled for all Python + # versions (e.g. Python 3.15 before the manylinux image carries it). Define + # a stub so profiler.py can import this module and skip the collector via + # CollectorUnavailable rather than failing at import time. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable + + class ExceptionCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable __all__ = ["ExceptionCollector"] diff --git a/ddtrace/profiling/collector/stack.py b/ddtrace/profiling/collector/stack.py index 406e875eef1..c72ff8305fd 100644 --- a/ddtrace/profiling/collector/stack.py +++ b/ddtrace/profiling/collector/stack.py @@ -13,7 +13,16 @@ from ddtrace.internal.datadog.profiling import stack from ddtrace.internal.settings.profiling import config from ddtrace.profiling import collector -from ddtrace.profiling.collector import _task + + +try: + from ddtrace.profiling.collector import _task +except ImportError: + # TODO(py-315): _task is a Cython extension not compiled for all Python versions. + # Provide a no-op stub so StackCollector can be imported on Python 3.15. + import types as _types + + _task = _types.SimpleNamespace(initialize_gevent_support=lambda: None) # type: ignore[assignment] from ddtrace.profiling.collector import threading from ddtrace.trace import Tracer diff --git a/ddtrace/profiling/collector/threading.py b/ddtrace/profiling/collector/threading.py index 2d15d124b85..2ec931f1a9e 100644 --- a/ddtrace/profiling/collector/threading.py +++ b/ddtrace/profiling/collector/threading.py @@ -6,67 +6,75 @@ from ddtrace.internal.datadog.profiling import stack from ddtrace.internal.settings.profiling import config -from . import _lock +try: + from . import _lock -class _ProfiledThreadingLock(_lock._ProfiledLock): - pass + class _ProfiledThreadingLock(_lock._ProfiledLock): + pass + class _ProfiledThreadingRLock(_lock._ProfiledLock): + pass -class _ProfiledThreadingRLock(_lock._ProfiledLock): - pass + class _ProfiledThreadingSemaphore(_lock._ProfiledLock): + pass + class _ProfiledThreadingBoundedSemaphore(_lock._ProfiledLock): + pass -class _ProfiledThreadingSemaphore(_lock._ProfiledLock): - pass + class _ProfiledThreadingCondition(_lock._ProfiledLock): + pass + class ThreadingLockCollector(_lock.LockCollector): + """Record threading.Lock usage.""" -class _ProfiledThreadingBoundedSemaphore(_lock._ProfiledLock): - pass + PROFILED_LOCK_CLASS: type[_ProfiledThreadingLock] = _ProfiledThreadingLock + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Lock" + class ThreadingRLockCollector(_lock.LockCollector): + """Record threading.RLock usage.""" -class _ProfiledThreadingCondition(_lock._ProfiledLock): - pass + PROFILED_LOCK_CLASS: type[_ProfiledThreadingRLock] = _ProfiledThreadingRLock + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "RLock" + class ThreadingSemaphoreCollector(_lock.LockCollector): + """Record threading.Semaphore usage.""" -class ThreadingLockCollector(_lock.LockCollector): - """Record threading.Lock usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingSemaphore] = _ProfiledThreadingSemaphore + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Semaphore" - PROFILED_LOCK_CLASS: type[_ProfiledThreadingLock] = _ProfiledThreadingLock - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Lock" + class ThreadingBoundedSemaphoreCollector(_lock.LockCollector): + """Record threading.BoundedSemaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingBoundedSemaphore] = _ProfiledThreadingBoundedSemaphore + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "BoundedSemaphore" -class ThreadingRLockCollector(_lock.LockCollector): - """Record threading.RLock usage.""" + class ThreadingConditionCollector(_lock.LockCollector): + """Record threading.Condition usage.""" - PROFILED_LOCK_CLASS: type[_ProfiledThreadingRLock] = _ProfiledThreadingRLock - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "RLock" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingCondition] = _ProfiledThreadingCondition + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Condition" +except ImportError: + # TODO(py-315): _lock is a Cython extension not compiled for all Python versions + # (e.g. Python 3.15 before the manylinux image carries it). Stubs raise + # CollectorUnavailable so profiler.py skips them gracefully. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable -class ThreadingSemaphoreCollector(_lock.LockCollector): - """Record threading.Semaphore usage.""" + class ThreadingLockCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable - PROFILED_LOCK_CLASS: type[_ProfiledThreadingSemaphore] = _ProfiledThreadingSemaphore - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Semaphore" - - -class ThreadingBoundedSemaphoreCollector(_lock.LockCollector): - """Record threading.BoundedSemaphore usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledThreadingBoundedSemaphore] = _ProfiledThreadingBoundedSemaphore - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "BoundedSemaphore" - - -class ThreadingConditionCollector(_lock.LockCollector): - """Record threading.Condition usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledThreadingCondition] = _ProfiledThreadingCondition - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Condition" + ThreadingRLockCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingSemaphoreCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingBoundedSemaphoreCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingConditionCollector = ThreadingLockCollector # type: ignore[assignment,misc] # Also patch threading.Thread so echion can track thread lifetimes diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 75a0c77c098..9cbb2e2bda3 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -34,7 +34,10 @@ def call_me(): raise Exception("LOL") s = scheduler.Scheduler(before_flush=call_me) - s.flush() + # Patch ddup.upload so the test only checks scheduler logging behaviour and + # doesn't attempt a real upload (which would log a writer error and pollute caplog). + with mock.patch("ddtrace.profiling.scheduler.ddup.upload"): + s.flush() assert caplog.record_tuples == [ (("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed")) ] From 6f9a44c2560a2f4cb8cea9db178ba221903f9cfb Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:28:48 -0400 Subject: [PATCH 4/5] typing: annotate profiling collectors for py3.15 --- ddtrace/internal/monitoring.py | 2 +- ddtrace/profiling/_asyncio.py | 16 +++++++++++----- ddtrace/profiling/collector/stack.py | 5 ++++- tests/profiling/test_scheduler.py | 6 ++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ddtrace/internal/monitoring.py b/ddtrace/internal/monitoring.py index e7ae535c81a..10e7ee87525 100644 --- a/ddtrace/internal/monitoring.py +++ b/ddtrace/internal/monitoring.py @@ -282,7 +282,7 @@ def _set_local_events(tool_id: int, code: CodeType, events: int) -> None: try: sys.monitoring.set_local_events(tool_id, code, events) except ValueError: - fallback = events & ~_E.PY_UNWIND + fallback: int = events & ~_E.PY_UNWIND if fallback != events: sys.monitoring.set_local_events(tool_id, code, fallback) else: diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 259bae9eaeb..9ac02e80dec 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -120,7 +120,9 @@ def _( @partial(wrap, sys.modules["asyncio"].tasks._GatheringFuture.__init__) def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: f(*args, **kwargs) - children = get_argument_value(args, kwargs, 1, "children") + children: list[aio.Future[typing.Any]] = typing.cast( + "list[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 1, "children") + ) assert children is not None # nosec: assert is used for typing # TODO(py-315): current_task() raises RuntimeError on Python 3.15+ when there @@ -128,7 +130,7 @@ def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[ # context to build a coroutine for later scheduling). In that case there is # no parent task to link from, so we skip link_tasks entirely. try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return if parent is not None: @@ -141,14 +143,18 @@ def _( args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any], ) -> typing.Any: - result = f(*args, **kwargs) - futures = typing.cast(set["aio.Future[typing.Any]"], get_argument_value(args, kwargs, 0, "fs")) + result: tuple[set[aio.Future[typing.Any]], set[aio.Future[typing.Any]]] = f(*args, **kwargs) + futures: set[aio.Future[typing.Any]] = typing.cast( + "set[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs") + ) # TODO(py-315): same guard as the _GatheringFuture wrapper above — _wait may # also be invoked outside a running loop. Skip link_tasks when current_task() # raises. try: - parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) + parent: typing.Optional[aio.Task[typing.Any]] = typing.cast( + "aio.Task[typing.Any]", globals()["current_task"]() + ) except RuntimeError: return result if parent is not None: diff --git a/ddtrace/profiling/collector/stack.py b/ddtrace/profiling/collector/stack.py index c72ff8305fd..bf2c45ef4ef 100644 --- a/ddtrace/profiling/collector/stack.py +++ b/ddtrace/profiling/collector/stack.py @@ -22,7 +22,10 @@ # Provide a no-op stub so StackCollector can be imported on Python 3.15. import types as _types - _task = _types.SimpleNamespace(initialize_gevent_support=lambda: None) # type: ignore[assignment] + def _initialize_gevent_support() -> None: + return None + + _task = _types.SimpleNamespace(initialize_gevent_support=_initialize_gevent_support) # type: ignore[assignment] from ddtrace.profiling.collector import threading from ddtrace.trace import Tracer diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 9cbb2e2bda3..53e1406bbae 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -2,6 +2,8 @@ import logging from unittest import mock +import pytest + from ddtrace.profiling import scheduler @@ -29,8 +31,8 @@ def call_me(): assert x["OK"] -def test_before_flush_failure(caplog): - def call_me(): +def test_before_flush_failure(caplog: pytest.LogCaptureFixture) -> None: + def call_me() -> None: raise Exception("LOL") s = scheduler.Scheduler(before_flush=call_me) From 10fd21c0ec18e46641f074e7758e861b3bd78f8e Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 21 Aug 2026 07:52:05 +0300 Subject: [PATCH 5/5] ci(profiling): wire py3.15 into build matrix, riotfile, and CI Profiling-native py3.15 job, setup.py guards, and crashtracker 3.15 opt-in. Rebased onto the #17849 split stack (PR 17624). # Conflicts: # riotfile.py # setup.py --- .../workflows/generate-package-versions.yml | 5 +++ .gitlab-ci.yml | 16 +++++++ .gitlab/templates/build-base-venvs.yml | 2 + .riot/requirements/1c6cb02.txt | 34 ++++++++++++++ .riot/requirements/222bcd0.txt | 45 +++++++++++++++++++ .riot/requirements/95077af.txt | 33 ++++++++++++++ .riot/requirements/e26245b.txt | 37 +++++++++++++++ riotfile.py | 7 +-- scripts/requirements_to_csv.py | 4 +- setup.py | 11 ++++- 10 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 .riot/requirements/1c6cb02.txt create mode 100644 .riot/requirements/222bcd0.txt create mode 100644 .riot/requirements/95077af.txt create mode 100644 .riot/requirements/e26245b.txt diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml index 955d1552cb4..e742ecfb564 100644 --- a/.github/workflows/generate-package-versions.yml +++ b/.github/workflows/generate-package-versions.yml @@ -53,6 +53,11 @@ jobs: with: python-version: "3.14" + - name: Setup Python 3.15 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.15" + - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7620cfeedf2..cc047127451 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -703,6 +703,22 @@ profiling_native: - PYTHON_VERSION: ["3.12", "3.14"] SANITIZER: ["valgrind"] +# AIDEV-TODO(py315): fold 3.15 back into the profiling_native matrix once the +# dd/images/dd-trace-py/profiling_native base image installs a Python 3.15 +# pyenv version. Today it only carries 3.9–3.14, so PYENV_VERSION=3.15 fails +# with "pyenv: version `3.15' is not installed". allow_failure keeps the +# pipeline green while the upstream image catches up; delete this job and +# re-add "3.15" to the PYTHON_VERSION arrays above once the image is updated. +profiling_native_py315: + extends: .profiling_native_base + allow_failure: true + retry: 2 + rules: !reference [profiling_native, rules] + parallel: + matrix: + - PYTHON_VERSION: ["3.15"] + SANITIZER: ["safety", "thread", "", "valgrind"] + test-dd-sts: stage: tests needs: [] diff --git a/.gitlab/templates/build-base-venvs.yml b/.gitlab/templates/build-base-venvs.yml index dc32b3e40d8..797b73938df 100644 --- a/.gitlab/templates/build-base-venvs.yml +++ b/.gitlab/templates/build-base-venvs.yml @@ -49,3 +49,5 @@ build_base_venvs: - core.* - ddtrace/**/*.so* - .riot/venv_* + - ddtrace/internal/datadog/profiling/test/test_* + - ddtrace/internal/datadog/profiling/test/py315/test_* diff --git a/.riot/requirements/1c6cb02.txt b/.riot/requirements/1c6cb02.txt new file mode 100644 index 00000000000..b9724685738 --- /dev/null +++ b/.riot/requirements/1c6cb02.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1c6cb02.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gunicorn==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uvloop==0.22.1 +uwsgi==2.0.31 +zstandard==0.25.0 diff --git a/.riot/requirements/222bcd0.txt b/.riot/requirements/222bcd0.txt new file mode 100644 index 00000000000..4ec85bb3a9d --- /dev/null +++ b/.riot/requirements/222bcd0.txt @@ -0,0 +1,45 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +<<<<<<<< HEAD:.riot/requirements/222bcd0.txt +# pip-compile --allow-unsafe --no-annotate .riot/requirements/222bcd0.in +# +attrs==26.1.0 +cloudpickle==3.1.2 +coverage[toml]==7.14.3 +execnet==2.1.2 +gevent==26.5.0 +greenlet==3.5.3 +httpretty==1.1.4 +======== +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1857594.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +>>>>>>>> 49c1ffeaab (ci(profiling): wire py3.15 into build matrix, riotfile, and CI):.riot/requirements/1857594.txt +hypothesis==6.45.0 +iniconfig==2.3.0 +mock==5.2.0 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +pyfakefs==6.2.0 +pygments==2.20.0 +pytest==8.4.2 +pytest-asyncio==0.23.8 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +python-json-logger==2.0.7 +sortedcontainers==2.4.0 +<<<<<<<< HEAD:.riot/requirements/222bcd0.txt +uwsgi==2.0.31 +wrapt==2.2.2 +======== +>>>>>>>> 49c1ffeaab (ci(profiling): wire py3.15 into build matrix, riotfile, and CI):.riot/requirements/1857594.txt +zope-event==5.0 +zope-interface==7.2 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==81.0.0 diff --git a/.riot/requirements/95077af.txt b/.riot/requirements/95077af.txt new file mode 100644 index 00000000000..cb9c2e03246 --- /dev/null +++ b/.riot/requirements/95077af.txt @@ -0,0 +1,33 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/95077af.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gunicorn==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uwsgi==2.0.31 +zstandard==0.25.0 diff --git a/.riot/requirements/e26245b.txt b/.riot/requirements/e26245b.txt new file mode 100644 index 00000000000..9fc91691061 --- /dev/null +++ b/.riot/requirements/e26245b.txt @@ -0,0 +1,37 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e26245b.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gevent==26.4.0 +greenlet==3.4.0 +gunicorn[gevent]==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uwsgi==2.0.31 +zope-event==6.1 +zope-interface==8.3 +zstandard==0.25.0 diff --git a/riotfile.py b/riotfile.py index 10c3c9e6eaf..e783918bd63 100644 --- a/riotfile.py +++ b/riotfile.py @@ -591,7 +591,8 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pys=select_pys(min_version="3.9", max_version="3.11"), ), Venv( - pys=select_pys(min_version="3.12"), + # TODO(py-315): 3.15 explicitly opted in for crashtracker native validation. + pys=select_pys(min_version="3.12", max_version="3.14") + ["3.15"], env={ "PYTHONWARNINGS": "ignore:This process:DeprecationWarning::", }, @@ -2197,7 +2198,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys="3.14", + pys=["3.14", "3.15"], pkgs={ "grpcio": ">=1.75.0", }, @@ -3811,7 +3812,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), ], ), - # Python 3.14 - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new) + # Python 3.14+ - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new) Venv( pys="3.14", pkgs={"uwsgi": latest}, diff --git a/scripts/requirements_to_csv.py b/scripts/requirements_to_csv.py index ed25c87b9ef..50d041e511b 100644 --- a/scripts/requirements_to_csv.py +++ b/scripts/requirements_to_csv.py @@ -2,7 +2,7 @@ import os import re -import toml +import toml # type: ignore[import-untyped] def requirements_to_csv(): @@ -52,7 +52,7 @@ def process_deps(dependencies): if "lib-injection" in path: os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", newline="") as f: - writer = csv.writer(f) + writer = csv.writer(f, lineterminator="\n") writer.writerows(rows) diff --git a/setup.py b/setup.py index 40749a6a119..994a07fe1b8 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,15 @@ CARGO_TARGET_DIR = NATIVE_CRATE.absolute() / f"target{sys.version_info.major}.{sys.version_info.minor}" DD_CARGO_ARGS = shlex.split(os.getenv("DD_CARGO_ARGS", "")) +# TODO(py-315): pyo3-build-config 0.27.x (max Python 3.14) may be resolved by cargo +# if the lock file is regenerated without --locked (e.g. in some CI cache scenarios). +# Setting PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 tells pyo3-build-config to bypass +# the max-version check and build via the stable ABI, which is correct since we +# already use py_limited_api="auto" in the RustExtension definition. +# pyo3 0.28+ supports Python 3.15 natively, so this is only a safety net. +if sys.version_info >= (3, 15): + os.environ.setdefault("PYO3_USE_ABI3_FORWARD_COMPATIBILITY", "1") + def _env_truthy(name: str, default: str = "0") -> bool: return os.getenv(name, default).lower() in ("1", "yes", "on", "true") @@ -1830,7 +1839,7 @@ def check_rust_toolchain(): ), ] - if sys.version_info < (3, 15): + if sys.version_info < (3, 16): _cython_sources += [ CythonExtension( "ddtrace.profiling._threading",