PKG-15601 + PKG-2624 — CF-align freethreading and debug build_type

recipe/build_base.bat

@@ -1,6 +1,9 @@
 setlocal EnableDelayedExpansion
 echo on
 
+:: Avoids fetching nuget.exe from the internet.
+set PYTHON=%CONDA_PYTHON_EXE%
+
 :: Compile python, extensions and external libraries
 if "%ARCH%"=="64" (
    set PLATFORM=x64
@@ -44,7 +47,7 @@ if "%DEBUG_C%"=="yes" (
   set PGO=--pgo
 )
 
-if "%PY_GIL_DISABLED%" == "yes" (
+if "%PY_FREETHREADING%" == "yes" (
   set "FREETHREADING=--disable-gil"
   set "THREAD=t"
   set "EXE_T=%VER%t"

recipe/build_base.sh

@@ -33,46 +33,33 @@ _buildd_shared=build-shared
 _ENABLE_SHARED=--enable-shared
 # We *still* build a shared lib here for non-static embedded use cases
 _DISABLE_SHARED=--disable-shared
-# Hack to allow easily comparing static vs shared interpreter performance
-# .. hack because we just build it shared in both the build-static and
-# build-shared directories.
-# Yes this hack is a bit confusing, sorry about that.
-if [[ ${PY_INTERP_LINKAGE_NATURE} == shared ]]; then
-  _DISABLE_SHARED=--enable-shared
-  _ENABLE_SHARED=--enable-shared
-fi
 
 # For debugging builds, set this to no to disable profile-guided optimization
-if [[ ${DEBUG_C} == yes ]]; then
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
   _OPTIMIZED=no
 else
   _OPTIMIZED=yes
 fi
 
-if [[ ${target_platform} == linux-ppc64le ]]; then
-  _OPTIMIZED=no
-  # ppc64le cdt need to be rebuilt with files in powerpc64le-conda-linux-gnu instead of powerpc64le-conda_cos7-linux-gnu. In the mean time:
-  cp --force --archive --update --link $BUILD_PREFIX/powerpc64le-conda_cos7-linux-gnu/. $BUILD_PREFIX/powerpc64le-conda-linux-gnu
-fi
-
 declare -a _dbg_opts
-if [[ ${DEBUG_PY} == yes ]]; then
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
   # This Python will not be usable with non-debug Python modules.
   _dbg_opts+=(--with-pydebug)
   DBG=d
 else
   DBG=
 fi
 
-if [[ ${PY_GIL_DISABLED} == yes ]]; then
+if [[ ${PY_FREETHREADING} == yes ]]; then
   # This Python will not be usable with non-free threading Python modules.
   THREAD=t
 else
   THREAD=
 fi
 
-ABIFLAGS=${DBG}
+ABIFLAGS=${DBG}${THREAD}
 VERABI=${VER}${THREAD}${DBG}
+VERABI_NO_DBG=${VER}${THREAD}
 
 # Make sure the "python" value in conda_build_config.yaml is up to date.
 test "${PY_VER}" = "${VER}"
@@ -83,7 +70,7 @@ test "${PY_VER}" = "${VER}"
 unset _PYTHON_SYSCONFIGDATA_NAME
 unset _CONDA_PYTHON_SYSCONFIGDATA_NAME
 
-# Prevent lib/python${VERABI}/_sysconfigdata_*.py from ending up with full paths to these things
+# Prevent lib/python${VERABI_NO_DBG}/_sysconfigdata_*.py from ending up with full paths to these things
 # in _build_env because _build_env will not get found during prefix replacement, only _h_env_placeh ...
 AR=$(basename "${AR}")
 
@@ -111,6 +98,12 @@ if [[ ${_OPTIMIZED} = yes ]]; then
   CXXFLAGS=$(echo "${CXXFLAGS}" | sed "s/-O2/-O3/g")
 fi
 
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
+  CPPFLAGS=$(echo "${CPPFLAGS}" | sed "s/-O2/-O0/g")
+  CFLAGS=$(echo "${CFLAGS}" | sed "s/-O2/-O0/g")
+  CXXFLAGS=$(echo "${CXXFLAGS}" | sed "s/-O2/-O0/g")
+fi
+
 if [[ "$target_platform" == linux-* ]]; then
   cp ${PREFIX}/include/uuid/uuid.h ${PREFIX}/include/uuid.h
 fi
@@ -258,7 +251,7 @@ _common_configure_args+=("--with-tcltk-libs=-L${PREFIX}/lib -ltcl${TCLTK_VER} -l
 _common_configure_args+=(--with-platlibdir=lib)
 _common_configure_args+=(--with-system-libmpdec=yes)
 
-if [[ "${DEBUG_PY}" == "yes" || "${target_platform}" != *"64" || ${PY_GIL_DISABLED} == yes ]]; then
+if [[ "${PY_INTERP_DEBUG}" == "yes" || "${target_platform}" != *"-64" || ${PY_FREETHREADING} == yes ]]; then
  _common_configure_args+=(--enable-experimental-jit=no)
 else
  _common_configure_args+=(--enable-experimental-jit=yes-off)
@@ -269,7 +262,7 @@ if [[ "${target_platform}" == osx-* ]]; then
     _common_configure_args+=(--with-tail-call-interp)
 fi
 
-if [[ ${PY_GIL_DISABLED} == yes ]]; then
+if [[ ${PY_FREETHREADING} == yes ]]; then
     _common_configure_args+=(--disable-gil)
 fi
 
@@ -365,7 +358,7 @@ fi
 # build a static library with PIC objects and without LTO/PGO
 make -j${CPU_COUNT} -C ${_buildd_shared} \
         EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
-        LIBRARY=libpython${VERABI}-pic.a libpython${VERABI}-pic.a
+        LIBRARY=libpython${VERABI_NO_DBG}-pic.a libpython${VERABI_NO_DBG}-pic.a
 
 make -C ${_buildd_static} install
 
@@ -397,15 +390,17 @@ fi
 SYSCONFIG=$(find ${_buildd_static}/$(cat ${_buildd_static}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0)
 cat ${SYSCONFIG} | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \
   "${_FLAGS_REPLACE[@]}"  \
-    > ${PREFIX}/lib/python${VERABI}/$(basename ${SYSCONFIG})
-MAKEFILE=$(find ${PREFIX}/lib/python${VERABI}/ -path "*config-*/Makefile" -print0)
+    > ${PREFIX}/lib/python${VERABI_NO_DBG}/$(basename ${SYSCONFIG})
+BUILD_DETAILS=${_buildd_shared}/$(cat ${_buildd_shared}/pybuilddir.txt)/build-details.json
+cp ${BUILD_DETAILS} ${PREFIX}/lib/python${VERABI_NO_DBG}/
+MAKEFILE=$(find ${PREFIX}/lib/python${VERABI_NO_DBG}/ -path "*config-*/Makefile" -print0)
 cp ${MAKEFILE} /tmp/Makefile-$$
 cat /tmp/Makefile-$$ | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \
   "${_FLAGS_REPLACE[@]}"  \
     > ${MAKEFILE}
 # Check to see that our differences took.
-# echo diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VERABI}/$(basename ${SYSCONFIG})
-# diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VERABI}/$(basename ${SYSCONFIG})
+# echo diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VERABI_NO_DBG}/$(basename ${SYSCONFIG})
+# diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VERABI_NO_DBG}/$(basename ${SYSCONFIG})
 
 # Python installs python${VER}m and python${VER}, one as a hardlink to the other. conda-build breaks these
 # by copying. Since the executable may be static it may be very large so change one to be a symlink
@@ -424,7 +419,7 @@ ln -s ${PREFIX}/bin/python${VER} ${PREFIX}/bin/python3.1
 # Remove test data to save space
 # Though keep `support` as some things use that.
 # TODO :: Make a subpackage for this once we implement multi-level testing.
-pushd ${PREFIX}/lib/python${VERABI}
+pushd ${PREFIX}/lib/python${VERABI_NO_DBG}
   mkdir test_keep
   mv test/__init__.py test/support test/test_support* test/test_script_helper* test_keep/
   rm -rf test */test
@@ -437,33 +432,17 @@ pushd ${PREFIX}
     chmod +w lib/libpython${VERABI}.a
     ${STRIP} -S lib/libpython${VERABI}.a
   fi
-  CONFIG_LIBPYTHON=$(find lib/python${VERABI}/config-${VERABI}* -name "libpython${VERABI}.a")
+  CONFIG_LIBPYTHON=$(find lib/python${VERABI_NO_DBG}/config-${VERABI}* -name "libpython${VERABI}.a")
   if [[ -f lib/libpython${VERABI}.a ]] && [[ -f ${CONFIG_LIBPYTHON} ]]; then
     chmod +w ${CONFIG_LIBPYTHON}
     rm ${CONFIG_LIBPYTHON}
   fi
 popd
 
-# OLD_HOST is with CentOS version in them. When building this recipe
-# with the compilers from conda-forge OLD_HOST != HOST, but when building
-# with the compilers from defaults OLD_HOST == HOST. Both cases are handled in the
-# code below
-case "$target_platform" in
-  linux-64)
-    OLD_HOST=$(echo ${HOST} | sed -e 's/-conda-/-conda_cos6-/g')
-    ;;
-  linux-*)
-    OLD_HOST=$(echo ${HOST} | sed -e 's/-conda-/-conda_cos7-/g')
-    ;;
-  *)
-    OLD_HOST=$HOST
-    ;;
-esac
-
 # Copy sysconfig that gets recorded to a non-default name
 # using the new compilers with python will require setting _PYTHON_SYSCONFIGDATA_NAME
 # to the name of this file (minus the .py extension)
-pushd "${PREFIX}"/lib/python${VERABI}
+pushd "${PREFIX}"/lib/python${VERABI_NO_DBG}
   # On Python 3.5 _sysconfigdata.py was getting copied in here and compiled for some reason.
   # This breaks our attempt to find the right one as recorded_name.
   find lib-dynload -name "_sysconfigdata*.py*" -exec rm {} \;
@@ -478,9 +457,9 @@ pushd "${PREFIX}"/lib/python${VERABI}
   # Append the conda-forge zoneinfo to the end
   sed -i.bak "s@zoneinfo'@zoneinfo:$PREFIX/share/tzinfo'@g" sysconfigfile
   # Remove osx sysroot as it depends on the build machine
-  sed -i.bak "s@-isysroot @@g" sysconfigfile
-  # make sure $CONDA_BUILD_SYSROOT is not empty ...
-  if [[ ${HOST} =~ .*darwin.* ]] && [[ -n ${CONDA_BUILD_SYSROOT} ]]; then 
+  # be sure CONDA_BUILD_SYSROOT has value, as other we will remove here instead spaces
+  if [[ "${target_platform}" == osx-* ]] && [[ -n ${CONDA_BUILD_SYSROOT} ]]; then
+    sed -i.bak "s@-isysroot @@g" sysconfigfile
     sed -i.bak "s@$CONDA_BUILD_SYSROOT @@g" sysconfigfile
   fi
   # Remove unfilled config option
@@ -490,16 +469,12 @@ pushd "${PREFIX}"/lib/python${VERABI}
   sed -i.bak "s/'GNULD': 'yes'/'GNULD': 'no'/g" sysconfigfile
   cp sysconfigfile ${our_compilers_name}
 
-  sed -i.bak "s@${HOST}@${OLD_HOST}@g" sysconfigfile
-  old_compiler_name=_sysconfigdata_$(echo ${OLD_HOST} | sed -e 's/[.-]/_/g').py
-  cp sysconfigfile ${old_compiler_name}
-
   # For system gcc remove the triple
-  sed -i.bak "s@$OLD_HOST-c++@g++@g" sysconfigfile
-  sed -i.bak "s@$OLD_HOST-@@g" sysconfigfile
+  sed -i.bak "s@$HOST-c++@g++@g" sysconfigfile
+  sed -i.bak "s@$HOST-@@g" sysconfigfile
   if [[ "$target_platform" == linux* ]]; then
     # For linux, make sure the system gcc uses our linker
-    sed -i.bak "s@-pthread@-pthread -B $PREFIX/compiler_compat@g" sysconfigfile
+    sed -i.bak "s@-pthread@-pthread -B $PREFIX/share/python_compiler_compat@g" sysconfigfile
   fi
   # Don't set -march and -mtune for system gcc
   sed -i.bak "s@-march=[^( |\\\"|\\\')]*@@g" sysconfigfile
@@ -524,14 +499,22 @@ pushd "${PREFIX}"/lib/python${VERABI}
 popd
 
 if [[ ${HOST} =~ .*linux.* ]]; then
-  mkdir -p ${PREFIX}/compiler_compat
-  ln -s ${PREFIX}/bin/${HOST}-ld ${PREFIX}/compiler_compat/ld
-  echo "Files in this folder are to enhance backwards compatibility of anaconda software with older compilers."   > ${PREFIX}/compiler_compat/README
-  echo "See: https://github.com/conda/conda/issues/6030 for more information."                                   >> ${PREFIX}/compiler_compat/README
+  mkdir -p ${PREFIX}/share/python_compiler_compat
+  ln -s ${PREFIX}/bin/${HOST}-ld ${PREFIX}/share/python_compiler_compat/ld
+  echo "Files in this folder are to enhance backwards compatibility of anaconda software with older compilers."   > ${PREFIX}/share/python_compiler_compat/README
+  echo "See: https://github.com/conda/conda/issues/6030 for more information."                                   >> ${PREFIX}/share/python_compiler_compat/README
 fi
 
 python -c "import compileall,os;compileall.compile_dir(os.environ['PREFIX'])"
 rm ${PREFIX}/lib/libpython${VERABI}.a
+
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
+  rm ${PREFIX}/bin/python${VER}
+  ln -s ${PREFIX}/bin/python${VERABI} ${PREFIX}/bin/python${VER}
+  ln -s ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} ${PREFIX}/lib/libpython${VERABI_NO_DBG}${SHLIB_EXT}
+  ln -s ${PREFIX}/include/python${VERABI} ${PREFIX}/include/python${VER}
+fi
+
 if [[ "$target_platform" == linux-* ]]; then
   rm ${PREFIX}/include/uuid.h
 fi
@@ -549,7 +532,7 @@ fi
 # <prefix>/lib/python3.13t/site-packages.
 # Note that these directories are not added to sys.path if they do not exist.
 SP_DIR="${PREFIX}/lib/python${PY_VER}${THREAD}/site-packages"
-if [[ ${PY_GIL_DISABLED} == yes ]]; then
+if [[ ${PY_FREETHREADING} == yes ]]; then
     echo "${PREFIX}/lib/python${PY_VER}/site-packages" >> $SP_DIR/conda-site.pth
 fi
 # Workaround for https://github.com/conda/conda/issues/10969

recipe/build_static.sh

@@ -1,46 +1,33 @@
 #!/bin/bash
 set -ex
 
-cd ${SRC_DIR}
-
 _buildd_static=build-static
 _buildd_shared=build-shared
-if [[ ${DEBUG_PY} == yes ]]; then
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
   DBG=d
 else
   DBG=
 fi
-if [[ ${PY_GIL_DISABLED} == yes ]]; then
+if [[ ${PY_FREETHREADING} == yes ]]; then
+  # This Python will not be usable with non-free threading Python modules.
   THREAD=t
 else
   THREAD=
 fi
-VER=${PKG_VERSION%.*}
-VERABI=${VER}${DBG}${THREAD}
 
-case "$target_platform" in
-  linux-64)
-    OLD_HOST=$(echo ${HOST} | sed -e 's/-conda_cos6//g')
-    OLD_HOST=$(echo ${OLD_HOST} | sed -e 's/-conda_cos7//g')
-    OLD_HOST=$(echo ${OLD_HOST} | sed -e 's/-conda//g')
-    ;;
-  linux-*)
-    OLD_HOST=$(echo ${HOST} | sed -e 's/-conda_cos7//g')
-    OLD_HOST=$(echo ${OLD_HOST} | sed -e 's/-conda//g')
-    ;;
-  *)
-    OLD_HOST=$HOST
-    ;;
-esac
+VER=${PKG_VERSION%.*}
+ABIFLAGS=${DBG}${THREAD}
+VERABI=${VER}${THREAD}${DBG}
+VERABI_NO_DBG=${VER}${THREAD}
 
 cp -pf ${_buildd_static}/libpython${VERABI}.a ${PREFIX}/lib/libpython${VERABI}.a
 if [[ ${HOST} =~ .*linux.* ]]; then
-  pushd ${PREFIX}/lib/python${VERABI}/config-${VERABI}-${OLD_HOST}
+  pushd ${PREFIX}/lib/python${VERABI_NO_DBG}/config-${VERABI}-${HOST/-conda/}
 elif [[ ${HOST} =~ .*darwin.* ]]; then
-  pushd ${PREFIX}/lib/python${VERABI}/config-${VERABI}-darwin
+  pushd ${PREFIX}/lib/python${VERABI_NO_DBG}/config-${VERABI}-darwin
 fi
 ln -s ../../libpython${VERABI}.a libpython${VERABI}.a
 popd
 # If the LTO info in the normal lib is problematic (using different compilers for example
 # we also provide a 'nolto' version).
-cp -pf ${_buildd_shared}/libpython${VERABI}-pic.a ${PREFIX}/lib/libpython${VERABI}.nolto.a
+cp -pf ${_buildd_shared}/libpython${VERABI_NO_DBG}-pic.a ${PREFIX}/lib/libpython${VERABI_NO_DBG}.nolto.a

recipe/conda_build_config.yaml

@@ -4,7 +4,9 @@ python_impl:
   - cpython
 numpy:
   - 2.5
-gil_type:
-  - normal
-  # Will be enabled as part of https://anaconda.atlassian.net/browse/PKG-5855
-  - disabled # [not (s390x or (osx and x86_64))]
+freethreading:
+  - yes
+  - no  # [not (s390x or (osx and x86_64))]
+build_type:
+  - release
+  - debug  # [not win]