From b424fc591e11a22de4ff163d1d8d7942bc25e5f4 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Tue, 5 May 2026 11:34:54 -0300 Subject: [PATCH 1/8] build(simd): add xsimd dependency and simd verification --- .github/workflows/unit-tests.yml | 41 ++++++++++++++++++++++ LICENSES/XSIMD_LICENSE | 29 +++++++++++++++ meson.build | 4 ++- meson.options | 5 +++ pandas/_libs/meson.build | 3 ++ pandas/_libs/simd/meson.build | 41 ++++++++++++++++++++++ pyproject.toml | 1 + subprojects/packagefiles/xsimd/meson.build | 12 +++++++ subprojects/xsimd.wrap | 9 +++++ 9 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 LICENSES/XSIMD_LICENSE create mode 100644 meson.options create mode 100644 pandas/_libs/simd/meson.build create mode 100644 subprojects/packagefiles/xsimd/meson.build create mode 100644 subprojects/xsimd.wrap diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 73018750f156d..983f24bcb4b04 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -261,6 +261,47 @@ jobs: pytest_marker_expression: 'not slow and not db and not network and not single_cpu' pytest_target: 'pandas' + Ubuntu-no-SIMD: + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 90 + name: Ubuntu (no SIMD) + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-ubuntu-no-simd + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Create virtual environment with Pixi + uses: ./.github/actions/setup-pixi + with: + environment: py313 + + - name: Build pandas + id: build + run: | + pixi run \ + --environment py313 \ + build-pandas \ + --editable \ + -Csetup-args="--werror" -- \ + -Csetup-args="-Dsimd=disabled" + shell: bash -euox pipefail {0} + + - name: Test + uses: ./.github/actions/run-tests + with: + environment: py313 + numprocesses: 'auto' + pytest_marker_expression: 'not slow and not db and not network and not single_cpu' + pytest_target: 'pandas' + Linux-32-bit: runs-on: ubuntu-24.04 permissions: diff --git a/LICENSES/XSIMD_LICENSE b/LICENSES/XSIMD_LICENSE new file mode 100644 index 0000000000000..eee7a54bc956b --- /dev/null +++ b/LICENSES/XSIMD_LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and Martin Renou +Copyright (c) 2016, QuantStack +Copyright (c) 2018, Serge Guelton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/meson.build b/meson.build index d2874e85cc3a7..33790ca88fb3f 100644 --- a/meson.build +++ b/meson.build @@ -37,6 +37,7 @@ add_project_arguments( ) cc = meson.get_compiler('c') +cxx = meson.get_compiler('cpp') if cc.get_id() == 'msvc' # Tracking issue: https://github.com/pandas-dev/pandas/issues/63701 # Ignore some MSVC specific warnings: @@ -44,8 +45,9 @@ if cc.get_id() == 'msvc' # C4267: conversion from `size_t` to smaller type. # C4551: occurs due to Cython generating code with (void)func. # https://github.com/cython/cython/issues/3579 + # C4146: unary minus operator applied to unsigned type. Occurs in xsimd. add_project_arguments( - ['/wd4244', '/wd4267', '/wd4551'], + ['/wd4244', '/wd4267', '/wd4551', '/wd4146'], language: ['c', 'cpp'], ) endif diff --git a/meson.options b/meson.options new file mode 100644 index 0000000000000..4dd59bfe2fb91 --- /dev/null +++ b/meson.options @@ -0,0 +1,5 @@ +option( + 'simd', + type: 'feature', + description: 'Compile with SIMD instructions', +) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 56ff2a01b450c..a8a411629e2a0 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -53,6 +53,9 @@ _khash_primitive_helper_dep = declare_dependency( m_dep = cc.find_library('m', required: false) fast_float = subproject('fast_float') fast_float_dep = fast_float.get_variable('fast_float_dep') +xsimd_dep = dependency('xsimd', version: '>=14') + +subdir('simd') subdir('tslibs') diff --git a/pandas/_libs/simd/meson.build b/pandas/_libs/simd/meson.build new file mode 100644 index 0000000000000..fa4011bf4760e --- /dev/null +++ b/pandas/_libs/simd/meson.build @@ -0,0 +1,41 @@ +# All architectures we might support +# Key is the architecture name used in file suffixes and macros +is_msvc_syntax = cxx.get_argument_syntax() == 'msvc' +simd_arch_flags = { + 'sse2': is_msvc_syntax ? ['/arch:SSE2'] : ['-msse2'], + 'avx2': is_msvc_syntax ? ['/arch:AVX2'] : ['-mavx2'], + 'avx512cd': is_msvc_syntax ? ['/arch:AVX512'] : ['-mavx512cd'], + 'neon': [], +} + +simd_config = configuration_data() +supported_simd_archs = {} +if get_option('simd').allowed() + foreach name, flags : simd_arch_flags + if host_machine.cpu_family() in ['x86', 'x86_64'] + if name in ['sse2', 'avx2', 'avx512cd'] + if cxx.has_multi_arguments(flags) + supported_simd_archs += {name: flags} + endif + endif + elif host_machine.cpu_family() == 'aarch64' + if name == 'neon' + supported_simd_archs += {name: flags} + endif + endif + endforeach +endif + +foreach arch_name, arch_flags : supported_simd_archs + simd_config.set('PANDAS_HAVE_@0@'.format(arch_name.to_upper()), 1) +endforeach + +# Ensure scalar version on all architectures for now... +simd_config.set('PANDAS_HAVE_SCALAR', 1) + +configure_file( + output: 'pandas_simd_config.h', + configuration: simd_config, +) + +simd_config_inc = include_directories('.') diff --git a/pyproject.toml b/pyproject.toml index f5c2a5f606c1f..d03232583873c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ license-files = [ "LICENSES/PYUPGRADE_LICENSE", # MIT "LICENSES/SAS7BDAT_LICENSE", # MIT "LICENSES/ULTRAJSON_LICENSE", # BSD-3-Clause AND TCL + "LICENSES/XSIMD_LICENSE", # BSD-3-Clause "subprojects/fast_float-*/LICENSE-APACHE", # Apache-2.0 "subprojects/fast_float-*/LICENSE-BOOST", # BSL "subprojects/fast_float-*/LICENSE-MIT", # MIT diff --git a/subprojects/packagefiles/xsimd/meson.build b/subprojects/packagefiles/xsimd/meson.build new file mode 100644 index 0000000000000..c450c0831d3f3 --- /dev/null +++ b/subprojects/packagefiles/xsimd/meson.build @@ -0,0 +1,12 @@ +project( + 'xsimd', + 'cpp', + meson_version: '>=0.58.0', + license: 'BSD-3-Clause', + version: '14.1.0', +) + +xsimd_inc = include_directories('include') + +xsimd_dep = declare_dependency(include_directories: xsimd_inc) +meson.override_dependency('xsimd', xsimd_dep) diff --git a/subprojects/xsimd.wrap b/subprojects/xsimd.wrap new file mode 100644 index 0000000000000..652c0e7c9317d --- /dev/null +++ b/subprojects/xsimd.wrap @@ -0,0 +1,9 @@ +[wrap-file] +directory = xsimd-14.1.0 +source_url = https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.1.0.tar.gz +source_filename = xsimd-14.1.0.tar.gz +source_hash = 8da77b66214e73565f6111ba494322c853552b7b40cff69779b42e2100eed16f +patch_directory = xsimd + +[provide] +dependency_names = xsimd From 74de0653e6d8505f6a1fbc234030406b2024867f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Kothe?= Date: Wed, 6 May 2026 13:31:34 -0300 Subject: [PATCH 2/8] Update pandas/_libs/meson.build Co-authored-by: William Ayd --- pandas/_libs/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index a8a411629e2a0..b93e226dacede 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -53,7 +53,7 @@ _khash_primitive_helper_dep = declare_dependency( m_dep = cc.find_library('m', required: false) fast_float = subproject('fast_float') fast_float_dep = fast_float.get_variable('fast_float_dep') -xsimd_dep = dependency('xsimd', version: '>=14') +xsimd_dep = dependency('xsimd', version: '>=14.1') subdir('simd') From a7961ee93919e14655608b324bb19304109726d9 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Wed, 6 May 2026 13:22:18 -0300 Subject: [PATCH 3/8] ci: remove no simd job --- .github/workflows/unit-tests.yml | 41 -------------------------------- 1 file changed, 41 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 983f24bcb4b04..73018750f156d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -261,47 +261,6 @@ jobs: pytest_marker_expression: 'not slow and not db and not network and not single_cpu' pytest_target: 'pandas' - Ubuntu-no-SIMD: - runs-on: ubuntu-24.04 - permissions: - contents: read - timeout-minutes: 90 - name: Ubuntu (no SIMD) - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-ubuntu-no-simd - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - - - name: Create virtual environment with Pixi - uses: ./.github/actions/setup-pixi - with: - environment: py313 - - - name: Build pandas - id: build - run: | - pixi run \ - --environment py313 \ - build-pandas \ - --editable \ - -Csetup-args="--werror" -- \ - -Csetup-args="-Dsimd=disabled" - shell: bash -euox pipefail {0} - - - name: Test - uses: ./.github/actions/run-tests - with: - environment: py313 - numprocesses: 'auto' - pytest_marker_expression: 'not slow and not db and not network and not single_cpu' - pytest_target: 'pandas' - Linux-32-bit: runs-on: ubuntu-24.04 permissions: From ed01d8a0e29ea7bcd1dbd5de906c44974257798e Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Wed, 6 May 2026 13:30:50 -0300 Subject: [PATCH 4/8] refactor: create arch specific loop --- pandas/_libs/simd/meson.build | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pandas/_libs/simd/meson.build b/pandas/_libs/simd/meson.build index fa4011bf4760e..de7c298c5216c 100644 --- a/pandas/_libs/simd/meson.build +++ b/pandas/_libs/simd/meson.build @@ -1,29 +1,24 @@ # All architectures we might support # Key is the architecture name used in file suffixes and macros is_msvc_syntax = cxx.get_argument_syntax() == 'msvc' -simd_arch_flags = { +simd_x86_flags = { 'sse2': is_msvc_syntax ? ['/arch:SSE2'] : ['-msse2'], 'avx2': is_msvc_syntax ? ['/arch:AVX2'] : ['-mavx2'], 'avx512cd': is_msvc_syntax ? ['/arch:AVX512'] : ['-mavx512cd'], - 'neon': [], } simd_config = configuration_data() supported_simd_archs = {} if get_option('simd').allowed() - foreach name, flags : simd_arch_flags - if host_machine.cpu_family() in ['x86', 'x86_64'] - if name in ['sse2', 'avx2', 'avx512cd'] - if cxx.has_multi_arguments(flags) - supported_simd_archs += {name: flags} - endif - endif - elif host_machine.cpu_family() == 'aarch64' - if name == 'neon' + if host_machine.cpu_family() == 'aarch64' + supported_simd_archs += {'neon': []} + elif host_machine.cpu_family() in ['x86', 'x86_64'] + foreach name, flags : simd_x86_flags + if cxx.has_multi_arguments(flags) supported_simd_archs += {name: flags} endif - endif - endforeach + endforeach + endif endif foreach arch_name, arch_flags : supported_simd_archs From 99fb9bc484a5d30870aa3074105d6b90fbb24803 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Wed, 6 May 2026 13:38:05 -0300 Subject: [PATCH 5/8] refactor: move configuration set inside main verification --- pandas/_libs/simd/meson.build | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/simd/meson.build b/pandas/_libs/simd/meson.build index de7c298c5216c..dfbc1de5d5bda 100644 --- a/pandas/_libs/simd/meson.build +++ b/pandas/_libs/simd/meson.build @@ -12,19 +12,17 @@ supported_simd_archs = {} if get_option('simd').allowed() if host_machine.cpu_family() == 'aarch64' supported_simd_archs += {'neon': []} + simd_config.set('PANDAS_HAVE_NEON', 1) elif host_machine.cpu_family() in ['x86', 'x86_64'] foreach name, flags : simd_x86_flags if cxx.has_multi_arguments(flags) supported_simd_archs += {name: flags} + simd_config.set('PANDAS_HAVE_@0@'.format(name.to_upper()), 1) endif endforeach endif endif -foreach arch_name, arch_flags : supported_simd_archs - simd_config.set('PANDAS_HAVE_@0@'.format(arch_name.to_upper()), 1) -endforeach - # Ensure scalar version on all architectures for now... simd_config.set('PANDAS_HAVE_SCALAR', 1) From 7775e5a2d2b5ee44d2ac3dbfdb6329fa92358c53 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Wed, 6 May 2026 14:29:56 -0300 Subject: [PATCH 6/8] fix: bump xsimd to 14.2 for MSVC ARM support --- pandas/_libs/meson.build | 2 +- subprojects/packagefiles/xsimd/meson.build | 2 +- subprojects/xsimd.wrap | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index b93e226dacede..674901292979d 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -53,7 +53,7 @@ _khash_primitive_helper_dep = declare_dependency( m_dep = cc.find_library('m', required: false) fast_float = subproject('fast_float') fast_float_dep = fast_float.get_variable('fast_float_dep') -xsimd_dep = dependency('xsimd', version: '>=14.1') +xsimd_dep = dependency('xsimd', version: '>=14.2') subdir('simd') diff --git a/subprojects/packagefiles/xsimd/meson.build b/subprojects/packagefiles/xsimd/meson.build index c450c0831d3f3..595fb6122ef51 100644 --- a/subprojects/packagefiles/xsimd/meson.build +++ b/subprojects/packagefiles/xsimd/meson.build @@ -3,7 +3,7 @@ project( 'cpp', meson_version: '>=0.58.0', license: 'BSD-3-Clause', - version: '14.1.0', + version: '14.2.0', ) xsimd_inc = include_directories('include') diff --git a/subprojects/xsimd.wrap b/subprojects/xsimd.wrap index 652c0e7c9317d..39706456be925 100644 --- a/subprojects/xsimd.wrap +++ b/subprojects/xsimd.wrap @@ -1,8 +1,8 @@ [wrap-file] -directory = xsimd-14.1.0 -source_url = https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.1.0.tar.gz -source_filename = xsimd-14.1.0.tar.gz -source_hash = 8da77b66214e73565f6111ba494322c853552b7b40cff69779b42e2100eed16f +directory = xsimd-14.2.0 +source_url = https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.2.0.tar.gz +source_filename = xsimd-14.2.0.tar.gz +source_hash = 21e841ab684b05331e81e7f782431753a029ef7b7d9d6d3ddab837e7782a40ee patch_directory = xsimd [provide] From 997cedbdb3877500cd4953712d59d3c0638d4de7 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Fri, 8 May 2026 16:04:18 -0300 Subject: [PATCH 7/8] build: remove simd option --- meson.options | 5 ----- pandas/_libs/simd/meson.build | 22 ++++++++++------------ 2 files changed, 10 insertions(+), 17 deletions(-) delete mode 100644 meson.options diff --git a/meson.options b/meson.options deleted file mode 100644 index 4dd59bfe2fb91..0000000000000 --- a/meson.options +++ /dev/null @@ -1,5 +0,0 @@ -option( - 'simd', - type: 'feature', - description: 'Compile with SIMD instructions', -) diff --git a/pandas/_libs/simd/meson.build b/pandas/_libs/simd/meson.build index dfbc1de5d5bda..21c7f59624bda 100644 --- a/pandas/_libs/simd/meson.build +++ b/pandas/_libs/simd/meson.build @@ -9,18 +9,16 @@ simd_x86_flags = { simd_config = configuration_data() supported_simd_archs = {} -if get_option('simd').allowed() - if host_machine.cpu_family() == 'aarch64' - supported_simd_archs += {'neon': []} - simd_config.set('PANDAS_HAVE_NEON', 1) - elif host_machine.cpu_family() in ['x86', 'x86_64'] - foreach name, flags : simd_x86_flags - if cxx.has_multi_arguments(flags) - supported_simd_archs += {name: flags} - simd_config.set('PANDAS_HAVE_@0@'.format(name.to_upper()), 1) - endif - endforeach - endif +if host_machine.cpu_family() == 'aarch64' + supported_simd_archs += {'neon': []} + simd_config.set('PANDAS_HAVE_NEON', 1) +elif host_machine.cpu_family() in ['x86', 'x86_64'] + foreach name, flags : simd_x86_flags + if cxx.has_multi_arguments(flags) + supported_simd_archs += {name: flags} + simd_config.set('PANDAS_HAVE_@0@'.format(name.to_upper()), 1) + endif + endforeach endif # Ensure scalar version on all architectures for now... From aab9418ced780a2f4d25ce7116708e1ae69cb272 Mon Sep 17 00:00:00 2001 From: Alvaro-Kothe Date: Tue, 19 May 2026 18:19:49 -0300 Subject: [PATCH 8/8] build(simd): target only baseline for x86-64 and aarch64 --- pandas/_libs/simd/meson.build | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/pandas/_libs/simd/meson.build b/pandas/_libs/simd/meson.build index 21c7f59624bda..61925a1d90927 100644 --- a/pandas/_libs/simd/meson.build +++ b/pandas/_libs/simd/meson.build @@ -1,29 +1,17 @@ -# All architectures we might support -# Key is the architecture name used in file suffixes and macros -is_msvc_syntax = cxx.get_argument_syntax() == 'msvc' -simd_x86_flags = { - 'sse2': is_msvc_syntax ? ['/arch:SSE2'] : ['-msse2'], - 'avx2': is_msvc_syntax ? ['/arch:AVX2'] : ['-mavx2'], - 'avx512cd': is_msvc_syntax ? ['/arch:AVX512'] : ['-mavx512cd'], -} +host_cpu = host_machine.cpu_family() simd_config = configuration_data() -supported_simd_archs = {} -if host_machine.cpu_family() == 'aarch64' - supported_simd_archs += {'neon': []} +if host_cpu == 'aarch64' + simd_target = 'neon' simd_config.set('PANDAS_HAVE_NEON', 1) -elif host_machine.cpu_family() in ['x86', 'x86_64'] - foreach name, flags : simd_x86_flags - if cxx.has_multi_arguments(flags) - supported_simd_archs += {name: flags} - simd_config.set('PANDAS_HAVE_@0@'.format(name.to_upper()), 1) - endif - endforeach +elif host_cpu == 'x86_64' + simd_target = 'sse2' + simd_config.set('PANDAS_HAVE_SSE2', 1) +else + simd_target = 'scalar' + simd_config.set('PANDAS_HAVE_SCALAR', 1) endif -# Ensure scalar version on all architectures for now... -simd_config.set('PANDAS_HAVE_SCALAR', 1) - configure_file( output: 'pandas_simd_config.h', configuration: simd_config,