From e5ec764646d6763b7fbe9563dfc1efd11b2ac11c Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Thu, 25 Sep 2025 15:26:26 +0100 Subject: [PATCH 01/27] Removed submodule pybind11 --- .bettercodehub.yml | 21 ------------------- .gitmodules | 5 +---- pybind11 | 1 - src/{ => asmc/preparedecoding}/__init__.py | 0 .../preparedecoding}/decoding_quantities.py | 0 5 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 .bettercodehub.yml delete mode 160000 pybind11 rename src/{ => asmc/preparedecoding}/__init__.py (100%) rename src/{ => asmc/preparedecoding}/decoding_quantities.py (100%) diff --git a/.bettercodehub.yml b/.bettercodehub.yml deleted file mode 100644 index 0c919c5..0000000 --- a/.bettercodehub.yml +++ /dev/null @@ -1,21 +0,0 @@ -component_depth: 1 -languages: - - name: cpp - production: - include: - - /exe/.*\.cpp - exclude: - - /cmake/.* - - /docs/.* - - /scripts/.* - - /test/.* - - /vcpkg/.* - - /src/3rd_party/.* - test: - include: - - /test/.*\.cpp - exclude: - - /cmake/.* - - /docs/.* - - /scripts/.* - - /vcpkg/.* diff --git a/.gitmodules b/.gitmodules index 75ff16e..9c10ce8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "vcpkg"] path = vcpkg - url = https://github.com/microsoft/vcpkg -[submodule "pybind11"] - path = pybind11 - url = https://github.com/pybind/pybind11 + url = https://github.com/microsoft/vcpkg \ No newline at end of file diff --git a/pybind11 b/pybind11 deleted file mode 160000 index 0bd8896..0000000 --- a/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0bd8896a4010f2d91b2340570c24fa08606ec406 diff --git a/src/__init__.py b/src/asmc/preparedecoding/__init__.py similarity index 100% rename from src/__init__.py rename to src/asmc/preparedecoding/__init__.py diff --git a/src/decoding_quantities.py b/src/asmc/preparedecoding/decoding_quantities.py similarity index 100% rename from src/decoding_quantities.py rename to src/asmc/preparedecoding/decoding_quantities.py From a43300ff43810dd31f3e9f3c468b582d60decbcf Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Thu, 25 Sep 2025 15:27:17 +0100 Subject: [PATCH 02/27] Replace setup.py with pyproject.toml --- pyproject.toml | 52 ++++++++++++++++++++ setup.py | 125 ------------------------------------------------- 2 files changed, 52 insertions(+), 125 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..af74323 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = [ + "scikit-build-core>=0.11.6", + "pybind11==2.13.6" +] +build-backend = "scikit_build_core.build" + +[project] +dynamic = ["readme"] +name = "asmc-preparedecoding" +version = "2.2.3" +description = "Prepare decoding quantities for ASMC & FastSMC" +authors = [ + { name = "PalamaraLab (https://palamaralab.github.io/)" } +] +requires-python = ">=3.9" + +dependencies = [ + 'numpy', +] + +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] + +[tool.scikit-build] +minimum-version = "build-system.requires" +build.verbose = true +cmake.build-type = "Release" +build.targets = ["preparedecoding_python_bindings"] +wheel.packages = ["src/asmc"] +metadata.readme.provider = "scikit_build_core.metadata.fancy_pypi_readme" + +[tool.scikit-build.cmake.define] +PREPARE_DECODING_PYTHON_BINDINGS = "ON" +PREPARE_DECODING_BUILDING_FROM_PYPROJECT = "ON" + +[tool.hatch.metadata.hooks.fancy-pypi-readme] +content-type = "text/markdown" + +[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]] +path = "PyPI_README.md" + +[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]] +path = "RELEASE_NOTES.md" diff --git a/setup.py b/setup.py deleted file mode 100644 index a5141d1..0000000 --- a/setup.py +++ /dev/null @@ -1,125 +0,0 @@ -# Based on https://github.com/pybind/cmake_example - -import os -import sys -import subprocess - -from setuptools import setup, Extension, find_namespace_packages -from setuptools.command.build_ext import build_ext - -# Convert distutils Windows platform specifiers to CMake -A arguments -PLAT_TO_CMAKE = { - "win32": "Win32", - "win-amd64": "x64", - "win-arm32": "ARM", - "win-arm64": "ARM64", -} - - -# A CMakeExtension needs a sourcedir instead of a file list. -# The name must be the _single_ output extension from the CMake build. -# If you need multiple extensions, see scikit-build. -class CMakeExtension(Extension): - def __init__(self, name, sourcedir=""): - Extension.__init__(self, name, sources=[]) - self.sourcedir = os.path.abspath(sourcedir) - - -class CMakeBuild(build_ext): - - def build_extension(self, ext): - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - - # required for auto-detection of auxiliary "native" libs - if not extdir.endswith(os.path.sep): - extdir += os.path.sep - - cfg = "Debug" if self.debug else "Release" - - # CMake lets you override the generator - we need to check this. - # Can be set with Conda-Build, for example. - cmake_generator = os.environ.get("CMAKE_GENERATOR", "") - - # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON - # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code - # from Python. - cmake_args = [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", - f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DCMAKE_BUILD_TYPE={cfg}", - f"-DWARNINGS_AS_ERRORS=OFF", - f"-DENABLE_TESTING=OFF", - f"-DMAKE_DOCS=OFF", - ] - build_args = [] - - if self.compiler.compiler_type != "msvc": - # Using Ninja-build since it a) is available as a wheel and b) - # multithreads automatically. MSVC would require all variables be - # exported for Ninja to pick it up, which is a little tricky to do. - # Users can override the generator with CMAKE_GENERATOR in CMake - # 3.15+. - if not cmake_generator: - cmake_args += ["-GNinja"] - - else: - - # Single config generators are handled "normally" - single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) - - # CMake allows an arch-in-generator style for backward compatibility - contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) - - # Specify the arch if using MSVC generator, but only if it doesn't - # contain a backward-compatibility arch spec already in the - # generator name. - if not single_config and not contains_arch: - cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] - - # Multi-config generators have a different way to specify configs - if not single_config: - cmake_args += [ - "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) - ] - build_args += ["--config", cfg] - - # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level - # across all generators. - if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: - # self.parallel is a Python 3 only way to set parallel jobs by hand - # using -j in the build_ext call, not supported by pip or PyPA-build. - if hasattr(self, "parallel") and self.parallel: - # CMake 3.12+ only. - build_args += ["-j{}".format(self.parallel)] - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - - subprocess.check_call( - ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp - ) - subprocess.check_call( - ["cmake", "--build", "."] + build_args, cwd=self.build_temp - ) - - -with open('PyPI_README.md', encoding='utf-8') as f: - long_description = f.read() - -with open('RELEASE_NOTES.md', encoding='utf-8') as f: - release_notes = f.read() - -setup( - name='asmc-preparedecoding', - version='2.2.3', - author='PalamaraLab (https://palamaralab.github.io/)', - url='https://github.com/PalamaraLab/PrepareDecoding/', - install_requires=['numpy'], - description='Prepare decoding quantities for ASMC & FastSMC', - packages=find_namespace_packages(include=['asmc.*']), - long_description='\n'.join([long_description, release_notes]), - long_description_content_type='text/markdown', - ext_modules=[CMakeExtension('asmc/preparedecoding')], - cmdclass=dict(build_ext=CMakeBuild), - zip_safe=False, -) From 665225021c68053a58d551d291c4b7e43668c8b6 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Thu, 25 Sep 2025 15:27:40 +0100 Subject: [PATCH 03/27] Get pybind either from build system or from fetch content --- src/CMakeLists.txt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28be6f7..595e158 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -117,8 +117,25 @@ if (BUILD_WITH_SMCPP_SUPPORT) endif () -if (PYTHON_BINDINGS) +if (PREPARE_DECODING_PYTHON_BINDINGS) + option(PREPARE_DECODING_BUILDING_FROM_PYPROJECT "Are we building from pyproject.toml (pip install)?" OFF) + + if(PREPARE_DECODING_BUILDING_FROM_PYPROJECT) + message(STATUS "Using pybind11 from pyproject.toml build environment") + find_package(pybind11 REQUIRED) + else() + message(STATUS "Using FetchContent to get pybind11") + include(FetchContent) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11.git + GIT_TAG a2e59f0e7065404b44dfe92a28aca47ba1378dc4 # Version 2.13.6 + ) + FetchContent_MakeAvailable(pybind11) + endif() + set_target_properties(prepare_decoding_lib PROPERTIES POSITION_INDEPENDENT_CODE TRUE) pybind11_add_module(preparedecoding_python_bindings pybind.cpp) target_link_libraries(preparedecoding_python_bindings PRIVATE Eigen3::Eigen fmt::fmt prepare_decoding_lib project_settings) + install(TARGETS preparedecoding_python_bindings LIBRARY DESTINATION asmc/preparedecoding) endif () From db1ab9724d9090f24e00f467b3fdaaa699a78fed Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Thu, 25 Sep 2025 15:31:55 +0100 Subject: [PATCH 04/27] Add standard python .gitignores --- .gitignore | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/.gitignore b/.gitignore index a57e72c..9bbb52a 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,144 @@ __pycache__/ # Directory containing output generated for ASMC utils/dq_for_asmc + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +####################### +# Custom ASMC-related # +####################### + +.vscode +.idea +cmake-build-* +build + +**/.uuid From 893ec52e9893e5b0efae768758eef84c718168ce Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Thu, 25 Sep 2025 16:43:38 +0100 Subject: [PATCH 05/27] Commit WIP --- .github/workflows/python-wheels.yml | 53 ++++++++++++++++++++-------- src/asmc/preparedecoding/__init__.py | 12 +++---- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index b871447..efa5129 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -11,31 +11,54 @@ on: types: - published -env: - CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* - CIBW_SKIP: cp3*-musllinux_* - CIBW_ARCHS_MACOS: x86_64 - CIBW_ARCHS_LINUX: auto64 - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_BEFORE_ALL_LINUX: "yum -y groupinstall 'Development Tools' && yum -y install git" - CIBW_BEFORE_ALL_MACOS: brew install libomp pkg-config automake autoconf autoconf-archive libtool - CIBW_BEFORE_BUILD: pip install --upgrade pip setuptools wheel ninja numpy cython - CIBW_ENVIRONMENT_MACOS: CXX="$(brew --prefix llvm@15)/bin/clang++" - MACOSX_DEPLOYMENT_TARGET: 10.15 - jobs: - build_wheels: + build_wheels_cloud: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-12] + include: + - os: ubuntu-24.04 + arch: x86_64 + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + before-all: "" + extra-env: "" + mdt: "" + - os: ubuntu-24.04-arm + arch: aarch64 + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + before-all: dnf -y install boost-devel zlib + extra-env: "" + mdt: "" + - os: macos-15-intel + arch: x86_64 + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool + extra-env: CC=clang CXX=clang++ + mdt: "" + - os: macos-15 + arch: arm64 + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool + extra-env: CC=clang CXX=clang++ + mdt: "" + + env: + CIBW_BUILD: ${{ matrix.py-vers }} + CIBW_SKIP: cp3*-musllinux_* + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL: ${{ matrix.before-all }} + CIBW_BEFORE_BUILD: pip install --upgrade pip setuptools wheel ninja numpy cython + CIBW_ENVIRONMENT: ${{ matrix.extra-env }} + # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.mdt }} + steps: - name: checkout repo & submodules - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 diff --git a/src/asmc/preparedecoding/__init__.py b/src/asmc/preparedecoding/__init__.py index f03065c..1f022de 100644 --- a/src/asmc/preparedecoding/__init__.py +++ b/src/asmc/preparedecoding/__init__.py @@ -1,9 +1,9 @@ -from asmc.preparedecoding_python_bindings import DecodingQuantities -from asmc.preparedecoding_python_bindings import prepareDecoding -from asmc.preparedecoding_python_bindings import Demography -from asmc.preparedecoding_python_bindings import Discretization -from asmc.preparedecoding_python_bindings import Frequencies -from asmc.preparedecoding_python_bindings import save_demography +from .preparedecoding_python_bindings import DecodingQuantities +from .preparedecoding_python_bindings import prepareDecoding +from .preparedecoding_python_bindings import Demography +from .preparedecoding_python_bindings import Discretization +from .preparedecoding_python_bindings import Frequencies +from .preparedecoding_python_bindings import save_demography from typing import Iterable, Union import numbers From f4dd2a918c7eefc5b354a32eddb7740ad978ba79 Mon Sep 17 00:00:00 2001 From: Fergus Cooper <3770306+fcooper8472@users.noreply.github.com> Date: Thu, 25 Sep 2025 23:15:09 +0100 Subject: [PATCH 06/27] Update workflows, remove caching and windows --- .github/workflows/coverage.yml | 25 ++++-------- .github/workflows/python-wheels.yml | 28 +++++--------- .github/workflows/regression.yml | 23 +++-------- .github/workflows/sanitisers.yml | 24 ++++-------- .github/workflows/static-analysis.yml | 18 ++------- .github/workflows/unit-macos.yml | 20 +++------- .github/workflows/unit-ubuntu.yml | 22 +++-------- .github/workflows/unit-windows.yml | 56 --------------------------- asmc/preparedecoding | 1 - 9 files changed, 45 insertions(+), 172 deletions(-) delete mode 100644 .github/workflows/unit-windows.yml delete mode 120000 asmc/preparedecoding diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1e4f560..b81e0ef 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,41 +12,30 @@ jobs: build-and-test: name: Unit test coverage - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - CXX: g++-10 + CXX: g++-13 steps: - - name: checkout repo & vcpkg submodule - uses: actions/checkout@v3 + - name: checkout repo & submodules + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ env.CXX }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - name: install tools run: | - sudo apt -y update - sudo apt -y install g++-10 lcov libcurl4-openssl-dev + sudo apt -y install lcov libcurl4-openssl-dev - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: run unit tests with coverage run: | cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON - cmake --build . --parallel 2 --target unit_tests - ctest -j2 --output-on-failure + cmake --build . --parallel 4 --target unit_tests + ctest -j4 --output-on-failure working-directory: build_dir - name: upload coverage results diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index efa5129..0bf5218 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -54,7 +54,6 @@ jobs: CIBW_ENVIRONMENT: ${{ matrix.extra-env }} # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.mdt }} - steps: - name: checkout repo & submodules @@ -63,37 +62,30 @@ jobs: submodules: true fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v3 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.12.0 - - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse + uses: pypa/cibuildwheel@v3.2.0 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-cloud-${{ matrix.os }} path: ./wheelhouse/*.whl retention-days: 1 upload_all: name: Upload to PyPI - needs: build_wheels + needs: [build_wheels_cloud] runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'published' + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} steps: - - uses: actions/setup-python@v3 - - - name: Download wheels - uses: actions/download-artifact@v3 + - name: Download wheels from cloud runners + uses: actions/download-artifact@v4 with: - name: wheels + pattern: wheels-cloud-* + merge-multiple: true path: wheels - - uses: pypa/gh-action-pypi-publish@v1.6.4 + - uses: pypa/gh-action-pypi-publish@v1.13.0 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5d5d792..67da0ee 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -17,30 +17,20 @@ jobs: CXX: g++-10 steps: - - uses: actions/checkout@v3 + - name: checkout repo & submodules + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache + - name: Set up Python + uses: actions/setup-python@v6 with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ env.CXX }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - architecture: x64 + python-version: 3.13 - name: install python dependencies run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ninja numpy cython - name: install python bindings run: | @@ -48,12 +38,11 @@ jobs: - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: compile prepare decoding run: | cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release - cmake --build . --parallel 2 --target prepare_decoding_exe + cmake --build . --parallel 4 --target prepare_decoding_exe working-directory: build_dir - name: generate & check decoding quantities (executable) diff --git a/.github/workflows/sanitisers.yml b/.github/workflows/sanitisers.yml index a826320..f362d5b 100644 --- a/.github/workflows/sanitisers.yml +++ b/.github/workflows/sanitisers.yml @@ -12,36 +12,26 @@ jobs: build-and-test: name: Sanitiser checks - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - CXX: clang++-10 + CXX: clang++-18 steps: - name: checkout repo & submodules - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ env.CXX }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: address sanitiser run: | mv vcpkg_installed .. && rm -rf * && mv ../vcpkg_installed . cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITISER_ADDRESS=ON - cmake --build . --parallel 2 --target unit_tests + cmake --build . --parallel 4 --target unit_tests ./test/unit_tests ~"Test prepare decoding" working-directory: build_dir @@ -49,7 +39,7 @@ jobs: run: | mv vcpkg_installed .. && rm -rf * && mv ../vcpkg_installed . cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITISER_LEAK=ON - cmake --build . --parallel 2 --target unit_tests + cmake --build . --parallel 4 --target unit_tests ./test/unit_tests working-directory: build_dir @@ -57,7 +47,7 @@ jobs: run: | mv vcpkg_installed .. && rm -rf * && mv ../vcpkg_installed . cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITISER_UNDEFINED_BEHAVIOUR=ON - cmake --build . --parallel 2 --target unit_tests + cmake --build . --parallel 4 --target unit_tests ./test/unit_tests working-directory: build_dir @@ -65,6 +55,6 @@ jobs: run: | mv vcpkg_installed .. && rm -rf * && mv ../vcpkg_installed . cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITISER_THREAD=ON - cmake --build . --parallel 2 --target unit_tests + cmake --build . --parallel 4 --target unit_tests ./test/unit_tests ~"Test prepare decoding" working-directory: build_dir diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 46c2051..9a7b69b 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -12,35 +12,25 @@ jobs: build-and-test: name: Static analysis checks - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - CXX: clang++-10 + CXX: clang++-18 steps: - name: checkout repo & submodules - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ env.CXX }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: install analysers run: | sudo apt -y update - sudo apt -y install cppcheck clang-tidy-10 + sudo apt -y install cppcheck - name: compile with cppcheck run: | diff --git a/.github/workflows/unit-macos.yml b/.github/workflows/unit-macos.yml index acd0535..c70e900 100644 --- a/.github/workflows/unit-macos.yml +++ b/.github/workflows/unit-macos.yml @@ -12,40 +12,30 @@ jobs: build-and-test: name: Unit tests on macOS - runs-on: macos-12 + runs-on: macos-15 steps: - name: checkout repo & submodules - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - name: install system packages run: brew install libomp pkg-config automake autoconf autoconf-archive libtool - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: cmake configure - run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@15)/bin/clang++ + run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=clang++ working-directory: build_dir - name: cmake build - run: cmake --build . --parallel 2 --target unit_tests + run: cmake --build . --parallel 4 --target unit_tests working-directory: build_dir - name: cmake test - run: ctest -j2 --output-on-failure + run: ctest -j4 --output-on-failure working-directory: build_dir diff --git a/.github/workflows/unit-ubuntu.yml b/.github/workflows/unit-ubuntu.yml index 3704f4c..6239a8e 100644 --- a/.github/workflows/unit-ubuntu.yml +++ b/.github/workflows/unit-ubuntu.yml @@ -12,39 +12,29 @@ jobs: build-and-test: name: Unit tests on Ubuntu - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - CXX: g++-9 + CXX: g++-13 steps: - - name: checkout repo & vcpkg submodule - uses: actions/checkout@v3 + - name: checkout repo & submodules + uses: actions/checkout@v5 with: submodules: true fetch-depth: 0 - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ env.CXX }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - name: make build directory run: mkdir build_dir - if: steps.cache.outputs.cache-hit != 'true' - name: cmake configure run: cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug working-directory: build_dir - name: cmake build - run: cmake --build . --parallel 2 --target unit_tests + run: cmake --build . --parallel 4 --target unit_tests working-directory: build_dir - name: cmake test - run: ctest -j2 --output-on-failure + run: ctest -j4 --output-on-failure working-directory: build_dir diff --git a/.github/workflows/unit-windows.yml b/.github/workflows/unit-windows.yml deleted file mode 100644 index c4cc483..0000000 --- a/.github/workflows/unit-windows.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: "Unit tests: Windows" - -on: - push: - branches: - - none - pull_request: - branches: - - none - -jobs: - - build-and-test: - name: Unit tests on Windows - runs-on: windows-latest - - steps: - - - name: checkout repo & vcpkg submodule - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: cache vcpkg installed packages - uses: actions/cache@v2 - id: cache - with: - path: | - vcpkg/ - build_dir/vcpkg_installed/ - key: ${{ runner.os }}-${{ hashFiles('vcpkg.json', 'vcpkg/CHANGELOG.md') }} - - - name: make build directory - run: | - mkdir build_dir - mkdir build_dir\vcpkg_installed - if: steps.cache.outputs.cache-hit != 'true' - - - name: install system packages - run: | - .\bootstrap-vcpkg.bat - .\vcpkg --x-install-root=${{ github.workspace }}\build_dir\vcpkg_installed install yasm-tool:x86-windows - working-directory: vcpkg - - - name: cmake configure - run: cmake .. -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake -DCMAKE_BUILD_TYPE=Debug - working-directory: build_dir - - - name: cmake build - run: cmake --build . --parallel 2 --target unit_tests - working-directory: build_dir - - - name: cmake test - run: ctest -j2 --output-on-failure - working-directory: build_dir diff --git a/asmc/preparedecoding b/asmc/preparedecoding deleted file mode 120000 index 5cd551c..0000000 --- a/asmc/preparedecoding +++ /dev/null @@ -1 +0,0 @@ -../src \ No newline at end of file From 11c33ba7524e028db73824da12bb173da6e7dc44 Mon Sep 17 00:00:00 2001 From: Fergus Cooper <3770306+fcooper8472@users.noreply.github.com> Date: Fri, 26 Sep 2025 00:36:20 +0100 Subject: [PATCH 07/27] Update vcpkg --- vcpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcpkg b/vcpkg index f14984a..e6e4bc7 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit f14984af3738e69f197bf0e647a8dca12de92996 +Subproject commit e6e4bc74aaf5c63dfc358810594f662f7e9bc4d4 From dd7f294d6cffe557a9c2087bba5d938259448648 Mon Sep 17 00:00:00 2001 From: Fergus Cooper <3770306+fcooper8472@users.noreply.github.com> Date: Fri, 26 Sep 2025 00:58:02 +0100 Subject: [PATCH 08/27] Upgrade deps --- vcpkg.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vcpkg.json b/vcpkg.json index 6e2b720..55a3908 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -15,9 +15,9 @@ "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", "overrides": [ { "name": "cereal", "version": "1.3.2#1" }, - { "name": "eigen3", "version": "3.3.9" }, - { "name": "fmt", "version": "7.1.3#5" }, - { "name": "gmp", "version": "6.2.1#14" }, - { "name": "mpfr", "version": "4.1.1" } + { "name": "eigen3", "version": "3.3.9#1" }, + { "name": "fmt", "version": "12.0.0" }, + { "name": "gmp", "version": "6.3.0#2" }, + { "name": "mpfr", "version": "4.2.2" } ] } From 78251d6f45eca8e23cd6872073d5bd24f0cc40c0 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 12:03:45 +0100 Subject: [PATCH 09/27] Roll back cmake version --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index af74323..a17e444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [build-system] requires = [ "scikit-build-core>=0.11.6", + "cmake<4", "pybind11==2.13.6" ] build-backend = "scikit_build_core.build" From efb1b64680dc4c8abaeca527ec20fb11f90b4d72 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 12:03:55 +0100 Subject: [PATCH 10/27] Roll back fmt version --- vcpkg.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcpkg.json b/vcpkg.json index 55a3908..3491e79 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -16,7 +16,7 @@ "overrides": [ { "name": "cereal", "version": "1.3.2#1" }, { "name": "eigen3", "version": "3.3.9#1" }, - { "name": "fmt", "version": "12.0.0" }, + { "name": "fmt", "version": "7.1.3#5" }, { "name": "gmp", "version": "6.3.0#2" }, { "name": "mpfr", "version": "4.2.2" } ] From 448f030c739bd78676038157a0491c1785a47f38 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 12:04:21 +0100 Subject: [PATCH 11/27] Only build vcpkg release binaries for wheels --- .github/workflows/python-wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 0bf5218..47c5f8a 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -23,25 +23,25 @@ jobs: arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: "" - extra-env: "" + extra-env: VCPKG_BUILD_TYPE=release mdt: "" - os: ubuntu-24.04-arm arch: aarch64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: dnf -y install boost-devel zlib - extra-env: "" + extra-env: VCPKG_BUILD_TYPE=release mdt: "" - os: macos-15-intel arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool - extra-env: CC=clang CXX=clang++ + extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release mdt: "" - os: macos-15 arch: arm64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool - extra-env: CC=clang CXX=clang++ + extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release mdt: "" env: From e3baeda0ed16643c54c7ed9d3227e411d28761f4 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 13:35:32 +0100 Subject: [PATCH 12/27] Upgrade runner to newer ubuntu --- .github/workflows/regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 67da0ee..b1b7b4d 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -12,9 +12,9 @@ jobs: build-and-test: name: Test decoding quantities can be generated - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - CXX: g++-10 + CXX: g++-13 steps: - name: checkout repo & submodules From d85db01ca0af8c3c8a07206493fb89e82ba12842 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 13:35:42 +0100 Subject: [PATCH 13/27] Ignore errors with coverage command --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b81e0ef..50d5bb5 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -41,7 +41,7 @@ jobs: - name: upload coverage results run: | lcov --directory . --capture --output-file coverage.info - lcov --remove coverage.info '/usr/*' '*/vcpkg/*' '*/test/*' --output-file coverage.info + lcov --remove coverage.info '/usr/*' '*/vcpkg/*' '*/test/*' --output-file coverage.info --ignore-errors unused lcov --list coverage.info bash <(curl https://codecov.io/bash) -f coverage.info env: From 28d4335e1ff2bc9efb18351a900327341f207431 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 13:39:08 +0100 Subject: [PATCH 14/27] Bump zlib --- vcpkg.json | 1 + 1 file changed, 1 insertion(+) diff --git a/vcpkg.json b/vcpkg.json index 3491e79..c8f94e7 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -14,6 +14,7 @@ ], "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", "overrides": [ + { "name": "zlib", "version": "1.3.1#0" }, { "name": "cereal", "version": "1.3.2#1" }, { "name": "eigen3", "version": "3.3.9#1" }, { "name": "fmt", "version": "7.1.3#5" }, From b4f239d0422113f630315e919b031c4dec8752c6 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 13:43:11 +0100 Subject: [PATCH 15/27] Add openmp, needed for clang compilation --- .github/workflows/sanitisers.yml | 4 ++++ .github/workflows/static-analysis.yml | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitisers.yml b/.github/workflows/sanitisers.yml index f362d5b..462d81c 100644 --- a/.github/workflows/sanitisers.yml +++ b/.github/workflows/sanitisers.yml @@ -24,6 +24,10 @@ jobs: submodules: true fetch-depth: 0 + - name: install tools + run: | + sudo apt -y install libomp-dev + - name: make build directory run: mkdir build_dir diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 9a7b69b..0625292 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -27,9 +27,12 @@ jobs: - name: make build directory run: mkdir build_dir + - name: install tools + run: | + sudo apt -y install libomp-dev + - name: install analysers run: | - sudo apt -y update sudo apt -y install cppcheck - name: compile with cppcheck From cc2074d213b7d05fbcf29290b728cf26a928792a Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 13:52:04 +0100 Subject: [PATCH 16/27] Beef up system packages needed for bootstrapping vcpkg --- .github/workflows/python-wheels.yml | 8 ++++---- .github/workflows/unit-macos.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 47c5f8a..191ebe7 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -22,25 +22,25 @@ jobs: - os: ubuntu-24.04 arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: "" + before-all: dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel extra-env: VCPKG_BUILD_TYPE=release mdt: "" - os: ubuntu-24.04-arm arch: aarch64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: dnf -y install boost-devel zlib + before-all: dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel extra-env: VCPKG_BUILD_TYPE=release mdt: "" - os: macos-15-intel arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool + before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release mdt: "" - os: macos-15 arch: arm64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool + before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release mdt: "" diff --git a/.github/workflows/unit-macos.yml b/.github/workflows/unit-macos.yml index c70e900..4ffe9fc 100644 --- a/.github/workflows/unit-macos.yml +++ b/.github/workflows/unit-macos.yml @@ -23,7 +23,7 @@ jobs: fetch-depth: 0 - name: install system packages - run: brew install libomp pkg-config automake autoconf autoconf-archive libtool + run: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - name: make build directory run: mkdir build_dir From 12b2e79e783e7dbda1ac28530ecf070b7c43755a Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 14:14:37 +0100 Subject: [PATCH 17/27] Treat clang tidy as warnings for time being --- .clang-tidy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 30a3f4e..06b5b1e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- Checks: '*,-android-*,-bugprone-bool-pointer-implicit-conversion,-bugprone-exception-escape,-bugprone-infinite-loop,-bugprone-signed-char-misuse,-cert-dcl16-c,-cert-dcl37-c,-cert-dcl50-cpp,-cert-dcl51-cpp,-cert-dcl54-cpp,-cert-dcl59-cpp,-cert-env33-c,-cert-err09-cpp,-cert-err61-cpp,-cert-fio38-c,-cert-flp30-c,-cert-mem57-cpp,-cert-msc30-c,-cert-msc32-c,-cert-oop11-cpp,-cert-oop57-cpp,-cert-oop58-cpp,-cert-pos44-c,-clang-analyzer-*,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-type-vararg,-darwin-*,-fuchsia-*,-google-*,google-default-arguments,google-explicit-constructor,google-runtime-operator,-hicpp-*,hicpp-exception-baseclass,hicpp-multiway-paths-covered,hicpp-signed-bitwise,-linuxkernel-*,-llvm-*,-llvmlibc-*,-misc-definitions-in-headers,-misc-non-private-member-variables-in-classes,-misc-unused-alias-decls,-misc-unused-parameters,-misc-unused-using-decls,-modernize-use-trailing-return-type,-objc-*,-openmp-exception-escape,-readability-braces-around-statements,-readability-else-after-return,-readability-function-size,-readability-identifier-naming,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-named-parameter,-readability-qualified-auto,-readability-redundant-access-specifiers,-readability-redundant-member-init,-readability-redundant-preprocessor,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-zircon-*' -WarningsAsErrors: '*' +WarningsAsErrors: '' HeaderFilterRegex: '' FormatStyle: none From 8e666b347fa2205f2ab07b3cef974b9b2f7ea62e Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 14:14:46 +0100 Subject: [PATCH 18/27] Specify clang compiler --- .github/workflows/unit-macos.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit-macos.yml b/.github/workflows/unit-macos.yml index 4ffe9fc..6dd7a7a 100644 --- a/.github/workflows/unit-macos.yml +++ b/.github/workflows/unit-macos.yml @@ -13,6 +13,9 @@ jobs: build-and-test: name: Unit tests on macOS runs-on: macos-15 + env: + CC: clang + CXX: clang++ steps: From b5418a1adc4501d1dbf4a3003f0581a28c1764df Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 15:02:58 +0100 Subject: [PATCH 19/27] Explicitly specify omp location and install dev tools --- .github/workflows/python-wheels.yml | 14 +++++++------- .github/workflows/unit-macos.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 191ebe7..fedd6b7 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -22,26 +22,26 @@ jobs: - os: ubuntu-24.04 arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel - extra-env: VCPKG_BUILD_TYPE=release + before-all: yum groupinstall -y "Development Tools" && dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel + extra-env: "" mdt: "" - os: ubuntu-24.04-arm arch: aarch64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* - before-all: dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel - extra-env: VCPKG_BUILD_TYPE=release + before-all: yum groupinstall -y "Development Tools" && dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel + extra-env: "" mdt: "" - os: macos-15-intel arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release + extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" mdt: "" - os: macos-15 arch: arm64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - extra-env: CC=clang CXX=clang++ VCPKG_BUILD_TYPE=release + extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" mdt: "" env: @@ -51,7 +51,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL: ${{ matrix.before-all }} CIBW_BEFORE_BUILD: pip install --upgrade pip setuptools wheel ninja numpy cython - CIBW_ENVIRONMENT: ${{ matrix.extra-env }} + CIBW_ENVIRONMENT: VCPKG_BUILD_TYPE=release ${{ matrix.extra-env }} # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.mdt }} steps: diff --git a/.github/workflows/unit-macos.yml b/.github/workflows/unit-macos.yml index 6dd7a7a..d4be979 100644 --- a/.github/workflows/unit-macos.yml +++ b/.github/workflows/unit-macos.yml @@ -32,7 +32,7 @@ jobs: run: mkdir build_dir - name: cmake configure - run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=clang++ + run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_PREFIX_PATH=$(brew --prefix libomp) working-directory: build_dir - name: cmake build From 8d7cdab76c002270f225617b57f2fbc63a7a88e6 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 15:49:42 +0100 Subject: [PATCH 20/27] Add mac deployment target 11 (Big Sur) and build py 3.14 wheels --- .github/workflows/python-wheels.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index fedd6b7..3920cf8 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -21,28 +21,28 @@ jobs: include: - os: ubuntu-24.04 arch: x86_64 - py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* before-all: yum groupinstall -y "Development Tools" && dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel extra-env: "" mdt: "" - os: ubuntu-24.04-arm arch: aarch64 - py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* before-all: yum groupinstall -y "Development Tools" && dnf -y install ninja-build pkgconf openssl-devel zlib-devel bzip2-devel xz-devel extra-env: "" mdt: "" - os: macos-15-intel arch: x86_64 - py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" - mdt: "" + mdt: 11 - os: macos-15 arch: arm64 - py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* + py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" - mdt: "" + mdt: 11 env: CIBW_BUILD: ${{ matrix.py-vers }} @@ -52,7 +52,7 @@ jobs: CIBW_BEFORE_ALL: ${{ matrix.before-all }} CIBW_BEFORE_BUILD: pip install --upgrade pip setuptools wheel ninja numpy cython CIBW_ENVIRONMENT: VCPKG_BUILD_TYPE=release ${{ matrix.extra-env }} - # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.mdt }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.mdt }} steps: From 8afec91bca8acee96c78fce75e2161e7f6c7a7bd Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 15:57:13 +0100 Subject: [PATCH 21/27] Bump pybind version --- CMakeLists.txt | 8 -------- pyproject.toml | 3 +-- src/CMakeLists.txt | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51a2c6d..ae0f0e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,14 +33,6 @@ include(cmake/StaticAnalysers.cmake) option(BUILD_SHARED_LIBS "Enable compilation of shared libraries" OFF) option(ENABLE_TESTING "Enable Test Builds" ON) -option(PYTHON_BINDINGS "Whether to build the python bindings" OFF) -if (EXISTS ${CMAKE_SOURCE_DIR}/pybind11/LICENSE) - set(PYTHON_BINDINGS ON) -endif() -if (PYTHON_BINDINGS) - add_subdirectory(pybind11) -endif () - add_subdirectory(src) add_subdirectory(exe) diff --git a/pyproject.toml b/pyproject.toml index a17e444..c57ec97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,7 @@ [build-system] requires = [ "scikit-build-core>=0.11.6", - "cmake<4", - "pybind11==2.13.6" + "pybind11==3.0.1" ] build-backend = "scikit_build_core.build" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 595e158..923d41c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -129,7 +129,7 @@ if (PREPARE_DECODING_PYTHON_BINDINGS) FetchContent_Declare( pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11.git - GIT_TAG a2e59f0e7065404b44dfe92a28aca47ba1378dc4 # Version 2.13.6 + GIT_TAG f5fbe867d2d26e4a0a9177a51f6e568868ad3dc8 # Version 3.0.1 ) FetchContent_MakeAvailable(pybind11) endif() From e757912b391b33de5b43bfdf68ee8629586d836d Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 16:29:00 +0100 Subject: [PATCH 22/27] Remove openmp completely --- .github/workflows/python-wheels.yml | 8 ++++---- .github/workflows/sanitisers.yml | 4 ---- .github/workflows/static-analysis.yml | 4 ---- .github/workflows/unit-macos.yml | 4 ++-- PyPI_README.md | 10 ++++++++++ cmake/AsmcPrepareDecodingConfig.cmake | 1 - src/CMakeLists.txt | 4 +--- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 3920cf8..a9bea68 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -34,14 +34,14 @@ jobs: - os: macos-15-intel arch: x86_64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* - before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" + before-all: brew install pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz + extra-env: CC=clang CXX=clang++ mdt: 11 - os: macos-15 arch: arm64 py-vers: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* - before-all: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - extra-env: CC=clang CXX=clang++ CMAKE_PREFIX_PATH="$(brew --prefix libomp)" + before-all: brew install pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz + extra-env: CC=clang CXX=clang++ mdt: 11 env: diff --git a/.github/workflows/sanitisers.yml b/.github/workflows/sanitisers.yml index 462d81c..f362d5b 100644 --- a/.github/workflows/sanitisers.yml +++ b/.github/workflows/sanitisers.yml @@ -24,10 +24,6 @@ jobs: submodules: true fetch-depth: 0 - - name: install tools - run: | - sudo apt -y install libomp-dev - - name: make build directory run: mkdir build_dir diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 0625292..52a64e6 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -27,10 +27,6 @@ jobs: - name: make build directory run: mkdir build_dir - - name: install tools - run: | - sudo apt -y install libomp-dev - - name: install analysers run: | sudo apt -y install cppcheck diff --git a/.github/workflows/unit-macos.yml b/.github/workflows/unit-macos.yml index d4be979..0820681 100644 --- a/.github/workflows/unit-macos.yml +++ b/.github/workflows/unit-macos.yml @@ -26,13 +26,13 @@ jobs: fetch-depth: 0 - name: install system packages - run: brew install libomp pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz + run: brew install pkg-config automake autoconf autoconf-archive libtool ninja openssl zlib bzip2 xz - name: make build directory run: mkdir build_dir - name: cmake configure - run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_PREFIX_PATH=$(brew --prefix libomp) + run: cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CXX_COMPILER=clang++ working-directory: build_dir - name: cmake build diff --git a/PyPI_README.md b/PyPI_README.md index 5413584..84c5964 100644 --- a/PyPI_README.md +++ b/PyPI_README.md @@ -12,6 +12,16 @@ Tool to compute decoding quantities. +This is a Python-wrapped C++ project with prebuilt CPython wheels targeting manylinux_2_28 on Linux and macOS 11.0+. + +| Platform \ CPython | ≤3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14 | +|-----------------------------| ---- | --- | ---- | ---- | ---- | ---- | ---- | +| Linux x86_64 | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Linux aarch64 | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| macOS Intel (x86_64) | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| macOS Apple Silicon (arm64) | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | + + ## Quickstart ### Install the Python module from PyPI diff --git a/cmake/AsmcPrepareDecodingConfig.cmake b/cmake/AsmcPrepareDecodingConfig.cmake index c936b18..d3ce992 100644 --- a/cmake/AsmcPrepareDecodingConfig.cmake +++ b/cmake/AsmcPrepareDecodingConfig.cmake @@ -4,7 +4,6 @@ find_dependency(cereal) find_dependency(Eigen3) find_dependency(fmt) find_dependency(GMP) -find_dependency(OpenMP) find_dependency(ZLIB) include(${CMAKE_CURRENT_LIST_DIR}/AsmcPrepareDecoding_Runtime.cmake) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 923d41c..e99d5e5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -99,8 +99,6 @@ if (BUILD_WITH_SMCPP_SUPPORT) find_package(cereal CONFIG REQUIRED) message(STATUS "Found cereal ${cereal_VERSION}") - find_package(OpenMP REQUIRED) - add_library(smcpp_lib 3rd_party/smcpp.cpp 3rd_party/smcpp.hpp 3rd_party/mpreal.h) target_include_directories( @@ -109,7 +107,7 @@ if (BUILD_WITH_SMCPP_SUPPORT) $ ) - target_link_libraries(smcpp_lib PRIVATE Eigen3::Eigen fmt::fmt PkgConfig::gmpxx PkgConfig::mpfr cereal::cereal OpenMP::OpenMP_CXX) + target_link_libraries(smcpp_lib PRIVATE Eigen3::Eigen fmt::fmt PkgConfig::gmpxx PkgConfig::mpfr cereal::cereal) set_target_properties(smcpp_lib PROPERTIES POSITION_INDEPENDENT_CODE TRUE) set_target_properties(smcpp_lib PROPERTIES CXX_CLANG_TIDY "") set_target_properties(smcpp_lib PROPERTIES CXX_CPPCHECK "") From 828148ef4362127cde1edbd5559885e4e31836c3 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 16:42:00 +0100 Subject: [PATCH 23/27] Begin preparing 2.2.4 --- CMakeLists.txt | 2 +- PyPI_README.md | 9 +++------ README.md | 13 ++++++------- RELEASE_NOTES.md | 6 ++++++ pyproject.toml | 2 +- vcpkg.json | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae0f0e1..bb1df97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ include(cmake/AutodetectVcpkgToolchainFile.cmake) set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) -project(PrepareDecoding LANGUAGES CXX VERSION 2.2.3) +project(PrepareDecoding LANGUAGES CXX VERSION 2.2.4) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/PyPI_README.md b/PyPI_README.md index 84c5964..e0dc86d 100644 --- a/PyPI_README.md +++ b/PyPI_README.md @@ -24,20 +24,17 @@ This is a Python-wrapped C++ project with prebuilt CPython wheels targeting many ## Quickstart -### Install the Python module from PyPI +### Install the Python package from PyPI -Most functionality is available through a Python module which can be installed with: +Most functionality is available through a Python package which can be installed with: ```bash pip install asmc-preparedecoding ``` -This Python module is currently available on Linux and macOS. -We hope it will be available soon on Windows. - ### Example notebook -Examples for using the Python module can be found in the following Jupyter notebook: +Examples for using the Python package can be found in the following Jupyter notebook: - [creating decoding quantities](https://github.com/PalamaraLab/PrepareDecoding/blob/4a206d577a8cd431ab6dd59bbccc4035ab8b1069/notebooks/CreatingDecodingQuantities.ipynb) Please note that to run the notebook you should first clone the repository and install Jupyter: diff --git a/README.md b/README.md index 944d6df..88738a5 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,17 @@ Tool to compute decoding quantities. ## Quickstart -### Install the Python module from PyPI +### Install the Python package from PyPI -Most functionality is available through a Python module which can be installed with: +Most functionality is available through a Python package which can be installed with: ```bash pip install asmc-preparedecoding ``` -This Python module is currently available on Linux and macOS. -We hope it will be available soon on Windows. +This Python package is currently available on Linux and macOS, x86_64 and arm64. -Examples for using the Python module can be found in the following Jupyter notebook: +Examples for using the Python package can be found in the following Jupyter notebook: - [creating decoding quantities](notebooks/CreatingDecodingQuantities.ipynb) Please note that you must install Jupyter in order to view the notebook, and then open it: @@ -36,7 +35,7 @@ jupyter-notebook notebooks/CreatingDecodingQuantities.ipynb ### Compiling the C++ library and executable -Get the source, together with the [vcpkg](https://github.com/microsoft/vcpkg) and [pybind11](https://github.com/pybind/pybind11) submodules: +Get the source, together with the [vcpkg](https://github.com/microsoft/vcpkg) submodule: ```bash git clone --recurse-submodules https://github.com/PalamaraLab/PrepareDecoding.git @@ -144,7 +143,7 @@ where `{{{SANITISER}}}` is one of: ## For developers: making a release -- Bump the version number in [setup.py](setup.py), [CMakeLists.txt](CMakeLists.txt), and [vcpkg.json](vcpkg.json) +- Bump the version number in [pyproject.toml](pyproject.toml), [CMakeLists.txt](CMakeLists.txt), and [vcpkg.json](vcpkg.json) - Update [RELEASE_NOTES.md](RELEASE_NOTES.md) - Update permalink to the notebook in [PyPI_README.md](PyPI_README.md) - Push changes and check that all [GitHub workflows](https://github.com/PalamaraLab/PrepareDecoding/actions) pass diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9dfecc8..cdb4bb5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,11 @@ # Release Notes +## v2.2.4 (2023-09-26) + +- wheels now available for ARM64 (Linux and macOS) +- wheels now available for Python 3.9-3.14 (inclusive) +- no change in functionality + ## v2.2.3 (2023-02-22) Infrastructure updates and building wheels for newer Python versions. diff --git a/pyproject.toml b/pyproject.toml index c57ec97..6abfb60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build" [project] dynamic = ["readme"] name = "asmc-preparedecoding" -version = "2.2.3" +version = "2.2.4" description = "Prepare decoding quantities for ASMC & FastSMC" authors = [ { name = "PalamaraLab (https://palamaralab.github.io/)" } diff --git a/vcpkg.json b/vcpkg.json index c8f94e7..48e1427 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,6 @@ { "name": "asmc-prepare-decoding", - "version-string": "2.2.3", + "version-string": "2.2.4", "homepage": "https://github.com/PalamaraLab/PrepareDecoding/", "description": "Prepare decoding quantities for ASMC & FastSMC", "dependencies": [ From 534adde6a8aa97d35c760312f40fcd0145284783 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 16:44:09 +0100 Subject: [PATCH 24/27] Add 3.14 to pyproject --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6abfb60..ac7c0a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] [tool.scikit-build] From 6fc81360bc88a8474af8bd86ef84fb396e88ace8 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 16:52:53 +0100 Subject: [PATCH 25/27] Add output of notebook cells --- notebooks/CreatingDecodingQuantities.ipynb | 163 +++++++++++++++++++-- 1 file changed, 148 insertions(+), 15 deletions(-) diff --git a/notebooks/CreatingDecodingQuantities.ipynb b/notebooks/CreatingDecodingQuantities.ipynb index 7663d3f..4f38de3 100644 --- a/notebooks/CreatingDecodingQuantities.ipynb +++ b/notebooks/CreatingDecodingQuantities.ipynb @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "42f85552", "metadata": {}, "outputs": [], @@ -47,10 +47,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "56e8e52f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Demography set to file /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_CEU.demo\n", + "Discretization file set to file /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_30-100-2000.disc\n", + "Frequencies file set to /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq\n", + "New CSFS will be calculated\n", + "Will read discretization intervals from /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_30-100-2000.disc ...\n", + "smcpp: init_cache() successful\n", + "smcpp: storing cache: /tmp/smcpp_matrices.dat\n", + "smcpp: store_cache() successful\n", + "Will use minor allele frequencies from /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq ...\n", + "Will use mutation rate mu = 1.65e-08.\n", + "Number of samples in CSFS calculations: 50.\n", + "Verified 69 CSFS entries.\n", + "Probability of a site being monomorphic due to subsampling: 0.222911939663710\n", + "\n", + "Building decoding quantities...\n", + "Genetic distances progress: 100%\t\n", + "Physical distances progress: 100%\t\n", + "Demography set to file /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_CEU.demo\n", + "Frequencies file set to /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq\n", + "New CSFS will be calculated\n", + "Using the following pre-specified discretization intervals: {0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, 390, 420, 450, 550, 650, 750, 850, 950, 1050, 1150, 1250, 1350, 1450, 1550, 1650, 1750, 1850, 1950}\n", + " and calculating 39 additional intervals from coalescent distribution.\n", + "smcpp: init_cache() successful\n", + "Will use minor allele frequencies from /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq ...\n", + "Will use mutation rate mu = 1.65e-08.\n", + "Number of samples in CSFS calculations: 50.\n", + "Verified 70 CSFS entries.\n", + "Probability of a site being monomorphic due to subsampling: 0.222911939663710\n", + "\n", + "Building decoding quantities...\n", + "Genetic distances progress: 100%\t\n", + "Physical distances progress: 100%\t\n", + "Demography set to built-in CEU\n", + "Frequencies file set to /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq\n", + "New CSFS will be calculated\n", + "Using the following pre-specified discretization intervals: {0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, 390, 420, 450, 550, 650, 750, 850, 950, 1050, 1150, 1250, 1350, 1450, 1550, 1650, 1750, 1850, 1950}\n", + " and calculating 39 additional intervals from coalescent distribution.\n", + "smcpp: init_cache() successful\n", + "Will use minor allele frequencies from /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/input_UKBB.frq ...\n", + "Will use mutation rate mu = 1.65e-08.\n", + "Number of samples in CSFS calculations: 50.\n", + "Verified 70 CSFS entries.\n", + "Probability of a site being monomorphic due to subsampling: 0.222911939663710\n", + "\n", + "Building decoding quantities...\n", + "Genetic distances progress: 100%\t\n", + "Physical distances progress: 100%\t\n", + "Demography set to built-in CEU\n", + "Frequencies set to built-in UKBB with 50 samples\n", + "New CSFS will be calculated\n", + "Using the following pre-specified discretization intervals: {0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, 390, 420, 450, 550, 650, 750, 850, 950, 1050, 1150, 1250, 1350, 1450, 1550, 1650, 1750, 1850, 1950}\n", + " and calculating 39 additional intervals from coalescent distribution.\n", + "smcpp: init_cache() successful\n", + "Using built-in frequency information from UKBB ...\n", + "Will use mutation rate mu = 1.65e-08.\n", + "Number of samples in CSFS calculations: 50.\n", + "Verified 70 CSFS entries.\n", + "\n", + "Building decoding quantities...\n", + "Genetic distances progress: 100%\t\n", + "Physical distances progress: 100%\t" + ] + } + ], "source": [ "files_dir = (pathlib.Path('..') / 'test' / 'regression').resolve()\n", "\n", @@ -104,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "e1d704d3", "metadata": {}, "outputs": [], @@ -133,10 +201,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "84e495bb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Demography set to built-in CEU\n", + "Frequencies set to built-in UKBB with 50 samples\n", + "Precomputed CSFS will be loaded from file: /home/fcooper/GitRepos/PalamaraLab/PrepareDecoding/test/regression/output.csfs\n", + "Using the following pre-specified discretization intervals: {0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, 390, 420, 450, 550, 650, 750, 850, 950, 1050, 1150, 1250, 1350, 1450, 1550, 1650, 1750, 1850, 1950}\n", + " and calculating 39 additional intervals from coalescent distribution.\n", + "Using built-in frequency information from UKBB ...\n", + "Will use mutation rate mu = 1.65e-08.\n", + "Number of samples in CSFS calculations: 50.\n", + "Verified 70 CSFS entries.\n", + "\n", + "Building decoding quantities...\n", + "Genetic distances progress: 100%\t\n", + "Physical distances progress: 100%\t" + ] + } + ], "source": [ "dq = prepare_decoding(\n", " csfs_file=str(files_dir / 'output.csfs'),\n", @@ -157,10 +246,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "6ecc7fd7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'states': 70, 'samples': 50, 'mu': 1.65e-08}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "{\"states\": dq.states, \"samples\": dq.samples, \"mu\": dq.mu}" ] @@ -175,10 +275,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "fad47cb3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(numpy.ndarray, (2, 70))" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "X = dq.compressedEmission\n", "type(X), X.shape" @@ -194,20 +305,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "aacfc6cd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "70" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "len([x for x in dq.CSFS])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "2a1689b4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(3, 49)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c0 = dq.CSFS[0]\n", "{\"mu\": c0.mu, \"from\": c0.csfsFrom, \"to\": c0.csfsTo, \"samples\": c0.samples}\n", @@ -231,7 +364,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.11.11" } }, "nbformat": 4, From 0e08b309a294579fb5c22fdeb39e6f799736aa98 Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 16:54:34 +0100 Subject: [PATCH 26/27] Generic link to notebook with output --- PyPI_README.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PyPI_README.md b/PyPI_README.md index e0dc86d..0ad6a29 100644 --- a/PyPI_README.md +++ b/PyPI_README.md @@ -35,7 +35,7 @@ pip install asmc-preparedecoding ### Example notebook Examples for using the Python package can be found in the following Jupyter notebook: -- [creating decoding quantities](https://github.com/PalamaraLab/PrepareDecoding/blob/4a206d577a8cd431ab6dd59bbccc4035ab8b1069/notebooks/CreatingDecodingQuantities.ipynb) +- [creating decoding quantities](https://github.com/PalamaraLab/PrepareDecoding/blob/main/notebooks/CreatingDecodingQuantities.ipynb) Please note that to run the notebook you should first clone the repository and install Jupyter: diff --git a/README.md b/README.md index 88738a5..bd010d4 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ where `{{{SANITISER}}}` is one of: - Bump the version number in [pyproject.toml](pyproject.toml), [CMakeLists.txt](CMakeLists.txt), and [vcpkg.json](vcpkg.json) - Update [RELEASE_NOTES.md](RELEASE_NOTES.md) -- Update permalink to the notebook in [PyPI_README.md](PyPI_README.md) +- Ensure supported python versions are up-to-date in [pyproject.toml](pyproject.toml) - Push changes and check that all [GitHub workflows](https://github.com/PalamaraLab/PrepareDecoding/actions) pass - Tag the commit in Git using syntax `vX.Y.Z` - Make a release on GitHub, which should trigger a new build that will upload Python wheels to PyPI From 9b138cbb4da23803b5abcc70194dae04eb2d6d5b Mon Sep 17 00:00:00 2001 From: Fergus Cooper Date: Fri, 26 Sep 2025 17:09:13 +0100 Subject: [PATCH 27/27] Update README badges --- PyPI_README.md | 2 -- README.md | 1 - 2 files changed, 3 deletions(-) diff --git a/PyPI_README.md b/PyPI_README.md index 0ad6a29..1f6d761 100644 --- a/PyPI_README.md +++ b/PyPI_README.md @@ -1,4 +1,3 @@ -[![Unit tests: Windows](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20Windows/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Unit tests: Ubuntu](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20Ubuntu/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Unit tests: macOS](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20macOS/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Regression test](https://github.com/PalamaraLab/PrepareDecoding/workflows/Regression%20test/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) @@ -6,7 +5,6 @@ [![Static analysis checks](https://github.com/PalamaraLab/PrepareDecoding/workflows/Static%20analysis%20checks/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Sanitiser checks](https://github.com/PalamaraLab/PrepareDecoding/workflows/Sanitiser%20checks/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![codecov](https://codecov.io/gh/PalamaraLab/PrepareDecoding/branch/master/graph/badge.svg)](https://codecov.io/gh/PalamaraLab/PrepareDecoding) -[![BCH compliance](https://bettercodehub.com/edge/badge/PalamaraLab/PrepareDecoding?branch=master)](https://bettercodehub.com/results/PalamaraLab/PrepareDecoding) # ASMC Prepare Decoding diff --git a/README.md b/README.md index bd010d4..40dee7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![Unit tests: Windows](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20Windows/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Unit tests: Ubuntu](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20Ubuntu/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Unit tests: macOS](https://github.com/PalamaraLab/PrepareDecoding/workflows/Unit%20tests:%20macOS/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions) [![Regression test](https://github.com/PalamaraLab/PrepareDecoding/workflows/Regression%20test/badge.svg)](https://github.com/PalamaraLab/PrepareDecoding/actions)