diff --git a/.github/workflows/httomo_version_tag.yml b/.github/workflows/httomo_version_tag.yml deleted file mode 100644 index 79b9d130e..000000000 --- a/.github/workflows/httomo_version_tag.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: version tag pushed - -# Run the workflow whenever a tag beginning with `v` is pushed to any branch -on: - push: - tags: - - v* - -jobs: - conda-build-upload: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.ref_name }} - - # setup Python 3.11 - - name: Setup Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - # setup conda - - name: Setup conda - uses: mamba-org/setup-micromamba@v2 - with: - post-cleanup: 'all' - init-shell: bash - - # install dependencies - - name: Install dependencies - run: | - micromamba install -c conda-forge conda-build anaconda-client - micromamba list - - - name: Decrypt a secret - run: ./.scripts/decrypt_secret.sh - env: - LARGE_SECRET_PASSPHRASE: ${{ secrets.LARGE_SECRET_PASSPHRASE }} - - - name: Build and upload the package to httomo conda cloud - env: - LABEL: main - run: | - chmod +x ./.scripts/conda_upload.sh - ./.scripts/conda_upload.sh diff --git a/.github/workflows/run_tests_framework_iris.yml b/.github/workflows/run_tests_framework_iris.yml index 0501d6e65..44ab5cd21 100644 --- a/.github/workflows/run_tests_framework_iris.yml +++ b/.github/workflows/run_tests_framework_iris.yml @@ -27,18 +27,19 @@ jobs: - name: Create conda environment uses: mamba-org/setup-micromamba@v1 with: - environment-file: conda/environment.yml environment-name: httomo + create-args: >- + cupy==12.3.0 + h5py=*=*mpi_openmpi* + mpi4py + tomopy==1.15 post-cleanup: 'all' init-shell: bash - name: Install httomo libraries and httomo run: | micromamba activate httomo - pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar - pip install --no-deps httomo-backends - pip install . + pip install .[dev-gpu] micromamba list - name: Generate full yaml pipelines using pipeline directives diff --git a/.github/workflows/run_tests_pipelines_iris.yml b/.github/workflows/run_tests_pipelines_iris.yml index d72e0ca21..871ff2571 100644 --- a/.github/workflows/run_tests_pipelines_iris.yml +++ b/.github/workflows/run_tests_pipelines_iris.yml @@ -27,18 +27,19 @@ jobs: - name: Create conda environment uses: mamba-org/setup-micromamba@v1 with: - environment-file: conda/environment.yml environment-name: httomo + create-args: >- + cupy==12.3.0 + h5py=*=*mpi_openmpi* + mpi4py + tomopy==1.15 post-cleanup: 'all' init-shell: bash - name: Install httomo libraries, httomo-backends and httomo run: | micromamba activate httomo - pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar - pip install --no-deps httomo-backends - pip install . + pip install .[dev-gpu] micromamba list - name: Generate full yaml pipelines using pipeline directives diff --git a/.scripts/conda_upload.sh b/.scripts/conda_upload.sh deleted file mode 100755 index f467a5f35..000000000 --- a/.scripts/conda_upload.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -PKG_NAME=httomo -USER=httomo-team -OS=linux-64 -CONDA_TOKEN=$(cat $HOME/.secrets/my_secret.json) - -mkdir ~/conda-bld -conda config --set anaconda_upload no -export CONDA_BLD_PATH=~/conda-bld - -$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test - -# upload packages to conda -find $CONDA_BLD_PATH/$OS -name *.tar.bz2 | while read file -do - echo $file - $CONDA/bin/anaconda -v --show-traceback --token $CONDA_TOKEN upload --label $LABEL $file --force -done diff --git a/.scripts/decrypt_secret.sh b/.scripts/decrypt_secret.sh deleted file mode 100755 index db5192c1e..000000000 --- a/.scripts/decrypt_secret.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Decrypt the file -mkdir $HOME/.secrets -# --batch to prevent interactive command -# --yes to assume "yes" for questions -gpg --quiet --batch --yes --decrypt --passphrase="$LARGE_SECRET_PASSPHRASE" \ ---output $HOME/.secrets/my_secret.json ./.scripts/my_secret.json.gpg diff --git a/.scripts/my_secret.json.gpg b/.scripts/my_secret.json.gpg deleted file mode 100644 index 0101187df..000000000 --- a/.scripts/my_secret.json.gpg +++ /dev/null @@ -1 +0,0 @@ -  -:"k]凞$nʷH1\?۝Q5|hcmGߐLP($ɦ;L5e&XGl3XJ :y~{b \ No newline at end of file diff --git a/conda/environment-cpu.yml b/conda/environment-cpu.yml deleted file mode 100644 index e9a026e0d..000000000 --- a/conda/environment-cpu.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: httomo -channels: - - conda-forge - - httomo -dependencies: - - conda-forge::tomopy==1.15 - - httomo::httomolib==2.1 - - conda-forge::click>=8.1.3 - - conda-forge::mpi4py>=3.1 - - conda-forge::h5py=*=*mpi_openmpi* - - conda-forge::pyyaml - - conda-forge::numpy<=1.25 - - conda-forge::python - - conda-forge::numexpr>=2.8.4 - - conda-forge::ipython - - conda-forge::loguru - - conda-forge::graypy - - conda-forge::plumbum - - conda-forge::tqdm - - conda-forge::typing_extensions diff --git a/conda/environment.yml b/conda/environment.yml deleted file mode 100644 index 52c5f7ddf..000000000 --- a/conda/environment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: httomo -channels: - - conda-forge - - httomo -dependencies: - - astra-toolbox - - click - - numpy - - cupy=12.3.0 - - graypy - - h5py=*=*mpi_openmpi* - - hdf5plugin - - loguru - - mpi4py - - pyyaml - - nvtx - - python - - plumbum - - tqdm - - typing_extensions - - ipython - - pytest - - pytest-xdist - - pytest-cov - - pytest-mock - - tomopy=1.15 - - ccpi-regulariser - diff --git a/conda/recipe/build.sh b/conda/recipe/build.sh deleted file mode 100755 index 76f9d82ac..000000000 --- a/conda/recipe/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -python -m pip install . - -# Define var for path to httomo installation dir -# -# `SP_DIR` is the path to the `site-packages/` dir in the conda env, see -# https://docs.conda.io/projects/conda-build/en/latest/user-guide/environment-variables.html#environment-variables-set-during-the-build-process -HTTOMO_TARGET="$SP_DIR/$PKG_NAME" - -# Create dirs for YAML templates -mkdir $HTTOMO_TARGET/yaml_templates/ -mkdir $HTTOMO_TARGET/yaml_templates/httomo -mkdir $HTTOMO_TARGET/yaml_templates/httomolib -mkdir $HTTOMO_TARGET/yaml_templates/httomolibgpu -mkdir $HTTOMO_TARGET/yaml_templates/tomopy - -# Generate YAML templates -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/httomo_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomo/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomolib/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomolibgpu/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o $HTTOMO_TARGET/yaml_templates/tomopy/ -python scripts/yaml_unsupported_tomopy_remove.py -t $HTTOMO_TARGET/yaml_templates/tomopy/ -l httomo/methods_database/packages/external/tomopy/tomopy.yaml diff --git a/conda/recipe/conda_build_config.yaml b/conda/recipe/conda_build_config.yaml deleted file mode 100755 index 37c414dd4..000000000 --- a/conda/recipe/conda_build_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -python: - - 3.10 -numpy: - - 1.25 -openmpi_build: - - regular - - external diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml deleted file mode 100644 index 1f9184db0..000000000 --- a/conda/recipe/meta.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{% set pyproject = load_file_data("../../pyproject.toml", from_recipe_dir=True) %} -{% set proj = pyproject.get("project") %} -{% set name = proj.get("name") %} - -package: - name: {{ name|lower }} - version: {{ GIT_DESCRIBE_TAG|trim("v") }} - -source: - path: ../../ - -build: - preserve_egg_dir: False - string: py{{ py }}_openmpi_{{ openmpi_build }} - -test: - files: - - samples/* - - yaml_templates/* - requires: - - pip - - pytest - imports: - - httomo - - httomo.data - source_files: - - tests/* - commands: - - pip check - - pytest -requirements: - build: - - h5py=*=*mpi_openmpi* - - httomolib==2.1.2 - - httomolibgpu==2.1.2 - - tomopy==1.15 - - loguru - - hdf5plugin # added because of the yaml templates dependency script - - mpi4py - - numpy<=1.25 - - pip - - python - - pyyaml - - setuptools - - setuptools-git-versioning - - typing_extensions - run: - - astra-toolbox - - click>=8.1.3 - - cupy=12.3.0 - - cudatoolkit - - graypy - - h5py=*=*mpi_openmpi* - - hdf5plugin - - httomolib==2.1.2 - - httomolibgpu==2.1.2 - - tomopy==1.15 - - ipython - - loguru - - mpi4py - - pyyaml - - numpy<=1.25 - - nvtx - {% if openmpi_build == 'regular' %} - - openmpi - {% else %} - - openmpi * *external* - {% endif %} - - python - - plumbum - - tomobar - - tqdm - - typing_extensions - -about: - home: https://github.com/DiamondLightSource/httomo - license: BSD 3-clause - license_file: LICENSE - summary: 'High Throughput Tomography framework' diff --git a/docs/source/howto/installation.rst b/docs/source/howto/installation.rst index d9fe63b91..06ed39560 100644 --- a/docs/source/howto/installation.rst +++ b/docs/source/howto/installation.rst @@ -1,46 +1,47 @@ Installation Guide ****************** +HTTomo is available on PyPI, so it can be installed into either a virtual environment or a +conda environment. -Install HTTomo as a PyPi package -========================================================= -.. code-block:: console +However, there are certain constraints under which a virtual environment can be used, due to +the dependence on an MPI implementation, the hdf5 library, CUDA libraries, and whether the user +requires using :code:`tomopy` methods in pipelines. - $ pip install httomo # this will install the CPU-only version - $ pip install httomo httomolibgpu # this will install the GPU backend +Virtual environment +=================== -Install HTTomo as a pre-built conda Python package -================================================== +A virtual environment can be used if the following conditions are met: -This installation is preferable as it should take care all of dependencies including :ref:`backends_list` by getting them from the dedicated anaconda channel. +- an MPI implementation is installed on the system (ie, OpenMPI) +- the hdf5 library is installed on the system +- CUDA libraries or CUDA toolkit are installed on the system +- methods from :code:`tomopy` are not required to be used in pipelines .. code-block:: console - $ conda env create --name httomo # create a fresh conda environment - $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge - -Install as a Python module -=========================== + $ python -m venv httomo + $ source httomo/bin/activate + $ MPICC=$(type -p mpicc) pip install mpi4py==3.1.6 + $ pip install cython numpy pkgconfig setuptools # build dependencies of h5py + $ CC=$(type -p mpicc) HDF5_MPI="ON" HDF5_DIR=/path/to/parallel-hdf5 pip install --no-build-isolation --no-binary=h5py h5py + $ pip install cupy-cuda12x # install cupy-cuda11x if CUDA library/CUDA toolkit version is 11.x + $ pip install aiofiles astra-toolbox ccpi-regularisation-cupy click graypy hdf5plugin loguru nvtx pillow pyyaml scikit-image scipy tomobar tqdm + $ pip install --no-deps httomo httomolib httomolibgpu httomo-backends -If installation above for some reason is not working for you, then the best way to install HTTomo is to create conda environment first and then -`pip install` HTTomo into it. You will need to `git clone` HTTomo repository to your disk first. Use `environment.yml` file to install -the GPU-supported HTTomo. For CPU-only version, please use `environment-cpu.yml` instead. +Conda environment +================= .. code-block:: console - $ git clone git@github.com:DiamondLightSource/HTTomo.git # clone the repo - $ conda env create --name httomo --file conda/environment.yml # install dependencies for GPU version - $ conda activate httomo # activate environment - $ pip install . # Install the module + $ conda env create --name httomo + $ conda activate httomo + $ conda install -c conda-forge cupy==12.3.0 openmpi==4.1.6 h5py[build=*openmpi*] + $ conda install -c conda-forge tomopy==1.15 # optional + $ pip install httomo httomolib httomolibgpu Setup HTTomo development environment: ====================================================== .. code-block:: console $ pip install -e .[dev] # development mode - -Build HTTomo as a conda Python package -====================================================== -.. code-block:: console - - $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test diff --git a/pyproject.toml b/pyproject.toml index 8006c5d71..0b7dadcee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,13 @@ requires-python = ">=3.10" dynamic = ["version"] dependencies = [ "click", + "cupy", "numpy", - "httomolib", + "hdf5plugin", + "httomo-backends", "loguru", - "typing-extensions", + "mpi4py", + "pyyaml", "tqdm", "graypy" ] @@ -53,7 +56,7 @@ httomo = "httomo.cli:main" repository = "https://github.com/DiamondLightSource/HTTomo" [project.optional-dependencies] -cpu = ["tomopy" ] +cpu = ["httomolib" ] gpu = ["httomolibgpu"] dev-cpu = [ "pytest", @@ -63,11 +66,8 @@ dev-cpu = [ "flake8", "pyproject-flake8", "pydocstyle", - "toml", "httomolib[dev]", - "tomopy", "plumbum", - "typing_extensions", "mypy", "types-PyYAML", "pre-commit" @@ -80,15 +80,13 @@ dev-gpu = [ "flake8", "pyproject-flake8", "pydocstyle", - "toml", "httomolib[dev]", "httomolibgpu[dev]", - "tomopy", "plumbum", - "typing_extensions", "mypy", "types-PyYAML", - "pre-commit" + "pre-commit", + "ccpi-regularisation-cupy", ] [tool.mypy]