From 959398f4f3581dc1439fdd479dfc0d84db95bb15 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 08:01:01 +0200 Subject: [PATCH 01/23] Add support for execution matrix in CI workflow use spack env proxy-geos- --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2906c5..5296017 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ jobs: continue-on-error: true strategy: matrix: + execution: [omp] preset: [default, kokkos] steps: @@ -26,7 +27,7 @@ jobs: - name: Configure CMake run: | source /data/cemosis/spack/share/spack/setup-env.sh - spack env activate proxy-geos + spack env activate proxy-geos-${{ matrix.execution }} cmake --preset ${{ matrix.preset }} - name: Build From 5e020366ae12746ff4a14f92c1eae8aac88d6be7 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 08:46:32 +0200 Subject: [PATCH 02/23] Refactor test case configuration in CMakeLists.txt to streamline Kokkos and RAJA test case handling --- src/tests/CMakeLists.txt | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 22b8498..a45fac9 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -11,7 +11,7 @@ #set (testcase Kokkos1) if ( USE_KOKKOS ) -LIST(APPEND TESTCASES "Kokkos" "Kokkos0" ) +LIST(APPEND TESTCASES "Kokkos" ) message(STATUS "Building tests with Kokkos: ${TESTCASES}") endif() #set (testcase FDTDKokkos) @@ -20,15 +20,18 @@ endif() #set (testcase raja_hip_daxpy) # the followings need "-DUSE_RAJA=ON -DENABLE_CUDA=ON" -#set (testcase LvArray) -#set (testcase LvArrayRAJACudaStencil) -#set (testcase LvArrayRAJACuda) -#set (testcase LvArray_cuda) -#set (testcase RAJAInline) -#set (testcase Raja_cuda) -#set (testcase FDTDRAJA) -#set (testcase raja_cuda_daxpy) - +if ( USE_RAJA ) + #set (testcase LvArray) + #set (testcase LvArrayRAJACudaStencil) + #set (testcase LvArrayRAJACuda) + #set (testcase LvArray_cuda) + #set (testcase RAJAInline) + #set (testcase Raja_cuda) + #set (testcase FDTDRAJA) + #set (testcase raja_cuda_daxpy) + LIST(APPEND TESTCASES "LvArray" "LvArrayRAJACudaStencil") + message(STATUS "Building tests with OpenMP: ${TESTCASES}") +endif(USE_RAJA) # the followings need "-DENABLE_CUDA=ON" #set (testcase cuda) #set (testcase cuda_runtime) From 2b81d31ecacc86e5ab4bb8b78b87dae5a8f1a2c1 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 09:05:12 +0200 Subject: [PATCH 03/23] Update CI workflow to use env_variant for execution matrix and add Spack environment configuration add spack environement for proxy-geos #8 --- .github/workflows/ci.yml | 4 ++-- .gitignore | 2 ++ spack/proxy-geos-omp/spack.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 spack/proxy-geos-omp/spack.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5296017..ab488ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: continue-on-error: true strategy: matrix: - execution: [omp] + env_variant: [omp] preset: [default, kokkos] steps: @@ -27,7 +27,7 @@ jobs: - name: Configure CMake run: | source /data/cemosis/spack/share/spack/setup-env.sh - spack env activate proxy-geos-${{ matrix.execution }} + spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} - name: Build diff --git a/.gitignore b/.gitignore index 7af28f1..7917bb9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ readme_folder env_old.sh ./install/ ./build/ +*.spack-env* +spack.lock \ No newline at end of file diff --git a/spack/proxy-geos-omp/spack.yaml b/spack/proxy-geos-omp/spack.yaml new file mode 100644 index 0000000..f1b6b1c --- /dev/null +++ b/spack/proxy-geos-omp/spack.yaml @@ -0,0 +1,24 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + # add package specs to the `specs` list + packages: + all: + compiler: [gcc@14:, clang@14:18] + mpi: + require: ['openmpi@:4 fabrics=auto'] + definitions: + - compilers: [gcc@14:] + - packages: [caliper,chai+openmp,raja+openmp,umpire+openmp,kokkos+openmp] + specs: + - $compilers + - matrix: + - [$packages] + - [$%compilers] + view: true + concretizer: + unify: when_possible + config: + target: x86_64_v3 \ No newline at end of file From a727431c1cbe4bc3f84270f1e25f91d243906015 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 09:12:29 +0200 Subject: [PATCH 04/23] Add message output for custom CMAKE_CXX_STANDARD setting in CMakeLists.txt add support for CMAKE_CXX_STANDARD #6 --- CMakeLists.txt | 2 +- CMakePresets.json | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8ea4f7..17c0a7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 20) endif() - +message(STATUS "CMAKE_CXX_STANDARD is set to ${CMAKE_CXX_STANDARD}") #----------------------------------------------------- # Setup Flags for whether considering an external libs diff --git a/CMakePresets.json b/CMakePresets.json index 1a801b9..1da672e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -36,6 +36,7 @@ "description": "kokkos config", "generator": "Unix Makefiles", "binaryDir": "${sourceDir}/build/kokkos", + "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", @@ -58,6 +59,7 @@ "description": "raja config", "generator": "Unix Makefiles", "binaryDir": "${sourceDir}/build/raja", + "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", From 294040ebd214fe7dfc63c8b767922940eeb07339 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 09:48:29 +0200 Subject: [PATCH 05/23] Enhance CI workflow by adding build-cache job and integrating Spack for package management --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++++++++++-- spack/proxy-geos-omp/spack.yaml | 5 +++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab488ad..ea9a4f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,35 @@ on: - main jobs: + build-cache: + runs-on: ubuntu-24.04 + permissions: + packages: write + continue-on-error: true + strategy: + matrix: + env_variant: [omp] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Spack + uses: spack/setup-spack@v2 + + - name: Install + run: spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature + + - name: Push packages and update index + run: | + spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index local-buildcache + if: ${{ !cancelled() }} + build: - runs-on: self-spack + needs: build-cache + runs-on: ubuntu-24.04 continue-on-error: true strategy: matrix: @@ -24,9 +51,13 @@ jobs: with: submodules: true + - name: Set up Spack + uses: spack/setup-spack@v2 + + - name: Configure CMake + shell: spack-bash run: | - source /data/cemosis/spack/share/spack/setup-env.sh spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} @@ -35,3 +66,4 @@ jobs: - name: Test run: ctest --preset ${{ matrix.preset }} + \ No newline at end of file diff --git a/spack/proxy-geos-omp/spack.yaml b/spack/proxy-geos-omp/spack.yaml index f1b6b1c..c95435f 100644 --- a/spack/proxy-geos-omp/spack.yaml +++ b/spack/proxy-geos-omp/spack.yaml @@ -7,6 +7,7 @@ spack: packages: all: compiler: [gcc@14:, clang@14:18] + require: ['target=x86_64_v3'] mpi: require: ['openmpi@:4 fabrics=auto'] definitions: @@ -20,5 +21,5 @@ spack: view: true concretizer: unify: when_possible - config: - target: x86_64_v3 \ No newline at end of file + mirrors: + numpex-buildcache: oci://ghcr.io/numpex/spack-buildcache \ No newline at end of file From 002563f0e5a9647eb8ef678fb13c78eff6476225 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 09:52:06 +0200 Subject: [PATCH 06/23] change cache name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea9a4f2..019d49d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,8 @@ jobs: - name: Push packages and update index run: | - spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index local-buildcache + spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache + spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache if: ${{ !cancelled() }} build: From 9aed440f20e1d039e1c0a93481d000a354fa58a7 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 09:55:19 +0200 Subject: [PATCH 07/23] Refactor CI workflow to set up Spack earlier and streamline package installation steps --- .github/workflows/ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 019d49d..ad30ecb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,18 +19,20 @@ jobs: matrix: env_variant: [omp] steps: + - name: Set up Spack + uses: spack/setup-spack@v2 - name: Checkout repository uses: actions/checkout@v4 with: submodules: true - - - name: Set up Spack - uses: spack/setup-spack@v2 - - name: Install - run: spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature + shell: spack-bash + run: | + ls -l + spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature - name: Push packages and update index + shell: spack-bash run: | spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache From 49ad8fbfdaccd926fccf2d58f1ebe4d5031dd39d Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 10:04:30 +0200 Subject: [PATCH 08/23] Update CI workflow to disable continue-on-error and reorder Spack setup step --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad30ecb..abf2c91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,25 +14,25 @@ jobs: runs-on: ubuntu-24.04 permissions: packages: write - continue-on-error: true + continue-on-error: false strategy: matrix: env_variant: [omp] steps: - - name: Set up Spack - uses: spack/setup-spack@v2 + - name: Checkout repository uses: actions/checkout@v4 with: submodules: true + + - name: Set up Spack + uses: spack/setup-spack@v2 - name: Install - shell: spack-bash run: | ls -l spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature - name: Push packages and update index - shell: spack-bash run: | spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache From 12d3d035860885d3b9b71fdb73834baa139cd3ca Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 10:14:59 +0200 Subject: [PATCH 09/23] change path to spack clone --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abf2c91..4d54892 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,10 @@ jobs: submodules: true - name: Set up Spack - uses: spack/setup-spack@v2 + uses: spack/setup-spack@v2 + with: + color: true + path: _spack - name: Install run: | ls -l From 5d166f21c4b5024a7925a5fe9ed5e2425576395c Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 10:59:41 +0200 Subject: [PATCH 10/23] fix build cache --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d54892..4ab0a0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,8 @@ jobs: - name: Push packages and update index run: | - spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache - spack -e . buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache + spack -e spack/proxy-geos-${{ matrix.env_variant }} mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache + spack -e spack/proxy-geos-${{ matrix.env_variant }} buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache if: ${{ !cancelled() }} build: From dc399f3edf93412ef1e459de702f0fc60be9043b Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 11:16:48 +0200 Subject: [PATCH 11/23] remove spack-bash shell specification from CMake configuration step --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ab0a0a..0225da8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,6 @@ jobs: - name: Configure CMake - shell: spack-bash run: | spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} From c9fae1eb05888df2cadf3d699b3685ecbca024ff Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 11:37:23 +0200 Subject: [PATCH 12/23] use self-* --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0225da8..e1eb8cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: jobs: build-cache: - runs-on: ubuntu-24.04 + runs-on: self-ubuntu-24.04 permissions: packages: write continue-on-error: false @@ -43,7 +43,7 @@ jobs: build: needs: build-cache - runs-on: ubuntu-24.04 + runs-on: self-ubuntu-24.04 continue-on-error: true strategy: matrix: @@ -62,7 +62,7 @@ jobs: - name: Configure CMake run: | - spack env activate spack/proxy-geos-${{ matrix.env_variant }} + spack env activate --sh spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} - name: Build From 6f7341a823187bd8b67b31264e392d9941243327 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 12:08:17 +0200 Subject: [PATCH 13/23] use spack-bash --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1eb8cb..02d4263 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: build: needs: build-cache runs-on: self-ubuntu-24.04 + permissions: + packages: write continue-on-error: true strategy: matrix: @@ -59,10 +61,10 @@ jobs: - name: Set up Spack uses: spack/setup-spack@v2 - - name: Configure CMake + shell: spack-bash run: | - spack env activate --sh spack/proxy-geos-${{ matrix.env_variant }} + spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} - name: Build From 912166404ed861bc7f1ea7d77c4e4056cc89d224 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 12:13:56 +0200 Subject: [PATCH 14/23] set padded length for install tree in spack configuration --- spack/proxy-geos-omp/spack.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spack/proxy-geos-omp/spack.yaml b/spack/proxy-geos-omp/spack.yaml index c95435f..fb3b18f 100644 --- a/spack/proxy-geos-omp/spack.yaml +++ b/spack/proxy-geos-omp/spack.yaml @@ -3,7 +3,9 @@ # It describes a set of packages to be installed, along with # configuration settings. spack: - # add package specs to the `specs` list + config: + install_tree: + padded_length: 128 packages: all: compiler: [gcc@14:, clang@14:18] From cd5278f1caa08a64b6663520a56cd0e7244feac9 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 12:18:45 +0200 Subject: [PATCH 15/23] update CI workflow to use standard Ubuntu runner and latest Spack setup version --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02d4263..167a753 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: jobs: build-cache: - runs-on: self-ubuntu-24.04 + runs-on: ubuntu-24.04 permissions: packages: write continue-on-error: false @@ -43,7 +43,7 @@ jobs: build: needs: build-cache - runs-on: self-ubuntu-24.04 + runs-on: ubuntu-24.04 permissions: packages: write continue-on-error: true @@ -59,7 +59,7 @@ jobs: submodules: true - name: Set up Spack - uses: spack/setup-spack@v2 + uses: spack/setup-spack@v2.1.1 - name: Configure CMake shell: spack-bash From 3abfe6d7f9f5deedc36aeaadc548fac57b8eb714 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 12:20:23 +0200 Subject: [PATCH 16/23] up setup-spack action --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 167a753..0e12743 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,9 @@ jobs: submodules: true - name: Set up Spack - uses: spack/setup-spack@v2 + uses: spack/setup-spack@v2.1.1 with: + buildcache: true color: true path: _spack - name: Install From db1129ada3f98691e1fcbf71b1fc2601f1d98063 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 13:56:34 +0200 Subject: [PATCH 17/23] fix: update CMake shell configuration to include placeholder --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e12743..43664ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: uses: spack/setup-spack@v2.1.1 - name: Configure CMake - shell: spack-bash + shell: spack-bash {0} run: | spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} From f58d1c5f5fd54d1abb67584d40ea074d5b24ea13 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 14:03:24 +0200 Subject: [PATCH 18/23] feat: enhance CI workflow by adding buildcache and color options for Spack setup --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43664ff..777421b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,10 @@ jobs: - name: Set up Spack uses: spack/setup-spack@v2.1.1 + with: + buildcache: true + color: true + path: _spack - name: Configure CMake shell: spack-bash {0} From e26c03dbd6341cf859925290eaf0895bdaa18774 Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 14:37:24 +0200 Subject: [PATCH 19/23] install spack package --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 777421b..656ebe9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: color: true path: _spack + - name: Spack Install + run: spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature + - name: Configure CMake shell: spack-bash {0} run: | @@ -77,4 +80,3 @@ jobs: - name: Test run: ctest --preset ${{ matrix.preset }} - \ No newline at end of file From 4f0063acfb86aa617fe2023ca2235a55370278ec Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 9 Apr 2025 15:38:01 +0200 Subject: [PATCH 20/23] feat: add test for SEM_EXECUTABLE in CMake configuration --- src/sem/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sem/CMakeLists.txt b/src/sem/CMakeLists.txt index bf5463b..5e7c52a 100644 --- a/src/sem/CMakeLists.txt +++ b/src/sem/CMakeLists.txt @@ -43,3 +43,4 @@ blt_add_executable( NAME ${SEM_EXECUTABLE} install(TARGETS ${SEM_EXECUTABLE} RUNTIME DESTINATION bin) +add_test(NAME ${SEM_EXECUTABLE} COMMAND ${SEM_EXECUTABLE}) \ No newline at end of file From affbcfbc1c5af7f30233d24b10957d467bc1a5dc Mon Sep 17 00:00:00 2001 From: Thomas Saigre Date: Wed, 16 Apr 2025 14:44:20 +0200 Subject: [PATCH 21/23] tiny fix in CMakeLists to hide warnings --- src/CMakeLists.txt | 74 +++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 062dcf1..c8c22d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,9 +9,9 @@ # (3) If USE_KOKKOS is ON, it is using KOKKOS with or without CUDA and OpenMP. # # -# Edit the env_var.sh file in the TPLs folder to set the Env variables to be defined before build of both TPL and the ProxyApp +# Edit the env_var.sh file in the TPLs folder to set the Env variables to be defined before build of both TPL and the ProxyApp # _TPL must be first compiled and installed from the TPLs repo -# The exported Env variables are required to set the _TPL paths in the config file configs/config_proxy-app.cmake +# The exported Env variables are required to set the _TPL paths in the config file configs/config_proxy-app.cmake # #----------------------------------------------------------------------------------------------------------------------------- @@ -96,7 +96,7 @@ endif() if(USE_OMP) set (SOLVER "OMP" CACHE PATH "" FORCE) message(STATUS "BUILDING SOLVER including OpenMP+Vector on CPU") - + set (USE_KOKKOS OFF CACHE BOOL "") set (USE_RAJA OFF CACHE BOOL "") set (USE_LVARRAY OFF CACHE BOOL "" FORCE) @@ -118,24 +118,24 @@ if ( USE_RAJA) # disable OMP and KOKKOS set (USE_OMP OFF CACHE BOOL "") set (USE_KOKKOS OFF CACHE BOOL "") - + # set solver message( " " ) message( STATUS "-------------------RAJA SETTINGS---------------------------------------" ) set (SOLVER "Raja" CACHE PATH "" FORCE) - message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = "${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) + message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = " ${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) if(NOT (${ENABLE_OPENMP} OR ${ENABLE_CUDA})) message(STATUS "ENABLE_OPENMP is set to " ${ENABLE_OPENMP} " ENABLE_CUDA " ${ENABLE_CUDA}) message(STATUS "The TPLs have been built without enabling one of the OPENMP or CUDA feature required for USE_RAJA option: please rebuild the RAJA library ?") endif() - + #if(DEFINED CAMP_DIR) # find_package(camp REQUIRED # PATHS ${CAMP_DIR} # NO_DEFAULT_PATH) #else() - # # CAMP is required when using RAJA. Consider using find_package when RAJA_DIR not defined, that is BUILD_FROM_TPLMIRROR=OFF - # message(STATUS "ProxyApp requires CAMP: CAMP_DIR not set.") + # # CAMP is required when using RAJA. Consider using find_package when RAJA_DIR not defined, that is BUILD_FROM_TPLMIRROR=OFF + # message(STATUS "ProxyApp requires CAMP: CAMP_DIR not set.") # find_package(camp REQUIRED) #endif() @@ -148,21 +148,21 @@ if ( USE_RAJA) ${camp_DIR} NO_DEFAULT_PATH) else() - message(STATUS "ProxyApp requires RAJA: RAJA_DIR not set.") + message(STATUS "ProxyApp requires RAJA: RAJA_DIR not set.") find_package(camp REQUIRED) find_package(RAJA REQUIRED) endif() - message(STATUS "BUILDING SOLVER including RAJA with the following setting (after find_package raja): RAJA_ENABLE_CUDA = "${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) - + message(STATUS "BUILDING SOLVER including RAJA with the following setting (after find_package raja): RAJA_ENABLE_CUDA = " ${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) + get_target_property(CAMP_INCLUDE_DIRS camp INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(camp PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${CAMP_INCLUDE_DIRS}") message("-- camp_DIR: ${camp_DIR} or ${CAMP_DIR}") message("-- CAMP_INCLUDE_DIRS: ${CAMP_INCLUDE_DIRS}") set (dependencyList ${dependencyList} camp ) - + get_target_property(RAJA_INCLUDE_DIRS RAJA INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(RAJA PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${RAJA_INCLUDE_DIRS}") - message(STATUS "USE RAJA in RAJA_DIR = ${RAJA_DIR}") + message(STATUS "USE RAJA in RAJA_DIR = ${RAJA_DIR}") message("-- RAJA_INCLUDE_DIRS: ${RAJA_INCLUDE_DIRS}") if( ENABLE_HIP ) @@ -170,7 +170,7 @@ if ( USE_RAJA) set(RAJA_ENABLE_CUDA OFF CACHE BOOL "" FORCE) include_directories("${ROCM_ROOT_DIR}/include") endif() - + ################################ # Umpire ################################ @@ -184,24 +184,24 @@ if ( USE_RAJA) message(STATUS "ProxyApp requires Umpire: UMPIRE_DIR not set for finding package.") find_package(umpire REQUIRED) endif() - + get_target_property(UMPIRE_INCLUDE_DIRS umpire INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(umpire PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${UMPIRE_INCLUDE_DIRS}") message(STATUS " USE RAJA: RAJA will be compiled with UMPIRE in UMPIRE_DIR = ${UMPIRE_DIR}") - message("-- UMPIRE_INCLUDE_DIRS: ${UMPIRE_INCLUDE_DIRS} / umpire_DIR "${umpire_DIR}) + message("-- UMPIRE_INCLUDE_DIRS: ${UMPIRE_INCLUDE_DIRS} / umpire_DIR " ${umpire_DIR}) set (extraComponentsLinkList ${extraComponentsLinkList} umpire ) endif() #endif() - + ################################ # CHAI ################################ - # CHAI is not a requisite for RAJA. Umpire is a requisite for CHAI + # CHAI is not a requisite for RAJA. Umpire is a requisite for CHAI if(ENABLE_CHAI) set(ENABLE_UMPIRE ON CACHE BOOL "" FORCE) if(DEFINED CHAI_DIR) - find_package(chai REQUIRED - PATHS ${CHAI_DIR} + find_package(chai REQUIRED + PATHS ${CHAI_DIR} NO_DEFAULT_PATH) else() message(STATUS "ProxyApp requires CHAI: CHAI_DIR not set for finding package.") @@ -216,7 +216,7 @@ if ( USE_RAJA) if(ENABLE_OPENMP) set (extraComponentsLinkList ${extraComponentsLinkList} openmp ) endif() - + ################################ # CALIPER ################################ @@ -240,13 +240,13 @@ endif() if(USE_LVARRAY_EXT) # Find package if(NOT EXISTS ${LVARRAY_DIR}) - message(FATAL_ERROR "LVARRAY_DIR must be set to a defined directory to find the lvarray lib. LVARRAY_DIR is now set to "${LVARRAY_DIR}) + message(FATAL_ERROR "LVARRAY_DIR must be set to a defined directory to find the lvarray lib. LVARRAY_DIR is now set to " ${LVARRAY_DIR}) endif() find_package(LVARRAY REQUIRED) get_target_property(LVARRAY_INCLUDE_DIRS lvarray INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(lvarray PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${LVARRAY_INCLUDE_DIRS}") message(STATUS "-- Find package for lvarray - LVARRAY_INCLUDE_DIRS " ${LVARRAY_INCLUDE_DIRS}) -else() +else() add_subdirectory( LvArray ) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/LvArray/src) endif() @@ -273,7 +273,7 @@ if (NOT USE_KOKKOS) else() set (USE_LVARRAY OFF CACHE BOOL "" FORCE) set (USE_VECTOR OFF CACHE BOOL "" FORCE) - + message ("-- USE_KOKKOS flag is set to be true and KOKOS is now enabled") set (SOLVER "Kokkos" CACHE PATH "" FORCE) @@ -286,11 +286,11 @@ else() find_package(Kokkos REQUIRED) endif() message("-- Found the KOKKOS library in ${KOKKOS_DIR}") - + get_target_property(KOKKOS_INCLUDE_DIRS Kokkos::kokkos INTERFACE_INCLUDE_DIRECTORIES) get_target_property(KOKKOS_LINK_LIBS Kokkos::kokkos INTERFACE_LINK_LIBRARIES) - - message(STATUS "KOKKOS_INCLUDE_DIRS: ${KOKKOS_INCLUDE_DIRS}") + + message(STATUS "KOKKOS_INCLUDE_DIRS: ${KOKKOS_INCLUDE_DIRS}") message("-- KOKKOS_LINK_LIBS: ${KOKKOS_LINK_LIBS}") #get_target_property(KOKKOSCORE_LINK_OPTION Kokkos::kokkoscore INTERFACE_LINK_OPTIONS) @@ -317,31 +317,31 @@ if(ENABLE_OPENMP OR USE_OMP) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}" CACHE STRING "" FORCE) - - + + set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_RELWITHDEBINFO "${CMAKE_Fortran_FLAGS_RELWITHDEBINFO} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_MINSIZEREL "${CMAKE_Fortran_FLAGS_MINSIZEREL} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) endif() -message(STATUS "-- The dependencies for building the proxy-app are: " ${dependencyList} " extraComponentsLinkLists = "${extraComponentsLinkList}) +message(STATUS "-- The dependencies for building the proxy-app are: " ${dependencyList} " extraComponentsLinkLists = " ${extraComponentsLinkList}) if((USE_KOKKOS OR USE_RAJA) AND (ENABLE_OPENMP OR ENABLE_MPI)) # SettingFlag associated to the config.. used for the name of the executable ## What programming on the host - + if(ENABLE_OPENMP) set(flag_hmodel "OMP") elseif(ENABLE_MPI) @@ -349,7 +349,7 @@ if((USE_KOKKOS OR USE_RAJA) AND (ENABLE_OPENMP OR ENABLE_MPI)) #else() #set(flag_hmodel "SERIAL") endif() - + set(SOLVER "${SOLVER}_${flag_hmodel}" CACHE PATH "" FORCE) endif() # The solver's flag @@ -370,7 +370,7 @@ configure_file( utils/commonConfig.hpp.in add_subdirectory( sem ) add_subdirectory( fd ) -if ( ENABLE_TESTS ) +if ( ENABLE_TESTS ) message(STATUS "Building tests") add_subdirectory( tests ) else() From a1dd4103a81989969788d47a9d01542d1ee87218 Mon Sep 17 00:00:00 2001 From: Thomas Saigre Date: Wed, 16 Apr 2025 17:50:39 +0200 Subject: [PATCH 22/23] use gcc12: (for gaya) /cc @prudhomm --- spack/proxy-geos-omp/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack/proxy-geos-omp/spack.yaml b/spack/proxy-geos-omp/spack.yaml index fb3b18f..0c99230 100644 --- a/spack/proxy-geos-omp/spack.yaml +++ b/spack/proxy-geos-omp/spack.yaml @@ -13,7 +13,7 @@ spack: mpi: require: ['openmpi@:4 fabrics=auto'] definitions: - - compilers: [gcc@14:] + - compilers: [gcc@12:] - packages: [caliper,chai+openmp,raja+openmp,umpire+openmp,kokkos+openmp] specs: - $compilers From 3c874d402c90aa1fbc2d0d29fe071cd097af2caa Mon Sep 17 00:00:00 2001 From: Christophe Prud'homme Date: Wed, 16 Apr 2025 19:00:58 +0200 Subject: [PATCH 23/23] feat: update CMake presets for ROCm support and improve build configurations add spack rocm environment #9 --- CMakePresets.json | 40 ++++++++++++++++++++++++++------ spack/proxy-geos-rocm/spack.yaml | 25 ++++++++++++++++++++ src/CMakeLists.txt | 4 ++-- src/tests/CMakeLists.txt | 4 ++-- 4 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 spack/proxy-geos-rocm/spack.yaml diff --git a/CMakePresets.json b/CMakePresets.json index 1da672e..eecbffc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -12,7 +12,7 @@ "displayName": "Default Config", "description": "Default config", "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/default", + "binaryDir": "${sourceDir}/build/${presetName}", "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", @@ -34,13 +34,13 @@ "name": "kokkos", "displayName": "kokkos Config", "description": "kokkos config", - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/kokkos", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build/${presetName}", "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", - "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/kokkos/install/", + "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/${presetName}/install/", "CMAKE_BUILD_TYPE": "Release", "USE_OMP": "OFF", "USE_RAJA": "OFF", @@ -58,12 +58,12 @@ "displayName": "raja Config", "description": "raja config", "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/raja", + "binaryDir": "${sourceDir}/build/${presetName}", "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", - "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/raja/install/", + "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/${presetName}/install/", "CMAKE_BUILD_TYPE": "Release", "USE_OMP": "OFF", "USE_RAJA": "ON", @@ -99,6 +99,23 @@ "cacheVariables": { "CMAKE_CXX_STANDARD": "17" } + }, + { + "name": "rocm", + "hidden": true, + "cacheVariables": { + "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_C_COMPILER": "amdclang", + "DEVICE": "rocm" + } + }, + { + "name": "kokkos+rocm", + "inherits": ["rocm", "kokkos"], + "binaryDir": "${sourceDir}/build/${presetName}", + "cacheVariables": { + + } } ], "buildPresets": [ @@ -116,6 +133,10 @@ "name": "raja", "configurePreset": "raja", "jobs": 10 + }, + { + "name": "kokkos+rocm", + "configurePreset": "kokkos+rocm" } ], "testPresets": [ @@ -143,7 +164,12 @@ "name": "raja", "inherits": "default", "configurePreset": "raja" - } + }, + { + "name": "kokkos+rocm", + "inherits": ["kokkos"], + "configurePreset": "kokkos+rocm" + } ], "vendor": { "example.com/ExampleIDE/1.0": { diff --git a/spack/proxy-geos-rocm/spack.yaml b/spack/proxy-geos-rocm/spack.yaml new file mode 100644 index 0000000..8697a18 --- /dev/null +++ b/spack/proxy-geos-rocm/spack.yaml @@ -0,0 +1,25 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + config: + install_tree: + padded_length: 128 + packages: + all: + compiler: [gcc@14:, clang@14:18] + require: ['target=x86_64_v3'] + definitions: + - compilers: [gcc@14:] + - packages: ['caliper +rocm amdgpu_target=gfx90a','chai +rocm amdgpu_target=gfx90a','raja +rocm amdgpu_target=gfx90a','umpire +rocm amdgpu_target=gfx90a','kokkos@4.6: +rocm amdgpu_target=gfx90a'] + specs: + - $compilers + - matrix: + - [$packages] + - [$%compilers] + view: true + concretizer: + unify: when_possible + mirrors: + numpex-buildcache: oci://ghcr.io/numpex/spack-buildcache \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 062dcf1..8dcf218 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -123,9 +123,9 @@ if ( USE_RAJA) message( " " ) message( STATUS "-------------------RAJA SETTINGS---------------------------------------" ) set (SOLVER "Raja" CACHE PATH "" FORCE) - message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = "${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) + message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = ${RAJA_ENABLE_CUDA} and RAJA_ENABLE_OPENMP = ${RAJA_ENABLE_OPENMP}") if(NOT (${ENABLE_OPENMP} OR ${ENABLE_CUDA})) - message(STATUS "ENABLE_OPENMP is set to " ${ENABLE_OPENMP} " ENABLE_CUDA " ${ENABLE_CUDA}) + message(STATUS "ENABLE_OPENMP is set to ${ENABLE_OPENMP} ENABLE_CUDA ${ENABLE_CUDA}") message(STATUS "The TPLs have been built without enabling one of the OPENMP or CUDA feature required for USE_RAJA option: please rebuild the RAJA library ?") endif() diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index a45fac9..aae141a 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -9,9 +9,9 @@ #set (testcase Kokkos) #set (testcase Kokkos0) #set (testcase Kokkos1) -if ( USE_KOKKOS ) +if ( USE_KOKKOS AND NOT (DEVICE STREQUAL "rocm")) -LIST(APPEND TESTCASES "Kokkos" ) + LIST(APPEND TESTCASES "Kokkos" ) message(STATUS "Building tests with Kokkos: ${TESTCASES}") endif() #set (testcase FDTDKokkos)