diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2906c5..656ebe9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,11 +10,47 @@ on: - main jobs: + build-cache: + runs-on: ubuntu-24.04 + permissions: + packages: write + continue-on-error: false + strategy: + matrix: + env_variant: [omp] + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Spack + uses: spack/setup-spack@v2.1.1 + with: + buildcache: true + color: true + path: _spack + - name: Install + run: | + ls -l + spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature + + - name: Push packages and update index + run: | + spack -e spack/proxy-geos-${{ matrix.env_variant }} mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" numpex-buildcache + spack -e spack/proxy-geos-${{ matrix.env_variant }} buildcache push --base-image ubuntu:24.04 --unsigned --update-index numpex-buildcache + if: ${{ !cancelled() }} + build: - runs-on: self-spack + needs: build-cache + runs-on: ubuntu-24.04 + permissions: + packages: write continue-on-error: true strategy: matrix: + env_variant: [omp] preset: [default, kokkos] steps: @@ -23,10 +59,20 @@ jobs: with: submodules: true + - name: Set up Spack + uses: spack/setup-spack@v2.1.1 + with: + buildcache: true + color: true + path: _spack + + - name: Spack Install + run: spack -e spack/proxy-geos-${{ matrix.env_variant }} install --no-check-signature + - name: Configure CMake + shell: spack-bash {0} run: | - source /data/cemosis/spack/share/spack/setup-env.sh - spack env activate proxy-geos + spack env activate spack/proxy-geos-${{ matrix.env_variant }} cmake --preset ${{ matrix.preset }} - name: Build diff --git a/.gitignore b/.gitignore index 7af28f1..7917bb9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ readme_folder env_old.sh ./install/ ./build/ +*.spack-env* +spack.lock \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b8ea4f7..17c0a7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 20) endif() - +message(STATUS "CMAKE_CXX_STANDARD is set to ${CMAKE_CXX_STANDARD}") #----------------------------------------------------- # Setup Flags for whether considering an external libs diff --git a/CMakePresets.json b/CMakePresets.json index 1a801b9..eecbffc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -12,7 +12,7 @@ "displayName": "Default Config", "description": "Default config", "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/default", + "binaryDir": "${sourceDir}/build/${presetName}", "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", @@ -34,12 +34,13 @@ "name": "kokkos", "displayName": "kokkos Config", "description": "kokkos config", - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/kokkos", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build/${presetName}", + "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", - "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/kokkos/install/", + "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/${presetName}/install/", "CMAKE_BUILD_TYPE": "Release", "USE_OMP": "OFF", "USE_RAJA": "OFF", @@ -57,11 +58,12 @@ "displayName": "raja Config", "description": "raja config", "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/raja", + "binaryDir": "${sourceDir}/build/${presetName}", + "inherits": "cpp20", "cacheVariables": { "CMAKE_CXX_COMPILER": "g++", "CMAKE_C_COMPILER": "gcc", - "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/raja/install/", + "CMAKE_INSTALL_PREFIX" : "${sourceDir}/build/${presetName}/install/", "CMAKE_BUILD_TYPE": "Release", "USE_OMP": "OFF", "USE_RAJA": "ON", @@ -97,6 +99,23 @@ "cacheVariables": { "CMAKE_CXX_STANDARD": "17" } + }, + { + "name": "rocm", + "hidden": true, + "cacheVariables": { + "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_C_COMPILER": "amdclang", + "DEVICE": "rocm" + } + }, + { + "name": "kokkos+rocm", + "inherits": ["rocm", "kokkos"], + "binaryDir": "${sourceDir}/build/${presetName}", + "cacheVariables": { + + } } ], "buildPresets": [ @@ -114,6 +133,10 @@ "name": "raja", "configurePreset": "raja", "jobs": 10 + }, + { + "name": "kokkos+rocm", + "configurePreset": "kokkos+rocm" } ], "testPresets": [ @@ -141,7 +164,12 @@ "name": "raja", "inherits": "default", "configurePreset": "raja" - } + }, + { + "name": "kokkos+rocm", + "inherits": ["kokkos"], + "configurePreset": "kokkos+rocm" + } ], "vendor": { "example.com/ExampleIDE/1.0": { diff --git a/spack/proxy-geos-omp/spack.yaml b/spack/proxy-geos-omp/spack.yaml new file mode 100644 index 0000000..0c99230 --- /dev/null +++ b/spack/proxy-geos-omp/spack.yaml @@ -0,0 +1,27 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + config: + install_tree: + padded_length: 128 + packages: + all: + compiler: [gcc@14:, clang@14:18] + require: ['target=x86_64_v3'] + mpi: + require: ['openmpi@:4 fabrics=auto'] + definitions: + - compilers: [gcc@12:] + - packages: [caliper,chai+openmp,raja+openmp,umpire+openmp,kokkos+openmp] + specs: + - $compilers + - matrix: + - [$packages] + - [$%compilers] + view: true + concretizer: + unify: when_possible + mirrors: + numpex-buildcache: oci://ghcr.io/numpex/spack-buildcache \ No newline at end of file diff --git a/spack/proxy-geos-rocm/spack.yaml b/spack/proxy-geos-rocm/spack.yaml new file mode 100644 index 0000000..8697a18 --- /dev/null +++ b/spack/proxy-geos-rocm/spack.yaml @@ -0,0 +1,25 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + config: + install_tree: + padded_length: 128 + packages: + all: + compiler: [gcc@14:, clang@14:18] + require: ['target=x86_64_v3'] + definitions: + - compilers: [gcc@14:] + - packages: ['caliper +rocm amdgpu_target=gfx90a','chai +rocm amdgpu_target=gfx90a','raja +rocm amdgpu_target=gfx90a','umpire +rocm amdgpu_target=gfx90a','kokkos@4.6: +rocm amdgpu_target=gfx90a'] + specs: + - $compilers + - matrix: + - [$packages] + - [$%compilers] + view: true + concretizer: + unify: when_possible + mirrors: + numpex-buildcache: oci://ghcr.io/numpex/spack-buildcache \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 062dcf1..8e9f313 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,9 +9,9 @@ # (3) If USE_KOKKOS is ON, it is using KOKKOS with or without CUDA and OpenMP. # # -# Edit the env_var.sh file in the TPLs folder to set the Env variables to be defined before build of both TPL and the ProxyApp +# Edit the env_var.sh file in the TPLs folder to set the Env variables to be defined before build of both TPL and the ProxyApp # _TPL must be first compiled and installed from the TPLs repo -# The exported Env variables are required to set the _TPL paths in the config file configs/config_proxy-app.cmake +# The exported Env variables are required to set the _TPL paths in the config file configs/config_proxy-app.cmake # #----------------------------------------------------------------------------------------------------------------------------- @@ -96,7 +96,7 @@ endif() if(USE_OMP) set (SOLVER "OMP" CACHE PATH "" FORCE) message(STATUS "BUILDING SOLVER including OpenMP+Vector on CPU") - + set (USE_KOKKOS OFF CACHE BOOL "") set (USE_RAJA OFF CACHE BOOL "") set (USE_LVARRAY OFF CACHE BOOL "" FORCE) @@ -118,24 +118,24 @@ if ( USE_RAJA) # disable OMP and KOKKOS set (USE_OMP OFF CACHE BOOL "") set (USE_KOKKOS OFF CACHE BOOL "") - + # set solver message( " " ) message( STATUS "-------------------RAJA SETTINGS---------------------------------------" ) set (SOLVER "Raja" CACHE PATH "" FORCE) - message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = "${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) + message(STATUS "BUILDING SOLVER including RAJA with the following setting (before find_package raja): RAJA_ENABLE_CUDA = ${RAJA_ENABLE_CUDA} and RAJA_ENABLE_OPENMP = ${RAJA_ENABLE_OPENMP}") if(NOT (${ENABLE_OPENMP} OR ${ENABLE_CUDA})) - message(STATUS "ENABLE_OPENMP is set to " ${ENABLE_OPENMP} " ENABLE_CUDA " ${ENABLE_CUDA}) + message(STATUS "ENABLE_OPENMP is set to ${ENABLE_OPENMP} ENABLE_CUDA ${ENABLE_CUDA}") message(STATUS "The TPLs have been built without enabling one of the OPENMP or CUDA feature required for USE_RAJA option: please rebuild the RAJA library ?") endif() - + #if(DEFINED CAMP_DIR) # find_package(camp REQUIRED # PATHS ${CAMP_DIR} # NO_DEFAULT_PATH) #else() - # # CAMP is required when using RAJA. Consider using find_package when RAJA_DIR not defined, that is BUILD_FROM_TPLMIRROR=OFF - # message(STATUS "ProxyApp requires CAMP: CAMP_DIR not set.") + # # CAMP is required when using RAJA. Consider using find_package when RAJA_DIR not defined, that is BUILD_FROM_TPLMIRROR=OFF + # message(STATUS "ProxyApp requires CAMP: CAMP_DIR not set.") # find_package(camp REQUIRED) #endif() @@ -148,21 +148,21 @@ if ( USE_RAJA) ${camp_DIR} NO_DEFAULT_PATH) else() - message(STATUS "ProxyApp requires RAJA: RAJA_DIR not set.") + message(STATUS "ProxyApp requires RAJA: RAJA_DIR not set.") find_package(camp REQUIRED) find_package(RAJA REQUIRED) endif() - message(STATUS "BUILDING SOLVER including RAJA with the following setting (after find_package raja): RAJA_ENABLE_CUDA = "${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) - + message(STATUS "BUILDING SOLVER including RAJA with the following setting (after find_package raja): RAJA_ENABLE_CUDA = " ${RAJA_ENABLE_CUDA} " and RAJA_ENABLE_OPENMP = " ${RAJA_ENABLE_OPENMP}) + get_target_property(CAMP_INCLUDE_DIRS camp INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(camp PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${CAMP_INCLUDE_DIRS}") message("-- camp_DIR: ${camp_DIR} or ${CAMP_DIR}") message("-- CAMP_INCLUDE_DIRS: ${CAMP_INCLUDE_DIRS}") set (dependencyList ${dependencyList} camp ) - + get_target_property(RAJA_INCLUDE_DIRS RAJA INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(RAJA PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${RAJA_INCLUDE_DIRS}") - message(STATUS "USE RAJA in RAJA_DIR = ${RAJA_DIR}") + message(STATUS "USE RAJA in RAJA_DIR = ${RAJA_DIR}") message("-- RAJA_INCLUDE_DIRS: ${RAJA_INCLUDE_DIRS}") if( ENABLE_HIP ) @@ -170,7 +170,7 @@ if ( USE_RAJA) set(RAJA_ENABLE_CUDA OFF CACHE BOOL "" FORCE) include_directories("${ROCM_ROOT_DIR}/include") endif() - + ################################ # Umpire ################################ @@ -184,24 +184,24 @@ if ( USE_RAJA) message(STATUS "ProxyApp requires Umpire: UMPIRE_DIR not set for finding package.") find_package(umpire REQUIRED) endif() - + get_target_property(UMPIRE_INCLUDE_DIRS umpire INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(umpire PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${UMPIRE_INCLUDE_DIRS}") message(STATUS " USE RAJA: RAJA will be compiled with UMPIRE in UMPIRE_DIR = ${UMPIRE_DIR}") - message("-- UMPIRE_INCLUDE_DIRS: ${UMPIRE_INCLUDE_DIRS} / umpire_DIR "${umpire_DIR}) + message("-- UMPIRE_INCLUDE_DIRS: ${UMPIRE_INCLUDE_DIRS} / umpire_DIR " ${umpire_DIR}) set (extraComponentsLinkList ${extraComponentsLinkList} umpire ) endif() #endif() - + ################################ # CHAI ################################ - # CHAI is not a requisite for RAJA. Umpire is a requisite for CHAI + # CHAI is not a requisite for RAJA. Umpire is a requisite for CHAI if(ENABLE_CHAI) set(ENABLE_UMPIRE ON CACHE BOOL "" FORCE) if(DEFINED CHAI_DIR) - find_package(chai REQUIRED - PATHS ${CHAI_DIR} + find_package(chai REQUIRED + PATHS ${CHAI_DIR} NO_DEFAULT_PATH) else() message(STATUS "ProxyApp requires CHAI: CHAI_DIR not set for finding package.") @@ -216,7 +216,7 @@ if ( USE_RAJA) if(ENABLE_OPENMP) set (extraComponentsLinkList ${extraComponentsLinkList} openmp ) endif() - + ################################ # CALIPER ################################ @@ -240,13 +240,13 @@ endif() if(USE_LVARRAY_EXT) # Find package if(NOT EXISTS ${LVARRAY_DIR}) - message(FATAL_ERROR "LVARRAY_DIR must be set to a defined directory to find the lvarray lib. LVARRAY_DIR is now set to "${LVARRAY_DIR}) + message(FATAL_ERROR "LVARRAY_DIR must be set to a defined directory to find the lvarray lib. LVARRAY_DIR is now set to " ${LVARRAY_DIR}) endif() find_package(LVARRAY REQUIRED) get_target_property(LVARRAY_INCLUDE_DIRS lvarray INTERFACE_INCLUDE_DIRECTORIES) set_target_properties(lvarray PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${LVARRAY_INCLUDE_DIRS}") message(STATUS "-- Find package for lvarray - LVARRAY_INCLUDE_DIRS " ${LVARRAY_INCLUDE_DIRS}) -else() +else() add_subdirectory( LvArray ) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/LvArray/src) endif() @@ -273,7 +273,7 @@ if (NOT USE_KOKKOS) else() set (USE_LVARRAY OFF CACHE BOOL "" FORCE) set (USE_VECTOR OFF CACHE BOOL "" FORCE) - + message ("-- USE_KOKKOS flag is set to be true and KOKOS is now enabled") set (SOLVER "Kokkos" CACHE PATH "" FORCE) @@ -286,11 +286,11 @@ else() find_package(Kokkos REQUIRED) endif() message("-- Found the KOKKOS library in ${KOKKOS_DIR}") - + get_target_property(KOKKOS_INCLUDE_DIRS Kokkos::kokkos INTERFACE_INCLUDE_DIRECTORIES) get_target_property(KOKKOS_LINK_LIBS Kokkos::kokkos INTERFACE_LINK_LIBRARIES) - - message(STATUS "KOKKOS_INCLUDE_DIRS: ${KOKKOS_INCLUDE_DIRS}") + + message(STATUS "KOKKOS_INCLUDE_DIRS: ${KOKKOS_INCLUDE_DIRS}") message("-- KOKKOS_LINK_LIBS: ${KOKKOS_LINK_LIBS}") #get_target_property(KOKKOSCORE_LINK_OPTION Kokkos::kokkoscore INTERFACE_LINK_OPTIONS) @@ -317,31 +317,31 @@ if(ENABLE_OPENMP OR USE_OMP) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}" CACHE STRING "" FORCE) - - + + set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} ${OpenMP_C_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${OpenMP_CXX_FLAGS}" CACHE STRING "" FORCE) - + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_RELWITHDEBINFO "${CMAKE_Fortran_FLAGS_RELWITHDEBINFO} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) set (CMAKE_Fortran_FLAGS_MINSIZEREL "${CMAKE_Fortran_FLAGS_MINSIZEREL} ${OpenMP_Fortran_FLAGS}" CACHE STRING "" FORCE) endif() -message(STATUS "-- The dependencies for building the proxy-app are: " ${dependencyList} " extraComponentsLinkLists = "${extraComponentsLinkList}) +message(STATUS "-- The dependencies for building the proxy-app are: " ${dependencyList} " extraComponentsLinkLists = " ${extraComponentsLinkList}) if((USE_KOKKOS OR USE_RAJA) AND (ENABLE_OPENMP OR ENABLE_MPI)) # SettingFlag associated to the config.. used for the name of the executable ## What programming on the host - + if(ENABLE_OPENMP) set(flag_hmodel "OMP") elseif(ENABLE_MPI) @@ -349,7 +349,7 @@ if((USE_KOKKOS OR USE_RAJA) AND (ENABLE_OPENMP OR ENABLE_MPI)) #else() #set(flag_hmodel "SERIAL") endif() - + set(SOLVER "${SOLVER}_${flag_hmodel}" CACHE PATH "" FORCE) endif() # The solver's flag @@ -370,7 +370,7 @@ configure_file( utils/commonConfig.hpp.in add_subdirectory( sem ) add_subdirectory( fd ) -if ( ENABLE_TESTS ) +if ( ENABLE_TESTS ) message(STATUS "Building tests") add_subdirectory( tests ) else() diff --git a/src/sem/CMakeLists.txt b/src/sem/CMakeLists.txt index bf5463b..5e7c52a 100644 --- a/src/sem/CMakeLists.txt +++ b/src/sem/CMakeLists.txt @@ -43,3 +43,4 @@ blt_add_executable( NAME ${SEM_EXECUTABLE} install(TARGETS ${SEM_EXECUTABLE} RUNTIME DESTINATION bin) +add_test(NAME ${SEM_EXECUTABLE} COMMAND ${SEM_EXECUTABLE}) \ No newline at end of file diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 22b8498..aae141a 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -9,9 +9,9 @@ #set (testcase Kokkos) #set (testcase Kokkos0) #set (testcase Kokkos1) -if ( USE_KOKKOS ) +if ( USE_KOKKOS AND NOT (DEVICE STREQUAL "rocm")) -LIST(APPEND TESTCASES "Kokkos" "Kokkos0" ) + LIST(APPEND TESTCASES "Kokkos" ) message(STATUS "Building tests with Kokkos: ${TESTCASES}") endif() #set (testcase FDTDKokkos) @@ -20,15 +20,18 @@ endif() #set (testcase raja_hip_daxpy) # the followings need "-DUSE_RAJA=ON -DENABLE_CUDA=ON" -#set (testcase LvArray) -#set (testcase LvArrayRAJACudaStencil) -#set (testcase LvArrayRAJACuda) -#set (testcase LvArray_cuda) -#set (testcase RAJAInline) -#set (testcase Raja_cuda) -#set (testcase FDTDRAJA) -#set (testcase raja_cuda_daxpy) - +if ( USE_RAJA ) + #set (testcase LvArray) + #set (testcase LvArrayRAJACudaStencil) + #set (testcase LvArrayRAJACuda) + #set (testcase LvArray_cuda) + #set (testcase RAJAInline) + #set (testcase Raja_cuda) + #set (testcase FDTDRAJA) + #set (testcase raja_cuda_daxpy) + LIST(APPEND TESTCASES "LvArray" "LvArrayRAJACudaStencil") + message(STATUS "Building tests with OpenMP: ${TESTCASES}") +endif(USE_RAJA) # the followings need "-DENABLE_CUDA=ON" #set (testcase cuda) #set (testcase cuda_runtime)