diff --git a/test_mom/CMakeLists.txt b/test_mom/CMakeLists.txt index 82cd7df..c9b23e6 100644 --- a/test_mom/CMakeLists.txt +++ b/test_mom/CMakeLists.txt @@ -17,8 +17,9 @@ cmake_minimum_required(VERSION 3.20) project(test_mom CXX) +enable_language(C Fortran) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -26,6 +27,23 @@ if(AMReX_GPU_BACKEND STREQUAL "CUDA") enable_language(CUDA) endif() +# --- MPI based tests ------------------------------------------------------ +option(MPI_UNIT_TESTS "Build and run MPI based unit tests." OFF) + +# --- GTest ---------------------------------------------------------------- +option(TEST_MOM_FETCH_GTEST + "Download GoogleTest via FetchContent if find_package fails" ON) + +# --- Captured-data directory (used by --data-dir CLI flag) --------------- +set(TEST_MOM_DATA_DIR "" + CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files") +if(NOT TEST_MOM_DATA_DIR) + message(FATAL_ERROR + "TEST_MOM_DATA_DIR is not set. Pass\n" + " -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n" + "on the cmake configure line.") +endif() + # --- AMReX ---------------------------------------------------------------- # Prefer an installed AMReX if available; otherwise build the in-tree source. set(AMREX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../amrex" @@ -44,10 +62,6 @@ if(NOT AMReX_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/_amrex) endif() -# --- GTest ---------------------------------------------------------------- -option(TEST_MOM_FETCH_GTEST - "Download GoogleTest via FetchContent if find_package fails" ON) - find_package(GTest QUIET) if(NOT GTest_FOUND) if(TEST_MOM_FETCH_GTEST) @@ -69,28 +83,10 @@ endif() enable_testing() include(GoogleTest) -# --- Captured-data directory (used by --data-dir CLI flag) --------------- -set(TEST_MOM_DATA_DIR "" - CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files") -if(NOT TEST_MOM_DATA_DIR) - message(FATAL_ERROR - "TEST_MOM_DATA_DIR is not set. Pass\n" - " -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n" - "on the cmake configure line.") -endif() - # --- MOM C++ kernels library (target: mom_continuity) --------------------- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../mom/cpp ${CMAKE_CURRENT_BINARY_DIR}/mom_cpp) -# --- mom_continuity_ppm test ---------------------------------------------- -add_executable(test_mom_continuity_ppm - test_mom_main.cpp - common/captured_io.cpp - common/amrex_assertions.cpp - mom_continuity_ppm/test_mom_continuity_ppm.cpp -) - # Every source in this target includes AMReX headers. With the CUDA backend # those headers use __host__/__device__ qualifiers and CUDA kernel-launch # syntax, so nvcc must compile them. CMake won't infer this from .cpp, so we @@ -101,23 +97,18 @@ if(AMReX_GPU_BACKEND STREQUAL "CUDA") common/captured_io.cpp common/amrex_assertions.cpp mom_continuity_ppm/test_mom_continuity_ppm.cpp + boundary/test_boundary_exchange.cpp PROPERTIES LANGUAGE CUDA) endif() -target_include_directories(test_mom_continuity_ppm - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/common -) -target_link_libraries(test_mom_continuity_ppm - PRIVATE - mom_continuity - GTest::gtest -) -gtest_discover_tests(test_mom_continuity_ppm - EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR} - # Discover at test-run time rather than build time. With the CUDA backend - # the binary calls amrex::Initialize() which aborts on hosts without a GPU - # (e.g. Derecho login nodes), so running it at POST_BUILD would break the - # build. - DISCOVERY_MODE PRE_TEST -) +include(cmake/add_mpi_test.cmake) +add_subdirectory(common) + +add_library(data_driven_shared_main OBJECT test_mom_main.cpp) +target_link_libraries(data_driven_shared_main PUBLIC common) + +if(MPI_UNIT_TESTS) + add_subdirectory(boundary) +else() + add_subdirectory(mom_continuity_ppm) +endif() diff --git a/test_mom/boundary/CMakeLists.txt b/test_mom/boundary/CMakeLists.txt new file mode 100644 index 0000000..8d705f5 --- /dev/null +++ b/test_mom/boundary/CMakeLists.txt @@ -0,0 +1 @@ +add_mpi_test(test_boundary_exchange test_boundary_exchange.cpp double_gyre_init_pass_var) diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.bin b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.bin new file mode 100644 index 0000000..1d2fb8c Binary files /dev/null and b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.bin differ diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.meta b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.meta new file mode 100644 index 0000000..0107984 --- /dev/null +++ b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.meta @@ -0,0 +1,10 @@ +G%Domain%niglobal integer 1 +G%Domain%njglobal integer 5 +G%Domain%nihalo integer 9 +G%Domain%njhalo integer 13 +G%bathyT_brefore RealArray_t 17 +G%bathyT_brefore%isd_global integer 6769 +G%bathyT_brefore%jsd_global integer 6773 +G%bathyT_after RealArray_t 6777 +G%bathyT_after%isd_global integer 13529 +G%bathyT_after%jsd_global integer 13533 diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.bin b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.bin new file mode 100644 index 0000000..683b7e2 Binary files /dev/null and b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.bin differ diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.meta b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.meta new file mode 100644 index 0000000..0107984 --- /dev/null +++ b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.meta @@ -0,0 +1,10 @@ +G%Domain%niglobal integer 1 +G%Domain%njglobal integer 5 +G%Domain%nihalo integer 9 +G%Domain%njhalo integer 13 +G%bathyT_brefore RealArray_t 17 +G%bathyT_brefore%isd_global integer 6769 +G%bathyT_brefore%jsd_global integer 6773 +G%bathyT_after RealArray_t 6777 +G%bathyT_after%isd_global integer 13529 +G%bathyT_after%jsd_global integer 13533 diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.bin b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.bin new file mode 100644 index 0000000..fc73b62 Binary files /dev/null and b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.bin differ diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.meta b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.meta new file mode 100644 index 0000000..0107984 --- /dev/null +++ b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.meta @@ -0,0 +1,10 @@ +G%Domain%niglobal integer 1 +G%Domain%njglobal integer 5 +G%Domain%nihalo integer 9 +G%Domain%njhalo integer 13 +G%bathyT_brefore RealArray_t 17 +G%bathyT_brefore%isd_global integer 6769 +G%bathyT_brefore%jsd_global integer 6773 +G%bathyT_after RealArray_t 6777 +G%bathyT_after%isd_global integer 13529 +G%bathyT_after%jsd_global integer 13533 diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.bin b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.bin new file mode 100644 index 0000000..bed18ef Binary files /dev/null and b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.bin differ diff --git a/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.meta b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.meta new file mode 100644 index 0000000..0107984 --- /dev/null +++ b/test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.meta @@ -0,0 +1,10 @@ +G%Domain%niglobal integer 1 +G%Domain%njglobal integer 5 +G%Domain%nihalo integer 9 +G%Domain%njhalo integer 13 +G%bathyT_brefore RealArray_t 17 +G%bathyT_brefore%isd_global integer 6769 +G%bathyT_brefore%jsd_global integer 6773 +G%bathyT_after RealArray_t 6777 +G%bathyT_after%isd_global integer 13529 +G%bathyT_after%jsd_global integer 13533 diff --git a/test_mom/boundary/test_boundary_exchange.cpp b/test_mom/boundary/test_boundary_exchange.cpp new file mode 100644 index 0000000..86937fb --- /dev/null +++ b/test_mom/boundary/test_boundary_exchange.cpp @@ -0,0 +1,84 @@ +#include + +#include + +#include +#include +#include +#include + +#include "amrex_assertions.hpp" +#include "captured_io.hpp" +#include "data_dir.hpp" + +using test_mom::expect_arrays_equal; + +TEST(BoundaryExchange, BoundaryMatchesAfterOneTimeStep) { + int myRank = amrex::ParallelDescriptor::MyProc(); + test_mom::CapturedFile captured(test_mom::data_dir / + ("MOM_fixed_initialization_" + std::format("{:04}", myRank))); + + auto bathyTBefore = captured.fab_host("G%bathyT_brefore"); + auto bathyTAfter = captured.fab_host("G%bathyT_after" ); + int nihalo = captured.integer("G%Domain%nihalo"); + int njhalo = captured.integer("G%Domain%njhalo"); + int niglobal = captured.integer("G%Domain%niglobal"); + int njglobal = captured.integer("G%Domain%njglobal"); + + amrex::BoxList boxList; + amrex::Vector processBoxList; + int iLocalDataGridSize = bathyTBefore.box().length(0) - (2*nihalo); + int jLocalDataGridSize = bathyTBefore.box().length(1) - (2*njhalo); + + // Assumes equally sized grids + for(int i = 0; i < amrex::ParallelDescriptor::NProcs(); i++) + { + test_mom::CapturedFile rankc(test_mom::data_dir / + ("MOM_fixed_initialization_" + std::format("{:04}", i))); + + int iGridStartLoc = rankc.integer("G%bathyT_brefore%isd_global") + nihalo; + int jGridStartLoc = rankc.integer("G%bathyT_brefore%jsd_global") + njhalo; + amrex::IntVect lo(iGridStartLoc, jGridStartLoc, 0); + amrex::IntVect hi(lo[0] + iLocalDataGridSize - 1, lo[1] + jLocalDataGridSize - 1, 0); + + boxList.push_back(amrex::Box(lo, hi)); + processBoxList.push_back(i); + } + + // Setup multifab + amrex::BoxArray boxes(boxList); + amrex::DistributionMapping dm(processBoxList); + amrex::IntVect fourRowTwo2DGhostCells{nihalo, njhalo, 0}; + amrex::MultiFab mf(boxes, dm, /*ncomp=*/1, fourRowTwo2DGhostCells); + + // Setup geometry + amrex::Box globalComputationalDomain(amrex::IntVect( 0, 0, 0), + amrex::IntVect(niglobal - 1, njglobal - 1, 0)); + amrex::RealBox realBox({0.0, 0.0, 0.0}, {1.0, 1.0, 1.0}); + amrex::Geometry geom(globalComputationalDomain, &realBox); + + // Verify that the loops iterate only the expected number of times + int mf_iterations = 0; + for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi) + { + amrex::FArrayBox& localValidFab = mf[mfi]; + ASSERT_EQ(bathyTBefore.box(), localValidFab.box()); + + localValidFab.copy(bathyTBefore); + mf_iterations++; + } + ASSERT_EQ(1, mf_iterations); + + mf.FillBoundary(geom.periodicity()); + + mf_iterations = 0; + for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi) + { + amrex::FArrayBox& localValidFab = mf[mfi]; + ASSERT_EQ(bathyTAfter.box(), localValidFab.box()); + + expect_arrays_equal(bathyTAfter, localValidFab, "G%bathyT"); + mf_iterations++; + } + ASSERT_EQ(1, mf_iterations); +} diff --git a/test_mom/cmake/add_mpi_test.cmake b/test_mom/cmake/add_mpi_test.cmake new file mode 100644 index 0000000..5f594a4 --- /dev/null +++ b/test_mom/cmake/add_mpi_test.cmake @@ -0,0 +1,27 @@ + +function(add_mpi_test TEST_TARGET_NAME TEST_FILE DATA_DIR) + + set(prefix TIM) + set(optionalValues "") + set(singleValues MAX_PES) + set(multiValues "") + + include(CMakeParseArguments) + cmake_parse_arguments(${prefix} "${optionalValues}" "${singleValues}" "${multiValues}" ${ARGN}) + + if(DEFINED TIM_MAX_PES) + if (TIM_MAX_PES NOT MATCHES "^[1-9][0-9]*$") + message(FATAL_ERROR "MAX_PES expected a valid positive integer; recieved ${TIM_MAX_PES}") + endif() + else() + set(TIM_MAX_PES 4) + endif() + + add_executable(${TEST_TARGET_NAME} ${TEST_FILE}) + target_sources(${TEST_TARGET_NAME} PRIVATE $) + target_link_libraries(${TEST_TARGET_NAME} PRIVATE common) + + add_test(NAME MPI_${TEST_TARGET_NAME} + COMMAND ${MPIEXEC_EXECUTABLE} --tag-output ${MPIEXEC_NUMPROC_FLAG} ${TIM_MAX_PES} ./${TEST_TARGET_NAME} --data-dir=${CMAKE_CURRENT_SOURCE_DIR}/${DATA_DIR}) + set_tests_properties(MPI_${TEST_TARGET_NAME} PROPERTIES PROCESSORS ${TIM_MAX_PES}) +endfunction() diff --git a/test_mom/common/CMakeLists.txt b/test_mom/common/CMakeLists.txt new file mode 100644 index 0000000..dde066c --- /dev/null +++ b/test_mom/common/CMakeLists.txt @@ -0,0 +1,8 @@ +set(COMMON_SRC + amrex_assertions.cpp + captured_io.cpp) + +add_library(common OBJECT ${COMMON_SRC}) + +target_link_libraries(common PUBLIC GTest::gtest AMReX::amrex) +target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/test_mom/common/captured_io.cpp b/test_mom/common/captured_io.cpp index 617e4fc..871b43e 100644 --- a/test_mom/common/captured_io.cpp +++ b/test_mom/common/captured_io.cpp @@ -154,14 +154,21 @@ amrex::FArrayBox CapturedFile::fab_host(const std::string& name) const { std::to_string(nelem) + ", expected " + std::to_string(expected) + ")"); } - amrex::Box sbx(amrex::IntVect(0, 0, 0), - amrex::IntVect(shape[0] - 1, shape[1] - 1, shape[2] - 1)); + amrex::IntVect start(0, 0, 0); + if(entries_.contains(name + "%isd_global") && entries_.contains(name + "%jsd_global")) + { + int isd_global = this->integer(name + "%isd_global"); + int jsd_global = this->integer(name + "%jsd_global"); + start = amrex::IntVect(isd_global, jsd_global, 0); + } + amrex::IntVect end(start[0] + shape[0] - 1, start[1] + shape[1] -1, shape[2] - 1); + amrex::Box sbx(start, end); amrex::FArrayBox fab(sbx, 1, amrex::The_Pinned_Arena()); auto arr = fab.array(); // Fortran column-major in the file: i fastest, then j, then k. - for (int k = 0; k < shape[2]; ++k) - for (int j = 0; j < shape[1]; ++j) - for (int i = 0; i < shape[0]; ++i) + for (int k = start[2]; k < end[2]; ++k) + for (int j = start[1]; j < end[1]; ++j) + for (int i = start[0]; i < end[0]; ++i) arr(i, j, k) = read_be_f64(bin_, off); return fab; } diff --git a/test_mom/mom_continuity_ppm/CMakeLists.txt b/test_mom/mom_continuity_ppm/CMakeLists.txt new file mode 100644 index 0000000..3aafe70 --- /dev/null +++ b/test_mom/mom_continuity_ppm/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable(test_mom_continuity_ppm test_mom_continuity_ppm.cpp) +target_sources(test_mom_continuity_ppm PRIVATE $) +target_link_libraries(test_mom_continuity_ppm PRIVATE common mom_continuity) +gtest_discover_tests(test_mom_continuity_ppm + EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR} + # Discover at test-run time rather than build time. With the CUDA backend + # the binary calls amrex::Initialize() which aborts on hosts without a GPU + # (e.g. Derecho login nodes), so running it at POST_BUILD would break the + # build. + DISCOVERY_MODE PRE_TEST +)