Init boundary exchange test

test_mom/CMakeLists.txt

@@ -17,15 +17,33 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(test_mom CXX)
+enable_language(C Fortran)
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 if(AMReX_GPU_BACKEND STREQUAL "CUDA")
     enable_language(CUDA)
 endif()
 
+# --- MPI based tests ------------------------------------------------------
+option(MPI_UNIT_TESTS "Build and run MPI based unit tests." OFF)
+
+# --- GTest ----------------------------------------------------------------
+option(TEST_MOM_FETCH_GTEST
+       "Download GoogleTest via FetchContent if find_package fails" ON)
+
+# --- Captured-data directory (used by --data-dir CLI flag) ---------------
+set(TEST_MOM_DATA_DIR ""
+    CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files")
+if(NOT TEST_MOM_DATA_DIR)
+    message(FATAL_ERROR
+        "TEST_MOM_DATA_DIR is not set. Pass\n"
+        "  -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n"
+        "on the cmake configure line.")
+endif()
+
 # --- AMReX ----------------------------------------------------------------
 # Prefer an installed AMReX if available; otherwise build the in-tree source.
 set(AMREX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../amrex"
@@ -44,10 +62,6 @@ if(NOT AMReX_FOUND)
                      ${CMAKE_CURRENT_BINARY_DIR}/_amrex)
 endif()
 
-# --- GTest ----------------------------------------------------------------
-option(TEST_MOM_FETCH_GTEST
-       "Download GoogleTest via FetchContent if find_package fails" ON)
-
 find_package(GTest QUIET)
 if(NOT GTest_FOUND)
     if(TEST_MOM_FETCH_GTEST)
@@ -69,28 +83,10 @@ endif()
 enable_testing()
 include(GoogleTest)
 
-# --- Captured-data directory (used by --data-dir CLI flag) ---------------
-set(TEST_MOM_DATA_DIR ""
-    CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files")
-if(NOT TEST_MOM_DATA_DIR)
-    message(FATAL_ERROR
-        "TEST_MOM_DATA_DIR is not set. Pass\n"
-        "  -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n"
-        "on the cmake configure line.")
-endif()
-
 # --- MOM C++ kernels library (target: mom_continuity) ---------------------
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../mom/cpp
                  ${CMAKE_CURRENT_BINARY_DIR}/mom_cpp)
 
-# --- mom_continuity_ppm test ----------------------------------------------
-add_executable(test_mom_continuity_ppm
-    test_mom_main.cpp
-    common/captured_io.cpp
-    common/amrex_assertions.cpp
-    mom_continuity_ppm/test_mom_continuity_ppm.cpp
-)
-
 # Every source in this target includes AMReX headers. With the CUDA backend
 # those headers use __host__/__device__ qualifiers and CUDA kernel-launch
 # syntax, so nvcc must compile them. CMake won't infer this from .cpp, so we
@@ -101,23 +97,18 @@ if(AMReX_GPU_BACKEND STREQUAL "CUDA")
         common/captured_io.cpp
         common/amrex_assertions.cpp
         mom_continuity_ppm/test_mom_continuity_ppm.cpp
+        boundary/test_boundary_exchange.cpp
         PROPERTIES LANGUAGE CUDA)
 endif()
 
-target_include_directories(test_mom_continuity_ppm
-    PRIVATE
-        ${CMAKE_CURRENT_SOURCE_DIR}/common
-)
-target_link_libraries(test_mom_continuity_ppm
-    PRIVATE
-        mom_continuity
-        GTest::gtest
-)
-gtest_discover_tests(test_mom_continuity_ppm
-    EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR}
-    # Discover at test-run time rather than build time. With the CUDA backend
-    # the binary calls amrex::Initialize() which aborts on hosts without a GPU
-    # (e.g. Derecho login nodes), so running it at POST_BUILD would break the
-    # build.
-    DISCOVERY_MODE PRE_TEST
-)
+include(cmake/add_mpi_test.cmake)
+add_subdirectory(common)
+
+add_library(data_driven_shared_main OBJECT test_mom_main.cpp)
+target_link_libraries(data_driven_shared_main PUBLIC common)
+
+if(MPI_UNIT_TESTS)
+  add_subdirectory(boundary)
+else()
+  add_subdirectory(mom_continuity_ppm)
+endif()

test_mom/boundary/CMakeLists.txt

@@ -0,0 +1 @@
+add_mpi_test(test_boundary_exchange test_boundary_exchange.cpp double_gyre_init_pass_var)

test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0000.meta

@@ -0,0 +1,10 @@
+G%Domain%niglobal integer 1
+G%Domain%njglobal integer 5
+G%Domain%nihalo integer 9
+G%Domain%njhalo integer 13
+G%bathyT_brefore RealArray_t 17
+G%bathyT_brefore%isd_global integer 6769
+G%bathyT_brefore%jsd_global integer 6773
+G%bathyT_after RealArray_t 6777
+G%bathyT_after%isd_global integer 13529
+G%bathyT_after%jsd_global integer 13533

test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0001.meta

@@ -0,0 +1,10 @@
+G%Domain%niglobal integer 1
+G%Domain%njglobal integer 5
+G%Domain%nihalo integer 9
+G%Domain%njhalo integer 13
+G%bathyT_brefore RealArray_t 17
+G%bathyT_brefore%isd_global integer 6769
+G%bathyT_brefore%jsd_global integer 6773
+G%bathyT_after RealArray_t 6777
+G%bathyT_after%isd_global integer 13529
+G%bathyT_after%jsd_global integer 13533

test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0002.meta

@@ -0,0 +1,10 @@
+G%Domain%niglobal integer 1
+G%Domain%njglobal integer 5
+G%Domain%nihalo integer 9
+G%Domain%njhalo integer 13
+G%bathyT_brefore RealArray_t 17
+G%bathyT_brefore%isd_global integer 6769
+G%bathyT_brefore%jsd_global integer 6773
+G%bathyT_after RealArray_t 6777
+G%bathyT_after%isd_global integer 13529
+G%bathyT_after%jsd_global integer 13533

test_mom/boundary/double_gyre_init_pass_var/MOM_fixed_initialization_0003.meta

@@ -0,0 +1,10 @@
+G%Domain%niglobal integer 1
+G%Domain%njglobal integer 5
+G%Domain%nihalo integer 9
+G%Domain%njhalo integer 13
+G%bathyT_brefore RealArray_t 17
+G%bathyT_brefore%isd_global integer 6769
+G%bathyT_brefore%jsd_global integer 6773
+G%bathyT_after RealArray_t 6777
+G%bathyT_after%isd_global integer 13529
+G%bathyT_after%jsd_global integer 13533

test_mom/boundary/test_boundary_exchange.cpp

@@ -0,0 +1,84 @@
+#include <format>
+
+#include <gtest/gtest.h>
+
+#include <AMReX_FArrayBox.H>
+#include <AMReX_MultiFab.H>
+#include <AMReX_Gpu.H>
+#include <AMReX_ParallelDescriptor.H>
+
+#include "amrex_assertions.hpp"
+#include "captured_io.hpp"
+#include "data_dir.hpp"
+
+using test_mom::expect_arrays_equal;
+
+TEST(BoundaryExchange, BoundaryMatchesAfterOneTimeStep) {
+    int myRank = amrex::ParallelDescriptor::MyProc();
+    test_mom::CapturedFile captured(test_mom::data_dir /
+                                    ("MOM_fixed_initialization_" + std::format("{:04}", myRank)));
+
+    auto bathyTBefore = captured.fab_host("G%bathyT_brefore");
+    auto bathyTAfter  = captured.fab_host("G%bathyT_after"  );
+    int  nihalo       = captured.integer("G%Domain%nihalo");
+    int  njhalo       = captured.integer("G%Domain%njhalo");
+    int  niglobal     = captured.integer("G%Domain%niglobal");
+    int  njglobal     = captured.integer("G%Domain%njglobal");
+
+    amrex::BoxList     boxList;
+    amrex::Vector<int> processBoxList;
+    int iLocalDataGridSize = bathyTBefore.box().length(0) - (2*nihalo);
+    int jLocalDataGridSize = bathyTBefore.box().length(1) - (2*njhalo);
+
+    // Assumes equally sized grids
+    for(int i = 0; i < amrex::ParallelDescriptor::NProcs(); i++)
+    {
+        test_mom::CapturedFile rankc(test_mom::data_dir /
+                                    ("MOM_fixed_initialization_" + std::format("{:04}", i)));
+
+        int iGridStartLoc = rankc.integer("G%bathyT_brefore%isd_global") + nihalo;
+        int jGridStartLoc = rankc.integer("G%bathyT_brefore%jsd_global") + njhalo;
+        amrex::IntVect lo(iGridStartLoc, jGridStartLoc, 0);
+        amrex::IntVect hi(lo[0] + iLocalDataGridSize - 1, lo[1] + jLocalDataGridSize - 1, 0);
+
+        boxList.push_back(amrex::Box(lo, hi));
+        processBoxList.push_back(i);
+    }
+
+    // Setup multifab
+    amrex::BoxArray            boxes(boxList);
+    amrex::DistributionMapping dm(processBoxList);
+    amrex::IntVect             fourRowTwo2DGhostCells{nihalo, njhalo, 0};
+    amrex::MultiFab            mf(boxes, dm, /*ncomp=*/1, fourRowTwo2DGhostCells);
+
+    // Setup geometry
+    amrex::Box globalComputationalDomain(amrex::IntVect(           0,            0, 0),
+                                         amrex::IntVect(niglobal - 1, njglobal - 1, 0));
+    amrex::RealBox  realBox({0.0, 0.0, 0.0}, {1.0, 1.0, 1.0});
+    amrex::Geometry geom(globalComputationalDomain, &realBox);
+
+    // Verify that the loops iterate only the expected number of times
+    int mf_iterations = 0;
+    for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi)
+    {
+        amrex::FArrayBox& localValidFab  = mf[mfi];
+        ASSERT_EQ(bathyTBefore.box(), localValidFab.box());
+
+        localValidFab.copy(bathyTBefore);
+        mf_iterations++;
+    }
+    ASSERT_EQ(1, mf_iterations);
+
+    mf.FillBoundary(geom.periodicity());
+
+    mf_iterations = 0;
+    for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi)
+    {
+        amrex::FArrayBox& localValidFab  = mf[mfi];
+        ASSERT_EQ(bathyTAfter.box(), localValidFab.box());
+
+        expect_arrays_equal(bathyTAfter, localValidFab, "G%bathyT");
+        mf_iterations++;
+    }
+    ASSERT_EQ(1, mf_iterations);
+}

test_mom/cmake/add_mpi_test.cmake

@@ -0,0 +1,27 @@
+
+function(add_mpi_test TEST_TARGET_NAME TEST_FILE DATA_DIR)
+
+  set(prefix TIM)
+  set(optionalValues "")
+  set(singleValues MAX_PES)
+  set(multiValues "")
+
+  include(CMakeParseArguments)
+  cmake_parse_arguments(${prefix} "${optionalValues}" "${singleValues}" "${multiValues}" ${ARGN})
+
+  if(DEFINED TIM_MAX_PES)
+    if (TIM_MAX_PES NOT MATCHES "^[1-9][0-9]*$")
+      message(FATAL_ERROR "MAX_PES expected a valid positive integer; recieved ${TIM_MAX_PES}")
+    endif()
+  else()
+    set(TIM_MAX_PES 4)
+  endif()
+
+  add_executable(${TEST_TARGET_NAME} ${TEST_FILE})
+  target_sources(${TEST_TARGET_NAME} PRIVATE $<TARGET_OBJECTS:data_driven_shared_main>)
+  target_link_libraries(${TEST_TARGET_NAME} PRIVATE common)
+
+  add_test(NAME MPI_${TEST_TARGET_NAME}
+         COMMAND ${MPIEXEC_EXECUTABLE} --tag-output ${MPIEXEC_NUMPROC_FLAG} ${TIM_MAX_PES} ./${TEST_TARGET_NAME} --data-dir=${CMAKE_CURRENT_SOURCE_DIR}/${DATA_DIR})
+  set_tests_properties(MPI_${TEST_TARGET_NAME} PROPERTIES PROCESSORS ${TIM_MAX_PES})
+endfunction()

test_mom/common/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(COMMON_SRC
+    amrex_assertions.cpp
+    captured_io.cpp)
+
+add_library(common OBJECT ${COMMON_SRC})
+
+target_link_libraries(common PUBLIC GTest::gtest AMReX::amrex)
+target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})

test_mom/common/captured_io.cpp

@@ -154,14 +154,21 @@ amrex::FArrayBox CapturedFile::fab_host(const std::string& name) const {
             std::to_string(nelem) + ", expected " + std::to_string(expected) +
             ")");
     }
-    amrex::Box sbx(amrex::IntVect(0, 0, 0),
-                   amrex::IntVect(shape[0] - 1, shape[1] - 1, shape[2] - 1));
+    amrex::IntVect start(0, 0, 0);
+    if(entries_.contains(name + "%isd_global") && entries_.contains(name + "%jsd_global"))
+    {
+        int isd_global = this->integer(name + "%isd_global");
+        int jsd_global = this->integer(name + "%jsd_global");
+        start = amrex::IntVect(isd_global, jsd_global, 0);
+    }
+    amrex::IntVect end(start[0] + shape[0] - 1, start[1] + shape[1] -1, shape[2] - 1);
+    amrex::Box sbx(start, end);
     amrex::FArrayBox fab(sbx, 1, amrex::The_Pinned_Arena());
     auto arr = fab.array();
     // Fortran column-major in the file: i fastest, then j, then k.
-    for (int k = 0; k < shape[2]; ++k)
-        for (int j = 0; j < shape[1]; ++j)
-            for (int i = 0; i < shape[0]; ++i)
+    for (int k = start[2]; k < end[2]; ++k)
+        for (int j = start[1]; j < end[1]; ++j)
+            for (int i = start[0]; i < end[0]; ++i)
                 arr(i, j, k) = read_be_f64(bin_, off);
     return fab;
 }

test_mom/mom_continuity_ppm/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_executable(test_mom_continuity_ppm test_mom_continuity_ppm.cpp)
+target_sources(test_mom_continuity_ppm PRIVATE $<TARGET_OBJECTS:data_driven_shared_main>)
+target_link_libraries(test_mom_continuity_ppm PRIVATE common mom_continuity)
+gtest_discover_tests(test_mom_continuity_ppm
+    EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR}
+    # Discover at test-run time rather than build time. With the CUDA backend
+    # the binary calls amrex::Initialize() which aborts on hosts without a GPU
+    # (e.g. Derecho login nodes), so running it at POST_BUILD would break the
+    # build.
+    DISCOVERY_MODE PRE_TEST
+)