diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 5186891ce8e..b06ddc31896 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -1036,6 +1036,7 @@ list (APPEND PUBLIC_HEADER_FILES opm/simulators/flow/HybridNewtonConfig.hpp opm/simulators/flow/InterRegFlows.hpp opm/simulators/flow/KeywordValidation.hpp + opm/simulators/flow/LgrOutputTransGather.hpp opm/simulators/flow/LogOutputHelper.hpp opm/simulators/flow/Main.hpp opm/simulators/flow/MechContainer.hpp diff --git a/compareECLFiles.cmake b/compareECLFiles.cmake index 3e4103826ae..0841d569a91 100644 --- a/compareECLFiles.cmake +++ b/compareECLFiles.cmake @@ -289,6 +289,7 @@ function(add_test_compare_parallel_simulation) DIR POSTFIX MPI_PROCS + COMPARE_MODE ) set(multiValueArgs TEST_ARGS) cmake_parse_arguments(PARAM "$" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -313,6 +314,17 @@ function(add_test_compare_parallel_simulation) set(PARAM_SIMULATOR ${PARAM_DEV_SIMULATOR}) endif() + # COMPARE_MODE "init": dry-run, compares EGRID+INIT only (parallel INIT-file regressions, + # e.g. parallel LGR transmissibility). Default (unset): compares SMRY+UNRST as before. + if(PARAM_COMPARE_MODE STREQUAL "init") + set(TEST_NAME_PREFIX compareParallelInitSim_) + elseif(PARAM_COMPARE_MODE) + message(FATAL_ERROR "add_test_compare_parallel_simulation(${PARAM_CASENAME}): " + "unknown COMPARE_MODE '${PARAM_COMPARE_MODE}' (expected 'init' or unset)") + else() + set(TEST_NAME_PREFIX compareParallelSim_) + endif() + if(MPIEXEC_MAX_NUMPROCS GREATER_EQUAL MPI_PROCS) # Local computer system has at least ${MPI_PROCS} CPUs. Register test. set(RESULT_PATH ${BASE_RESULT_PATH}/parallel/${PARAM_SIMULATOR}+${PARAM_CASENAME}) @@ -325,9 +337,12 @@ function(add_test_compare_parallel_simulation) -t ${PARAM_REL_TOL} -c $ -n ${MPI_PROCS}) + if(PARAM_COMPARE_MODE STREQUAL "init") + list(APPEND DRIVER_ARGS -m init) + endif() # Add test that runs flow_mpi and outputs the results to file - opm_add_test(compareParallelSim_${PARAM_SIMULATOR}+${PARAM_FILENAME}${PARAM_POSTFIX} + opm_add_test(${TEST_NAME_PREFIX}${PARAM_SIMULATOR}+${PARAM_FILENAME}${PARAM_POSTFIX} EXE_TARGET ${PARAM_SIMULATOR} DRIVER_ARGS @@ -335,7 +350,7 @@ function(add_test_compare_parallel_simulation) TEST_ARGS ${TEST_ARGS} ) - set_tests_properties(compareParallelSim_${PARAM_SIMULATOR}+${PARAM_FILENAME}${PARAM_POSTFIX} + set_tests_properties(${TEST_NAME_PREFIX}${PARAM_SIMULATOR}+${PARAM_FILENAME}${PARAM_POSTFIX} PROPERTIES PROCESSORS ${MPI_PROCS}) endif() endfunction() diff --git a/opm/simulators/flow/EclGenericWriter.hpp b/opm/simulators/flow/EclGenericWriter.hpp index f91798d312e..077f7f702dd 100644 --- a/opm/simulators/flow/EclGenericWriter.hpp +++ b/opm/simulators/flow/EclGenericWriter.hpp @@ -33,9 +33,11 @@ #include #include +#include #include #include +#include #include #include #include @@ -93,6 +95,15 @@ class EclGenericWriter globalTrans_ = globalTrans; } + // Parallel LGR INIT output: connection transmissibilities gathered from the + // ranks' own (distributed) simulator transmissibilities. When set, + // computeTrans_ / exportNncStructure_ take values from these instead of + // querying a whole-grid transmissibility object. Complete on the I/O rank only. + void setGatheredLgrTrans(std::optional gatheredTrans) + { + gatheredLgrTrans_ = std::move(gatheredTrans); + } + void setSubStepReport(const SimulatorReportSingle& report) { sub_step_report_ = report; @@ -118,6 +129,21 @@ class EclGenericWriter const TransmissibilityType& globalTrans() const; unsigned int gridEquilIdxToGridIdx(unsigned int elemIndex) const; + // Value for `key` from the gathered records, or nullptr when the key is + // absent or no gathered records are set. N = 3 selects the same-level + // records, N = 4 the level-crossing (NNC) ones -- see GatheredLgrOutputTrans. + template + const double* findGatheredTrans_(const std::array& key) const; + + // Output transmissibility value for a connection: from the gathered per-rank + // simulator transmissibilities when set (parallel LGR runs; a missing key is + // a hard error), otherwise from the whole-grid transmissibility object + // (c1, c2). Key shape as in findGatheredTrans_. + template + double gatheredOrGlobalTrans_(const std::array& key, + unsigned c1, + unsigned c2) const; + void doWriteOutput(const int reportStepNum, const std::optional timeStepNum, const bool isSubStep, @@ -165,6 +191,7 @@ class EclGenericWriter std::unique_ptr taskletRunner_; Scalar restartTimeStepSize_; const TransmissibilityType* globalTrans_ = nullptr; + std::optional gatheredLgrTrans_; const Dune::CartesianIndexMapper& cartMapper_; const Dune::CartesianIndexMapper* equilCartMapper_; const EquilGrid* equilGrid_; diff --git a/opm/simulators/flow/EclGenericWriter_impl.hpp b/opm/simulators/flow/EclGenericWriter_impl.hpp index dc178d77acb..f2647054a17 100644 --- a/opm/simulators/flow/EclGenericWriter_impl.hpp +++ b/opm/simulators/flow/EclGenericWriter_impl.hpp @@ -51,6 +51,8 @@ #include +#include + #if HAVE_MPI #include #endif @@ -66,6 +68,7 @@ #include #include #include +#include #include #include #include @@ -295,6 +298,7 @@ writeInit() } this->outputTrans_.reset(); } + this->gatheredLgrTrans_.reset(); } template @@ -302,6 +306,12 @@ void EclGenericWriter:: extractOutputTransAndNNC(const std::function& map) { + // The work below runs on the I/O rank only, but an exception there (e.g. a + // missing gathered LGR transmissibility) must not leave the other ranks + // blocked in the broadcast that follows -- so failure is agreed on + // collectively and every rank throws together. + OPM_BEGIN_PARALLEL_TRY_CATCH() + if (collectOnIORank_.isIORank()) { constexpr bool equilGridIsCpGrid = std::is_same_v; @@ -318,6 +328,9 @@ extractOutputTransAndNNC(const std::function& map) computeLevelCartDimensions, computeOriginIndices); } + OPM_END_PARALLEL_TRY_CATCH("EclGenericWriter::extractOutputTransAndNNC() failed: ", + grid_.comm()); + #if HAVE_MPI if (collectOnIORank_.isParallel()) { const auto& comm = grid_.comm(); @@ -574,12 +587,14 @@ computeTrans_(const std::vector>& levelCartToLevelC } if (maxLevelCartIdx - minLevelCartIdx == 1 && levelCartDims[0] > 1 ) { - outputTrans_->at(level).at("TRANX").template data()[minLevelCartIdx] = globalTrans().transmissibility(c1, c2); + outputTrans_->at(level).at("TRANX").template data()[minLevelCartIdx] = + gatheredOrGlobalTrans_(std::array{level, minLevelCartIdx, maxLevelCartIdx}, c1, c2); continue; // skip other if clauses as they are false, last one needs some computation } if (maxLevelCartIdx - minLevelCartIdx == levelCartDims[0] && levelCartDims[1] > 1) { - outputTrans_->at(level).at("TRANY").template data()[minLevelCartIdx] = globalTrans().transmissibility(c1, c2); + outputTrans_->at(level).at("TRANY").template data()[minLevelCartIdx] = + gatheredOrGlobalTrans_(std::array{level, minLevelCartIdx, maxLevelCartIdx}, c1, c2); continue; // skipt next if clause as it needs some computation } @@ -588,7 +603,8 @@ computeTrans_(const std::vector>& levelCartToLevelC levelCartToLevelCompressed[level], minLevelCartIdx, maxLevelCartIdx)) { - outputTrans_->at(level).at("TRANZ").template data()[minLevelCartIdx] = globalTrans().transmissibility(c1, c2); + outputTrans_->at(level).at("TRANZ").template data()[minLevelCartIdx] = + gatheredOrGlobalTrans_(std::array{level, minLevelCartIdx, maxLevelCartIdx}, c1, c2); } } } @@ -720,7 +736,9 @@ exportNncStructure_(const std::vector>& levelCartToL const auto& [smallerLevel, smallerLevelCartIdx] = smallerPair; const auto& [largerLevel, largerLevelCartIdx] = largerPair; - auto t = this->globalTrans().transmissibility(c1, c2); + auto t = this->gatheredOrGlobalTrans_(std::array{smallerLevel, smallerLevelCartIdx, + largerLevel, largerLevelCartIdx}, + c1, c2); // ECLIPSE ignores NNCs with zero transmissibility // (different threshold than for NNC with corresponding @@ -781,7 +799,9 @@ exportNncStructure_(const std::vector>& levelCartToL levelCartIdxIn, levelCartIdxOut)) { // We need to check whether an NNC for this face was also // specified via the NNC keyword in the deck. - auto t = this->globalTrans().transmissibility(c1, c2); + // (levelCartIdxIn/Out are already swapped into min/max order above.) + auto t = this->gatheredOrGlobalTrans_(std::array{level, levelCartIdxIn, levelCartIdxOut}, + c1, c2); if (level == 0) { auto candidate = std::lower_bound(nncData.begin(), nncData.end(), @@ -879,13 +899,22 @@ exportNncStructure_(const std::vector>& levelCartToL continue; } - trans = this->globalTrans().transmissibility(c1, c2); - - if (! generatedNnc.empty()) { - for (const auto& generated : generatedNnc) { - if (entry.cell1 == generated.cell1 && entry.cell2 == generated.cell2) { - trans -= generated.trans; - break; + // A deck NNC names a pair of level-zero cells. When one of them is + // refined by an LGR the pair is no longer a leaf connection, so the + // gathered records (correctly) hold no value for it -- keep the + // deck-specified transmissibility in that case. + const auto key = std::array{0, static_cast(entry.cell1), + static_cast(entry.cell2)}; + if (!this->gatheredLgrTrans_.has_value() || + this->findGatheredTrans_(key) != nullptr) { + trans = this->gatheredOrGlobalTrans_(key, c1, c2); + + if (! generatedNnc.empty()) { + for (const auto& generated : generatedNnc) { + if (entry.cell1 == generated.cell1 && entry.cell2 == generated.cell2) { + trans -= generated.trans; + break; + } } } } @@ -1101,6 +1130,57 @@ evalSummary(const int reportStepNum, #endif } +template +template +const double* +EclGenericWriter:: +findGatheredTrans_(const std::array& key) const +{ + static_assert(N == 3 || N == 4, "unknown gathered LGR record shape"); + + if (!gatheredLgrTrans_.has_value()) { + return nullptr; + } + + const auto lookup = [&key](const auto& records) -> const double* { + const auto candidate = std::lower_bound(records.begin(), records.end(), key, + [](const auto& record, const auto& k) + { return record.first < k; }); + return (candidate != records.end() && candidate->first == key) + ? &candidate->second : nullptr; + }; + + if constexpr (N == 3) { + return lookup(gatheredLgrTrans_->sameLevel); + } else { + return lookup(gatheredLgrTrans_->crossLevel); + } +} + +template +template +double +EclGenericWriter:: +gatheredOrGlobalTrans_(const std::array& key, + unsigned c1, + unsigned c2) const +{ + if (!gatheredLgrTrans_.has_value()) { + return this->globalTrans().transmissibility(c1, c2); + } + + if (const double* value = this->findGatheredTrans_(key)) { + return *value; + } + + std::string msg = "Gathered LGR transmissibilities: no value for connection key ("; + for (std::size_t j = 0; j < N; ++j) { + msg += std::to_string(key[j]); + msg += (j + 1 < N) ? ", " : ")"; + } + throw std::logic_error { msg }; +} + template const typename EclGenericWriter::TransmissibilityType& EclGenericWriter:: diff --git a/opm/simulators/flow/FlowProblemBlackoil.hpp b/opm/simulators/flow/FlowProblemBlackoil.hpp index bb5a4348a2a..0c2f16e3a6e 100644 --- a/opm/simulators/flow/FlowProblemBlackoil.hpp +++ b/opm/simulators/flow/FlowProblemBlackoil.hpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ #include #include #include +#include #include namespace Opm { @@ -347,8 +349,43 @@ class FlowProblemBlackoil : public FlowProblem // we try to avoid for the parallel running, has both global trans_ and transmissibilities_ allocated at the same time if (enableEclOutput_) { if (simulator.vanguard().grid().comm().size() > 1) { - if (simulator.vanguard().grid().comm().rank() == 0) + bool wholeGridTransNeeded = simulator.vanguard().grid().comm().rank() == 0; + // Parallel LGR: reuse the simulator's own (distributed) transmissibilities for the + // INIT output -- each rank contributes its interior connections, gathered on the + // I/O rank and keyed by level-Cartesian indices so the output walk over the global + // (equil) grid can look them up directly. This reuses the values already computed + // in parallel for the simulation itself instead of recomputing a whole-grid + // transmissibility. finishTransmissibilities() is idempotent; calling it here only + // moves the (anyway required) local trans build before the INIT write. + if constexpr (std::is_same_v, Dune::CpGrid>) { + // Gate on the deck's LGRs -- the same condition the writer keys its LGR + // output walk on (writeInit / extractOutputTransAndNNC). Grid refinement + // alone (grid().maxLevel() > 0, e.g. after adapt()) must not enable the + // gathered mode: the writer would look up different keys than the ones + // the leaf-grid walk records. + if (simulator.vanguard().eclState().getLgrs().size() > 0) { + if (simulator.vanguard().numOverlap() < 1) { + throw std::runtime_error("Parallel LGR output requires at least one " + "overlap layer (--num-overlap >= 1): a " + "rank-boundary connection is recorded by the " + "rank owning the lower-index cell, which needs " + "the partner cell in its overlap layer."); + } + finishTransmissibilities(); + const auto& localTrans = simulator.problem().eclTransmissibilities(); + eclWriter_->setGatheredLgrTrans( + gatherLgrOutputTrans(simulator.vanguard().grid(), + simulator.vanguard().gridView(), + [&localTrans](unsigned c1, unsigned c2) + { return static_cast(localTrans.transmissibility(c1, c2)); })); + // All output values (TRANX/Y/Z and NNC) come from the gathered records -- + // no whole-grid transmissibility object is needed on the I/O rank. + wholeGridTransNeeded = false; + } + } + if (wholeGridTransNeeded) { eclWriter_->setTransmissibilities(&simulator.vanguard().globalTransmissibility()); + } } else { finishTransmissibilities(); eclWriter_->setTransmissibilities(&simulator.problem().eclTransmissibilities()); diff --git a/opm/simulators/flow/LgrOutputTransGather.hpp b/opm/simulators/flow/LgrOutputTransGather.hpp new file mode 100644 index 00000000000..fd426310188 --- /dev/null +++ b/opm/simulators/flow/LgrOutputTransGather.hpp @@ -0,0 +1,186 @@ +// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- +// vi: set et ts=4 sw=4 sts=4: +/* + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . + + Consult the COPYING file in the top-level source directory of this + module for the precise wording of the license and the list of + copyright holders. +*/ +#ifndef OPM_LGR_OUTPUT_TRANS_GATHER_HPP +#define OPM_LGR_OUTPUT_TRANS_GATHER_HPP + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace Opm { + +/// Connection transmissibilities gathered for parallel LGR INIT output. +/// +/// Records are sorted by key (established by gatherLgrOutputTrans below) for +/// binary-search lookup. The I/O rank holds every connection of the global +/// grid, so a node-based container is deliberately avoided. +/// +/// sameLevel: key (level, min level-Cartesian index, max level-Cartesian index) +/// -- every connection between two cells of the same level grid +/// (TRANX/TRANY/TRANZ and same-level NNCs). +/// crossLevel: key (smaller level, its level-Cartesian index, +/// larger level, its level-Cartesian index) +/// -- every connection between cells of different level grids +/// (global<->LGR and LGR<->LGR NNCs; TRANGL/TRANLL). +struct GatheredLgrOutputTrans +{ + std::vector, double>> sameLevel; + std::vector, double>> crossLevel; +}; + +namespace detail { + +/// Zip flattened (key, value) records into key-sorted records ready for +/// binary-search lookup. keys holds N ints per record, values one double. +template +std::vector, double>> +sortLgrTransRecords(const std::vector& keys, + const std::vector& values) +{ + std::vector, double>> records; + records.reserve(values.size()); + for (std::size_t i = 0; i < values.size(); ++i) { + std::array key; + for (std::size_t j = 0; j < N; ++j) { + key[j] = keys[N*i + j]; + } + records.emplace_back(key, values[i]); + } + std::sort(records.begin(), records.end()); + // Keys are unique by construction: every connection is recorded by exactly + // one rank (see gatherLgrOutputTrans below). + assert(std::adjacent_find(records.begin(), records.end(), + [](const auto& a, const auto& b) + { return a.first == b.first; }) == records.end()); + return records; +} + +} // namespace detail + +/// Gather the simulator's own (distributed) transmissibilities for parallel LGR INIT output. +/// +/// Each rank walks its interior leaf cells and records every connection it owns, keyed by +/// level-Cartesian indices (see GatheredLgrOutputTrans), then the records are gathered on the +/// I/O rank (rank 0). The level-Cartesian key is geometrically canonical -- defined by the LGR +/// specification, identical on the distributed grid and the undistributed (equil) copy -- so +/// the I/O rank's output walk over the equil grid can look values up directly. +/// +/// Every connection is recorded exactly once. A same-level connection is recorded by the rank +/// that owns the cell with the smaller level-Cartesian index; that comparison is +/// rank-independent, so at a rank boundary only one of the two owner ranks records the +/// connection -- which it can, because its partner cell is present in its overlap layer (this +/// requires at least one overlap layer; the caller guards --num-overlap). A cross-level +/// connection is recorded from the smaller-level side only. +/// +/// This reuses the values the simulation itself computed in parallel instead of recomputing a +/// whole-grid transmissibility on the I/O rank. +/// +/// \return the complete, key-sorted records on rank 0; empty records on all other ranks. +template +GatheredLgrOutputTrans +gatherLgrOutputTrans(const Dune::CpGrid& grid, + const GridView& gridView, + TransFn&& transFn) +{ + std::vector sameKeys; // flattened: level, minCart, maxCart per record + std::vector sameValues; + std::vector crossKeys; // flattened: smallLevel, smallCart, largeLevel, largeCart + std::vector crossValues; + + const LevelCartesianIndexMapper levelCartMapp(grid); + const Dune::MultipleCodimMultipleGeomTypeMapper + elemMapper(gridView, Dune::mcmgElementLayout()); + + for (const auto& elem : elements(gridView, Dune::Partitions::interior)) { + // The inside cell is the same for every intersection of this element. + const int levelIn = elem.level(); + const int cartIn = levelCartMapp.cartesianIndex(elem.getLevelElem().index(), levelIn); + const auto idxIn = elemMapper.index(elem); + + for (const auto& is : intersections(gridView, elem)) { + if (!is.neighbor()) { + continue; + } + + const auto outside = is.outside(); + const int levelOut = outside.level(); + + if (levelIn != levelOut) { + if (levelIn > levelOut) { + continue; // recorded exactly once, from the smaller-level side + } + + crossKeys.push_back(levelIn); + crossKeys.push_back(cartIn); + crossKeys.push_back(levelOut); + crossKeys.push_back(levelCartMapp.cartesianIndex( + outside.getLevelElem().index(), levelOut)); + crossValues.push_back(transFn(idxIn, elemMapper.index(outside))); + continue; + } + + const int cartOut = levelCartMapp.cartesianIndex( + outside.getLevelElem().index(), levelIn); + + if (cartIn > cartOut) { + // Record each connection once, in canonical direction. The comparison + // is rank-independent, so at a rank boundary exactly one of the two + // owner ranks records the connection. + continue; + } + + sameKeys.push_back(levelIn); + sameKeys.push_back(cartIn); + sameKeys.push_back(cartOut); + sameValues.push_back(transFn(idxIn, elemMapper.index(outside))); + } + } + + // Gather every rank's records on the I/O rank; gatherv is collective and + // returns empty vectors on the other ranks. MPI's int counts/displacements + // bound the total record count across ALL ranks at ~2^31 ints -- a shared + // MPI-wide ceiling (~500M same-level records), not a per-rank one. + const auto& comm = grid.comm(); + const auto allSameKeys = gatherv(sameKeys, comm, 0).first; + const auto allSameValues = gatherv(sameValues, comm, 0).first; + const auto allCrossKeys = gatherv(crossKeys, comm, 0).first; + const auto allCrossValues = gatherv(crossValues, comm, 0).first; + + GatheredLgrOutputTrans gathered; + gathered.sameLevel = detail::sortLgrTransRecords<3>(allSameKeys, allSameValues); + gathered.crossLevel = detail::sortLgrTransRecords<4>(allCrossKeys, allCrossValues); + return gathered; +} + +} // namespace Opm + +#endif // OPM_LGR_OUTPUT_TRANS_GATHER_HPP diff --git a/parallelTests.cmake b/parallelTests.cmake index d08be1510e2..4a1c40bbbe4 100644 --- a/parallelTests.cmake +++ b/parallelTests.cmake @@ -946,6 +946,33 @@ add_test_compare_parallel_simulation( --linear-solver-reduction=1e-7 ) +# Parallel LGR INIT/EGRID regression: reuses the existing SPE1CASE1_CARFIN deck (opm-tests/lgr, +# already registered for spe1case1_carfin/spe1case1_carfin_parallel in compareECLFiles.cmake) and +# the existing run-parallel-regressionTest.sh driver (COMPARE_MODE init: dry-run, compares +# EGRID+INIT instead of SMRY+UNRST). Guards the parallel LGR INIT transmissibility output +# (gatherLgrOutputTrans + the EclGenericWriter gathered-record lookups) -- pre-fix, the +# parallel run deadlocks here. +add_test_compare_parallel_simulation( + CASENAME + spe1case1_carfin + FILENAME + SPE1CASE1_CARFIN + SIMULATOR + flow + DEV_SIMULATOR + flow_blackoil + DIR + lgr + ABS_TOL + 1e-3 + REL_TOL + 1e-5 + COMPARE_MODE + init + TEST_ARGS + --parsing-strictness=low +) + opm_set_test_driver(${PROJECT_SOURCE_DIR}/tests/run-comparison.sh "") add_test_compareSeparateECLFiles( @@ -1011,3 +1038,4 @@ add_test_compareSeparateECLFiles( --matrix-add-well-contributions=true --linear-solver=ilu0 ) + diff --git a/tests/run-parallel-regressionTest.sh b/tests/run-parallel-regressionTest.sh index bf35fad7f88..693f25bdd30 100755 --- a/tests/run-parallel-regressionTest.sh +++ b/tests/run-parallel-regressionTest.sh @@ -17,13 +17,18 @@ then echo -e "\t\t -e Simulator binary to use" echo -e "\tOptional options:" echo -e "\t\t -n Number of MPI processes to use" + echo -e "\t\t -m Comparison mode: summary (default, compares SMRY+UNRST" + echo -e "\t\t against a normal run) or init (dry-run, compares" + echo -e "\t\t EGRID+INIT only -- for tracking parallel INIT-file" + echo -e "\t\t regressions, e.g. parallel LGR transmissibility)" exit 1 fi MPI_PROCS=4 +MODE=summary OPTIND=1 -while getopts "i:r:f:a:t:c:e:n:" OPT +while getopts "i:r:f:a:t:c:e:n:m:" OPT do case "${OPT}" in i) INPUT_DATA_PATH=${OPTARG} ;; @@ -34,38 +39,71 @@ do c) COMPARE_ECL_COMMAND=${OPTARG} ;; e) EXE_NAME=${OPTARG} ;; n) MPI_PROCS=${OPTARG} ;; + m) MODE=${OPTARG} ;; esac done shift $(($OPTIND-1)) TEST_ARGS="$@" +case "${MODE:-summary}" in + summary|init) ;; + *) echo "Unknown mode '${MODE}' (expected 'summary' or 'init')"; exit 1 ;; +esac + +if [ "${MODE}" = "init" ] +then + RUN_FLAG="--enable-dry-run=true --enable-ecl-output=true" +else + RUN_FLAG="--enable-opm-rst-file=true" +fi + rm -Rf ${RESULT_PATH} mkdir -p ${RESULT_PATH} cd ${RESULT_PATH} -"${EXE_NAME}" ${TEST_ARGS} --enable-opm-rst-file=true --output-dir=${RESULT_PATH} +"${EXE_NAME}" ${TEST_ARGS} ${RUN_FLAG} --output-dir=${RESULT_PATH} test $? -eq 0 || exit 1 mkdir mpi cd mpi -mpirun -np ${MPI_PROCS} "${EXE_NAME}" ${TEST_ARGS} --enable-opm-rst-file=true --output-dir=${RESULT_PATH}/mpi +mpirun -np ${MPI_PROCS} "${EXE_NAME}" ${TEST_ARGS} ${RUN_FLAG} --output-dir=${RESULT_PATH}/mpi test $? -eq 0 || exit 1 cd .. ecode=0 -echo "=== Executing comparison for summary file ===" -${COMPARE_ECL_COMMAND} -t SMRY -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} -if [ $? -ne 0 ] +if [ "${MODE}" = "init" ] then - ecode=1 - ${COMPARE_ECL_COMMAND} -t SMRY -a -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} -fi + echo "=== Executing comparison for EGRID file ===" + ${COMPARE_ECL_COMMAND} -t EGRID ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + if [ $? -ne 0 ] + then + ecode=1 + ${COMPARE_ECL_COMMAND} -a -t EGRID ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + fi -echo "=== Executing comparison for restart file ===" -${COMPARE_ECL_COMMAND} -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} -if [ $? -ne 0 ] -then - ecode=1 - ${COMPARE_ECL_COMMAND} -a -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + # -x ignores the parallel-only MPI_RANK keyword, which has no serial counterpart to compare against. + echo "=== Executing comparison for INIT file (ignoring parallel-only MPI_RANK) ===" + ${COMPARE_ECL_COMMAND} -t INIT -x ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + if [ $? -ne 0 ] + then + ecode=1 + ${COMPARE_ECL_COMMAND} -a -t INIT -x ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + fi +else + echo "=== Executing comparison for summary file ===" + ${COMPARE_ECL_COMMAND} -t SMRY -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + if [ $? -ne 0 ] + then + ecode=1 + ${COMPARE_ECL_COMMAND} -t SMRY -a -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + fi + + echo "=== Executing comparison for restart file ===" + ${COMPARE_ECL_COMMAND} -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + if [ $? -ne 0 ] + then + ecode=1 + ${COMPARE_ECL_COMMAND} -a -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL} + fi fi exit $ecode