Stabilize MPI test timing

modules/runners/src/runners.cpp

@@ -82,22 +82,22 @@ int RunAllTests() {
 }
 
 void SyncGTestSeed() {
-  unsigned int seed = 0;
   int rank = -1;
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  if (rank == 0) {
+  int seed = ::testing::GTEST_FLAG(random_seed);
+  if (rank == 0 && seed == 0) {
     try {
-      seed = std::random_device{}();
+      seed = static_cast<int>((std::random_device{}() % 99999U) + 1U);
     } catch (...) {
       seed = 0;
     }
     if (seed == 0) {
       const auto now = static_cast<std::uint64_t>(std::chrono::steady_clock::now().time_since_epoch().count());
-      seed = static_cast<unsigned int>(((now & 0x7fffffffULL) | 1ULL));
+      seed = static_cast<int>((now % 99999ULL) + 1ULL);
     }
   }
-  MPI_Bcast(&seed, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD);
-  ::testing::GTEST_FLAG(random_seed) = static_cast<int>(seed);
+  MPI_Bcast(&seed, 1, MPI_INT, 0, MPI_COMM_WORLD);
+  ::testing::GTEST_FLAG(random_seed) = seed;
 }
 
 void SyncGTestFilter() {

modules/util/include/func_test_util.hpp

@@ -103,6 +103,7 @@ class BaseRunFuncTests : public ::testing::TestWithParam<FuncTestParam<InType, O
 
   void ValidateTask() {
     EXPECT_TRUE(task_->Validation());
+    SynchronizeMpiRanks();
     EXPECT_TRUE(task_->PreProcessing());
   }
 

modules/util/include/perf_test_util.hpp

@@ -85,6 +85,7 @@ class BaseRunPerfTests : public ::testing::TestWithParam<PerfTestParam<InType, O
     task_ = task_getter(GetTestInputData());
     ppc::performance::Perf perf(task_);
     ppc::performance::PerfAttr perf_attr;
+    SynchronizeMpiRanks();
     SetPerfAttributes(perf_attr);
 
     if (mode == ppc::performance::PerfResults::TypeOfRunning::kPipeline) {

modules/util/include/util.hpp

@@ -75,6 +75,7 @@ int GetNumThreads();
 int GetNumProc();
 double GetTaskMaxTime();
 double GetPerfMaxTime();
+void SynchronizeMpiRanks();
 
 template <typename T>
 std::string GetNamespace() {

modules/util/src/util.cpp

@@ -1,5 +1,7 @@
 #include "util/include/util.hpp"
 
+#include <mpi.h>
+
 #include <algorithm>
 #include <array>
 #include <filesystem>
@@ -65,3 +67,20 @@ bool ppc::util::IsUnderMpirun() {
     return static_cast<bool>(mpi_env.has_value());
   });
 }
+
+void ppc::util::SynchronizeMpiRanks() {
+  int initialized = 0;
+  if (MPI_Initialized(&initialized) != MPI_SUCCESS || initialized == 0) {
+    return;
+  }
+
+  int finalized = 0;
+  if (MPI_Finalized(&finalized) != MPI_SUCCESS || finalized != 0) {
+    return;
+  }
+
+  const int barrier_res = MPI_Barrier(MPI_COMM_WORLD);
+  if (barrier_res != MPI_SUCCESS) {
+    MPI_Abort(MPI_COMM_WORLD, barrier_res);
+  }
+}