fix: Infinite hang when MPI process crashes

src/coreComponents/codingUtilities/tests/testParallelTestUtilities.cpp

@@ -68,6 +68,6 @@ int main( int ac, char * av[] )
   ::testing::InitGoogleTest( &ac, av );
   geos::basicSetup( ac, av );
   int const result = RUN_ALL_TESTS();
-  geos::basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }

src/coreComponents/common/MpiWrapper.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "MpiWrapper.hpp"
+#include "LvArray/src/system.hpp"
 #include <unistd.h>
 
 #if defined(__clang__)
@@ -152,11 +153,19 @@ void internal::ManagedResources::finalize()
   m_mpiTypes.clear();
 }
 
-void MpiWrapper::finalize()
+void MpiWrapper::finalize( bool inError )
 {
 #ifdef GEOS_USE_MPI
-  internal::getManagedResources().finalize();
-  MPI_CHECK_ERROR( MPI_Finalize() );
+  if( !inError )
+  {
+    MpiWrapper::commFree( MPI_COMM_GEOS );
+    internal::getManagedResources().finalize();
+    MPI_CHECK_ERROR( MPI_Finalize() );
+  }
+  else
+  {
+    LvArray::system::callErrorHandler();
+  }
 #endif
 }
 

src/coreComponents/common/MpiWrapper.hpp

@@ -196,10 +196,11 @@ struct MpiWrapper
   static int init( int * argc, char * * * argv );
 
   /**
-   * @brief Free MPI managed resources, then call MPI_Finalize().
+   * @brief Free MPI managed resources
    * Please note that once called, MPI functions, communicators and resources can no longer be used.
+   * @param inError indicate if an exception occured
    */
-  static void finalize();
+  static void finalize( bool inError );
 
   static MPI_Comm commDup( MPI_Comm const comm );
 

src/coreComponents/common/format/table/unitTests/testMpiTable.cpp

@@ -35,7 +35,7 @@ class MpiTestScope
 
   ~MpiTestScope()
   {
-    geos::cleanupEnvironment();
+    geos::cleanupEnvironment( false );
   }
 
 };

src/coreComponents/common/initializeEnvironment.cpp

@@ -19,7 +19,6 @@
 #include "Path.hpp"
 #include "LvArray/src/system.hpp"
 #include "common/LifoStorageCommon.hpp"
-#include "common/MemoryInfos.hpp"
 #include "logger/ErrorHandling.hpp"
 #include "logger/ExternalErrorHandler.hpp"
 #include <umpire/TypedAllocator.hpp>
@@ -192,10 +191,9 @@ void setupMPI( int argc, char * argv[] )
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void finalizeMPI()
+void finalizeMPI( bool inError )
 {
-  MpiWrapper::commFree( MPI_COMM_GEOS );
-  MpiWrapper::finalize();
+  MpiWrapper::finalize( inError );
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -330,13 +328,12 @@ void setupEnvironment( int argc, char * argv[] )
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void cleanupEnvironment()
+void cleanupEnvironment( bool inError )
 {
-  MemoryLogging::getInstance().memoryStatsReport();
   LvArray::system::resetSignalHandling();
   finalizeLogger();
   finalizeCaliper();
-  finalizeMPI();
+  finalizeMPI( inError );
 }
 
 } // namespace geos

src/coreComponents/common/initializeEnvironment.hpp

@@ -136,7 +136,7 @@ void setupMPI( int argc, char * argv[] );
 /**
  * @brief Finalize MPI.
  */
-void finalizeMPI();
+void finalizeMPI( bool inError );
 
 /**
  * @brief Setup CUDA
@@ -153,8 +153,9 @@ void setupEnvironment( int argc, char * argv[] );
 
 /**
  * @brief Cleanup/finalize the environment.
+ * @param inError indicate if an exception occured
  */
-void cleanupEnvironment();
+void cleanupEnvironment( bool inError );
 
 #if defined( GEOS_USE_CALIPER )
 

src/coreComponents/common/unitTests/testMpiWrapper.cpp

@@ -33,7 +33,7 @@ class MpiTestScope
 
   ~MpiTestScope()
   {
-    geos::cleanupEnvironment();
+    geos::cleanupEnvironment( false );
   }
 
 };

src/coreComponents/constitutive/capillaryPressure/unitTests/testCapillaryPressure.cpp

@@ -725,7 +725,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/constitutive/relativePermeability/unitTests/testRelPerm.cpp

@@ -1105,7 +1105,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testCO2BrinePVTModels.cpp

@@ -1114,7 +1114,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testCO2SpycherPruessModels.cpp

@@ -378,7 +378,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testInverseCapillaryPressure.cpp

@@ -519,7 +519,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testMultiFluidBlackOil.cpp

@@ -277,7 +277,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testMultiFluidCO2Brine.cpp

@@ -403,7 +403,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testMultiFluidDeadOil.cpp

@@ -245,7 +245,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testMultiFluidThreePhaseCompositionalMultiphase.cpp

@@ -223,7 +223,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testMultiFluidTwoPhaseCompositionalMultiphase.cpp

@@ -275,7 +275,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/constitutive/unitTests/testTwoPhaseImmiscibleFluid.cpp

@@ -267,7 +267,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/dataRepository/unitTests/testErrorHandling.cpp

@@ -457,6 +457,6 @@ int main( int ac, char * av[] )
   ::testing::InitGoogleTest( &ac, av );
   geos::setupEnvironment( ac, av );
   int const result = RUN_ALL_TESTS();
-  geos::cleanupEnvironment( );
+  geos::cleanupEnvironment( false );
   return result;
 }

src/coreComponents/dataRepository/unitTests/testObjectCatalog.cpp

@@ -128,7 +128,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/dataRepository/unitTests/testWrapperHelpers.cpp

@@ -240,7 +240,7 @@ int main( int argc, char * argv[] )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/denseLinearAlgebra/unitTests/testBlasLapack.cpp

@@ -1319,7 +1319,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::cleanupEnvironment();
+  geos::cleanupEnvironment( false );
 
   return result;
 }

src/coreComponents/fileIO/Outputs/unitTests/testMemoryStats.cpp

@@ -205,6 +205,6 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }

src/coreComponents/fileIO/timeHistory/unitTests/testHDFFile.cpp

@@ -134,6 +134,6 @@ int main( int ac, char * av[] )
   ::testing::InitGoogleTest( &ac, av );
   geos::basicSetup( ac, av );
   int const result = RUN_ALL_TESTS();
-  geos::basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }

src/coreComponents/fileIO/timeHistory/unitTests/testHDFParallelFile.cpp

@@ -96,6 +96,6 @@ int main( int ac, char * av[] )
   ::testing::InitGoogleTest( &ac, av );
   geos::basicSetup( ac, av );
   int const result = RUN_ALL_TESTS();
-  geos::basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }

src/coreComponents/finiteElement/elementFormulations/unitTests/testConformingVirtualElementOrder1.cpp

@@ -363,6 +363,6 @@ int main( int argc, char * * argv )
   ::testing::InitGoogleTest( &argc, argv );
   g_commandLineOptions = *geos::basicSetup( argc, argv );
   int const result = RUN_ALL_TESTS();
-  geos::basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }

src/coreComponents/finiteVolume/mimeticInnerProducts/unitTests/testMimeticInnerProducts.cpp

@@ -1785,7 +1785,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/functions/unitTests/testFunctions.cpp

@@ -914,7 +914,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  // geos::basicCleanup();
+  // geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/constitutiveTests/testPVT.cpp

@@ -42,7 +42,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/constitutiveTests/testReactiveFluid.cpp

@@ -42,7 +42,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/constitutiveTests/testRelPermHysteresis.cpp

@@ -255,7 +255,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/constitutiveTests/testTriaxial.cpp

@@ -42,7 +42,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/dataRepositoryTests/testGroupPath.cpp

@@ -182,7 +182,7 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/dataRepositoryTests/testRestartBasic.cpp

@@ -140,7 +140,7 @@ int main( int argc, char * argv[] )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/dataRepositoryTests/testRestartExtended.cpp

@@ -483,7 +483,7 @@ int main( int argc, char * argv[] )
 
   int const result = RUN_ALL_TESTS();
 
-  geos::basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/fieldSpecificationTests/testAquiferBoundaryCondition.cpp

@@ -102,7 +102,7 @@ int main( int argc, char * * argv )
   ::testing::InitGoogleTest( &argc, argv );
   int const result = RUN_ALL_TESTS();
 
-  basicCleanup();
+  geos::basicCleanup( false );
 
   return result;
 }

src/coreComponents/integrationTests/fieldSpecificationTests/testFieldSpecification.cpp

@@ -445,6 +445,6 @@ int main( int argc, char * * argv )
 
   int const result = RUN_ALL_TESTS();
 
-  basicCleanup();
+  geos::basicCleanup( false );
   return result;
 }