Minimal viable custom MPI for mesh decomposition

.gitignore

@@ -6,4 +6,5 @@ heffte/
 docs_doxygen/
 docs_sphinx/
 tutorial/getting_started/Example0/build_*
-tutorial/getting_started/Example0/install*
+tutorial/getting_started/Example0/install*
+examples/mesh_decomp/lib/*

benchmark/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1.3)
+cmake_minimum_required(VERSION 3.5)
 
 project (matarbenchmark)
 

examples/CMakeLists.txt

@@ -156,6 +156,9 @@ if (KOKKOS)
   if (MPI)
     include_directories(laplaceMPI)
     add_subdirectory(laplaceMPI)
+
+    include_directories(mesh_decomp)
+    add_subdirectory(mesh_decomp)
   endif()
 
 endif()
@@ -191,11 +194,12 @@ add_subdirectory(sparsetests)
 include_directories(test_rocm)
 add_subdirectory(test_rocm)
 
-#include_directories(phaseField/srcKokkosVerbose)
-#add_subdirectory(phaseField/srcKokkosVerbose)
 
-#include_directories(phaseField/srcMacros)
-#add_subdirectory(phaseField/srcMacros)
+# include_directories(phaseField/srcKokkosVerbose)
+# add_subdirectory(phaseField/srcKokkosVerbose)
+
+# include_directories(phaseField/srcMacros)
+# add_subdirectory(phaseField/srcMacros)
 
-#include_directories(phaseFieldMPI)
-#add_subdirectory(phaseFieldMPI)
+# include_directories(phaseFieldMPI)
+# add_subdirectory(phaseFieldMPI)

examples/gArrayofgArrays/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1.3)
+cmake_minimum_required(VERSION 3.5)
 
 find_package(Matar REQUIRED)
 

examples/halfspace_cooling/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1.3)
+cmake_minimum_required(VERSION 3.5)
 
 find_package(Matar REQUIRED)
 

examples/laplace/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1.3)
+cmake_minimum_required(VERSION 3.5)
 
 find_package(Matar REQUIRED)
 

examples/laplaceMPI/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1.3)
+cmake_minimum_required(VERSION 3.5)
 
 if (KOKKOS)
   #find_package(Kokkos REQUIRED) #new

examples/mesh_decomp/CMakeLists.txt

@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Find MPI
+find_package(MPI REQUIRED)
+add_definitions(-DHAVE_MPI=1)
+
+find_package(Matar REQUIRED)
+
+execute_process(
+  COMMAND ${CMAKE_CURRENT_LIST_DIR}/install_ptscotch.sh
+  WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+  RESULT_VARIABLE INSTALL_PTSCOTCH_RESULT
+)
+
+if(NOT INSTALL_PTSCOTCH_RESULT EQUAL 0)
+  message(FATAL_ERROR "Failed to install PT-Scotch by running install_ptscotch.sh")
+endif()
+
+
+if (KOKKOS)
+  #find_package(Kokkos REQUIRED) #new
+
+  add_executable(mesh_decomp mesh_decomp.cpp)
+
+  add_definitions(-DHAVE_KOKKOS=1)
+
+  # Add include directories for MPI and Scotch/PT-Scotch
+  target_include_directories(mesh_decomp PRIVATE ${MPI_CXX_INCLUDE_PATH} ${CMAKE_CURRENT_LIST_DIR}/lib/scotch/build/src/include)
+
+  # Link libraries - order matters! libptscotch depends on libscotch
+  # Use -Wl,--whole-archive to ensure all symbols are included from static libraries
+  # Note: Only link libptscotcherr.a (not libscotcherr.a) to avoid multiple definitions
+  target_link_libraries(mesh_decomp ${LINKING_LIBRARIES} MPI::MPI_CXX 
+    -Wl,--whole-archive
+    ${CMAKE_CURRENT_LIST_DIR}/lib/scotch/build/lib/libscotch.a
+    -Wl,--no-whole-archive
+    -Wl,--whole-archive
+    ${CMAKE_CURRENT_LIST_DIR}/lib/scotch/build/lib/libptscotcherr.a
+    ${CMAKE_CURRENT_LIST_DIR}/lib/scotch/build/lib/libptscotch.a
+    -Wl,--no-whole-archive
+    -lz     # zlib for gzip compression
+    -lbz2   # bzip2 library
+    -llzma  # xz compression library
+  )
+endif()