Merge remote-tracking branch 'origin/1.2.0rc' into dev/1dsr

Author: StaticObserver

.github/workflows/actions.yml

@@ -27,9 +27,14 @@ jobs:
       matrix:
         device: [cpu, amd-gpu, nvidia-gpu]
         precision: [double, single]
+        mpi: [serial, parallel]
         exclude: # my AMD GPU doesn't support fp64 atomics : (
         - device: amd-gpu 
           precision: double
+        - device: amd-gpu
+          mpi: parallel
+        - device: nvidia-gpu
+          mpi: parallel
     runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
@@ -47,7 +52,7 @@ jobs:
             fi
           elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
             FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
-          elif [ "${{ matrix.device }}" = "cpu" ]; then
+          elif [ "${{ matrix.mpi }}" = "parallel" ]; then
             FLAGS="-D mpi=ON"
           fi
           cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS

.gitmodules

@@ -4,7 +4,6 @@
 [submodule "extern/adios2"]
 	path = extern/adios2
 	url = https://github.com/ornladios/ADIOS2.git
-	branch = master
 [submodule "extern/Kokkos"]
 	path = extern/Kokkos
 	url = https://github.com/kokkos/kokkos.git

CMakeLists.txt

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103,C0111,E1120,R0913,R0915
+
 cmake_minimum_required(VERSION 3.16)
 cmake_policy(SET CMP0110 NEW)
 
@@ -8,10 +10,10 @@ project(
   VERSION 1.2.0
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
+set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ")
+string(APPEND hash_cmd "|| echo $(git rev-parse HEAD)-mod")
 execute_process(
-  COMMAND
-    bash -c
-    "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod"
+  COMMAND bash -c ${hash_cmd}
   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
   OUTPUT_VARIABLE GIT_HASH
   ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -46,6 +48,10 @@ set(mpi
     ${default_mpi}
     CACHE BOOL "Use MPI")
 
+set(gpu_aware_mpi
+    ${default_gpu_aware_mpi}
+    CACHE BOOL "Enable GPU-aware MPI")
+
 # -------------------------- Compilation settings -------------------------- #
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -55,9 +61,7 @@ if(${DEBUG} STREQUAL "OFF")
   set(CMAKE_BUILD_TYPE
       Release
       CACHE STRING "CMake build type")
-  set(CMAKE_CXX_FLAGS
-      "${CMAKE_CXX_FLAGS} -DNDEBUG -Wno-unused-local-typedefs -Wno-unknown-cuda-version"
-  )
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
 else()
   set(CMAKE_BUILD_TYPE
       Debug
@@ -79,38 +83,60 @@ set(BUILD_TESTING
     CACHE BOOL "Build tests")
 
 # ------------------------ Third-party dependencies ------------------------ #
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake)
 
-find_or_fetch_dependency(Kokkos FALSE)
-find_or_fetch_dependency(plog TRUE)
+find_or_fetch_dependency(Kokkos FALSE QUIET)
+find_or_fetch_dependency(plog TRUE QUIET)
 set(DEPENDENCIES Kokkos::kokkos)
 include_directories(${plog_SRC}/include)
 
 # -------------------------------- Main code ------------------------------- #
 set_precision(${precision})
+if("${Kokkos_DEVICES}" MATCHES "CUDA")
+  add_compile_options("-D CUDA_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "HIP")
+  add_compile_options("-D HIP_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "SYCL")
+  add_compile_options("-D SYCL_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
+
+if(("${Kokkos_DEVICES}" MATCHES "CUDA")
+   OR ("${Kokkos_DEVICES}" MATCHES "HIP")
+   OR ("${Kokkos_DEVICES}" MATCHES "SYCL"))
+  set(DEVICE_ENABLED ON)
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
 
 # MPI
 if(${mpi})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/MPIConfig.cmake)
+  find_or_fetch_dependency(MPI FALSE REQUIRED)
+  include_directories(${MPI_CXX_INCLUDE_PATH})
+  add_compile_options("-D MPI_ENABLED")
   set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
+  if(${DEVICE_ENABLED})
+    if(${gpu_aware_mpi})
+      add_compile_options("-D GPU_AWARE_MPI")
+    endif()
+  else()
+    set(gpu_aware_mpi
+        OFF
+        CACHE BOOL "Use explicit copy when using MPI + GPU")
+  endif()
 endif()
 
 # Output
 if(${output})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake)
-  find_or_fetch_dependency(adios2 FALSE)
-  if(NOT DEFINED ENV{HDF5_ROOT})
-    if(DEFINED ENV{CONDA_PREFIX})
-      execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q"
-                      RESULT_VARIABLE HDF5_INSTALLED)
-      if(HDF5_INSTALLED EQUAL 0)
-        set(HDF5_ROOT $ENV{CONDA_PREFIX})
-      endif()
-    endif()
-  endif()
-  find_package(HDF5 REQUIRED)
-
+  find_or_fetch_dependency(adios2 FALSE QUIET)
+  add_compile_options("-D OUTPUT_ENABLED")
   if(${mpi})
     set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi)
   else()
@@ -129,11 +155,12 @@ elseif(BENCHMARK)
 else()
   # ----------------------------------- GUI ---------------------------------- #
   if(${gui})
-    find_or_fetch_dependency(nttiny FALSE)
+    find_or_fetch_dependency(nttiny FALSE QUIET)
   endif()
 
   # ------------------------------- Main source ------------------------------ #
   set_problem_generator(${pgen})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)
 endif()
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)

README.md

@@ -10,29 +10,20 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 
-## Lead developers
-
-☕ __Hayk Hakobyan__ {[@haykh](https://github.com/haykh)}
-
-🥔 __Jens Mahlmann__ {[@jmahlmann](https://github.com/jmahlmann)}
-
-💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov)}
-
-🧋 __Alisa Galishnikova__ {[@alisagk](https://github.com/alisagk)}
-
-🐬 __Sasha Philippov__ {[@sashaph](https://github.com/sashaph)}
-
 ## Contributors (alphabetical)
 
-👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC}
-
-🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere}
-
-:radio: __Siddhant Solanki__ {[@sidruns30](https://github.com/sidruns30): framework}
-
-🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC}
-
-😺 __Muni Zhou__ {[@munizhou](https://github.com/munizhou): PIC}
+* :guitar: Ludwig Böss {[@LudwigBoess](https://github.com/LudwigBoess)}
+* :eyes: Yangyang Cai {[@StaticObserver](https://github.com/StaticObserver)}
+* :person_tipping_hand: Alexander Chernoglazov {[@SChernoglazov](https://github.com/SChernoglazov)}
+* :tea: Benjamin Crinquand {[@bcrinquand](https://github.com/bcrinquand)}
+* :bubble_tea: Alisa Galishnikova {[@alisagk](https://github.com/alisagk)}
+* :locomotive: Evgeny Gorbunov {[@Alcauchy](https://github.com/Alcauchy)}
+* :coffee: Hayk Hakobyan {[@haykh](https://github.com/haykh)}
+* :potato: Jens Mahlmann {[@jmahlmann](https://github.com/jmahlmann)}
+* :dolphin: Sasha Philippov {[@sashaph](https://github.com/sashaph)}
+* :radio: Siddhant Solanki {[@sidruns30](https://github.com/sidruns30)}
+* :shrug: Arno Vanthieghem {[@vanthieg](https://github.com/vanthieg)}
+* :cat: Muni Zhou {[@munizhou](https://github.com/munizhou)}
 
 ## Branch policy
 

TASKLIST.md

@@ -12,16 +12,18 @@ set(ADIOS2_USE_Fortran
     CACHE BOOL "Use Fortran for ADIOS2")
 
 # Format/compression support
-set(ADIOS2_USE_ZeroMQ
-    OFF
-    CACHE BOOL "Use ZeroMQ for ADIOS2")
+set(ADIOS2_USE_HDF5
+    ON
+    CACHE BOOL "Use HDF5 for ADIOS2")
 
 set(ADIOS2_USE_MPI
     ${mpi}
     CACHE BOOL "Use MPI for ADIOS2")
 
+set(ADIOS2_USE_ZeroMQ
+    OFF
+    CACHE BOOL "Use ZeroMQ for ADIOS2")
+
 set(ADIOS2_USE_CUDA
     OFF
     CACHE BOOL "Use CUDA for ADIOS2")
-
-add_compile_options("-D OUTPUT_ENABLED")

cmake/MPIConfig.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
 add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global)

cmake/adios2Config.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 # -------------------------------- Precision ------------------------------- #
 function(set_precision precision_name)
   list(FIND precisions ${precision_name} PRECISION_FOUND)
@@ -28,10 +30,8 @@ function(set_problem_generator pgen_name)
   endforeach()
   list(FIND PGEN_NAMES ${pgen_name} PGEN_FOUND)
   if(NOT ${pgen_name} STREQUAL "." AND ${PGEN_FOUND} EQUAL -1)
-    message(
-      FATAL_ERROR
-        "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}"
-    )
+    message(FATAL_ERROR "Invalid problem generator: "
+                        "${pgen_name}\nValid options are: ${PGEN_NAMES}")
   endif()
   set(PGEN
       ${pgen_name}

cmake/benchmark.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 # ----------------------------- Defaults ---------------------------------- #
 if(DEFINED ENV{Entity_ENABLE_DEBUG})
   set(default_debug
@@ -33,7 +35,7 @@ if(DEFINED ENV{Entity_ENABLE_OUTPUT})
       CACHE INTERNAL "Default flag for output")
 else()
   set(default_output
-      OFF
+      ON
       CACHE INTERNAL "Default flag for output")
 endif()
 
@@ -51,50 +53,36 @@ endif()
 
 set_property(CACHE default_gui PROPERTY TYPE BOOL)
 
-if(DEFINED ENV{Kokkos_ENABLE_CUDA})
-  set(default_KOKKOS_ENABLE_CUDA
-      $ENV{Kokkos_ENABLE_CUDA}
-      CACHE INTERNAL "Default flag for CUDA")
-else()
-  set(default_KOKKOS_ENABLE_CUDA
-      OFF
-      CACHE INTERNAL "Default flag for CUDA")
-endif()
-
-set_property(CACHE default_KOKKOS_ENABLE_CUDA PROPERTY TYPE BOOL)
-
-if(DEFINED ENV{Kokkos_ENABLE_HIP})
-  set(default_KOKKOS_ENABLE_HIP
-      $ENV{Kokkos_ENABLE_HIP}
-      CACHE INTERNAL "Default flag for HIP")
+if(DEFINED ENV{Entity_ENABLE_MPI})
+  set(default_mpi
+      $ENV{Entity_ENABLE_MPI}
+      CACHE INTERNAL "Default flag for MPI")
 else()
-  set(default_KOKKOS_ENABLE_HIP
+  set(default_mpi
       OFF
-      CACHE INTERNAL "Default flag for HIP")
+      CACHE INTERNAL "Default flag for MPI")
 endif()
 
-set_property(CACHE default_KOKKOS_ENABLE_HIP PROPERTY TYPE BOOL)
-
-if(DEFINED ENV{Kokkos_ENABLE_OPENMP})
-  set(default_KOKKOS_ENABLE_OPENMP
-      $ENV{Kokkos_ENABLE_OPENMP}
-      CACHE INTERNAL "Default flag for OpenMP")
+if(DEFINED ENV{Entity_MPI_DEVICE_COPY})
+  set(default_mpi_device_copy
+      $ENV{Entity_MPI_DEVICE_COPY}
+      CACHE INTERNAL "Default flag for copying from device to host for MPI")
 else()
-  set(default_KOKKOS_ENABLE_OPENMP
+  set(default_mpi_device_copy
       OFF
-      CACHE INTERNAL "Default flag for OpenMP")
+      CACHE INTERNAL "Default flag for copying from device to host for MPI")
 endif()
 
-set_property(CACHE default_KOKKOS_ENABLE_OPENMP PROPERTY TYPE BOOL)
+set_property(CACHE default_mpi PROPERTY TYPE BOOL)
 
-if(DEFINED ENV{Entity_ENABLE_MPI})
-  set(default_mpi
-      $ENV{Entity_ENABLE_MPI}
-      CACHE INTERNAL "Default flag for MPI")
+if(DEFINED ENV{Entity_ENABLE_GPU_AWARE_MPI})
+  set(default_gpu_aware_mpi
+      $ENV{Entity_ENABLE_GPU_AWARE_MPI}
+      CACHE INTERNAL "Default flag for GPU-aware MPI")
 else()
-  set(default_mpi
-      OFF
-      CACHE INTERNAL "Default flag for MPI")
+  set(default_gpu_aware_mpi
+      ON
+      CACHE INTERNAL "Default flag for GPU-aware MPI")
 endif()
 
-set_property(CACHE default_mpi PROPERTY TYPE BOOL)
+set_property(CACHE default_gpu_aware_mpi PROPERTY TYPE BOOL)