[DRAFT] Add buildkite pipeline

.buildkite/pipeline.yml

@@ -0,0 +1,54 @@
+steps:
+  - label: ":partyparrot: Build & Test AMD MI210 (Coverage)"
+    key: "build-test-mi210"
+    command: |
+      source /opt/spack-environment/activate.sh
+      export WORKSPACE=/workspace
+      mkdir -p /workspace/build
+      rocminfo
+      cd /workspace/build
+      FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+            -DCMAKE_BUILD_TYPE="coverage" \
+            -DSELF_ENABLE_GPU=ON \
+            -DSELF_GPU_BACKEND=HIP \
+            -DSELF_ENABLE_TESTING=ON \
+            -DCMAKE_HIP_ARCHITECTURES="gfx90a" \
+            -DGPU_TARGETS="gfx90a" \
+            -DSELF_ENABLE_EXAMPLES=ON \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            /workspace/
+      make -j
+      lcov --capture --initial \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/initial.info
+      ctest --test-dir /workspace/build --output-on-failure
+      lcov --capture \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/ctest-capture.info
+      lcov --add-tracefile /workspace/initial.info \
+           --add-tracefile /workspace/ctest-capture.info \
+           --output-file /workspace/coverage.info
+    env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:mi210:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+    agents:
+      queue: "galapagos"
+
+  - label: ":codecov: Upload Coverage"
+    key: "upload-coverage"
+    depends_on: "build-test-mi210"
+    command: |
+      curl -Os https://uploader.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      ./codecov -t "$${CODECOV_TOKEN}" \
+                -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
+                -F buildkite-rocm-mi210
+    secrets:
+      - CODECOV_TOKEN
+    agents:
+      queue: "galapagos"

CMakeLists.txt

@@ -36,7 +36,12 @@ option(SELF_ENABLE_MULTITHREADING "Option to enable CPU multithreading for `do c
 option(SELF_ENABLE_TESTING "Option to enable build of tests. (Default On)"  ON)
 option(SELF_ENABLE_EXAMPLES "Option to enable build of examples. (Default On)"  ON)
 option(SELF_ENABLE_GPU "Option to enable GPU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
-option(SELF_ENABLE_APU "Option to enable APU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
+option(SELF_ENABLE_APU "Option to enable APU backend. (Default Off)"  OFF)
+
+set(SELF_GPU_BACKEND "AUTO" CACHE STRING "GPU backend to use when SELF_ENABLE_GPU is ON: AUTO, CUDA, or HIP. AUTO prefers HIP when available. (Default AUTO)")
+set_property(CACHE SELF_GPU_BACKEND PROPERTY STRINGS AUTO CUDA HIP)
+
+option(SELF_REQUIRE_GPU_AWARE_MPI "Require GPU-aware MPI when building with GPU support. Set OFF to downgrade the check to a warning. (Default ON)"  ON)
 option(SELF_ENABLE_DOUBLE_PRECISION "Option to enable double precision for floating point arithmetic. (Default On)"  ON)
 
 set(SELF_MPIEXEC_NUMPROCS "2" CACHE STRING "The number of MPI ranks to use to launch MPI tests. Only used when launching test programs via ctest.")
@@ -151,76 +156,103 @@ if(SELF_ENABLE_DOUBLE_PRECISION)
 
 endif()
 
+if(SELF_ENABLE_GPU AND SELF_ENABLE_APU)
+    message(FATAL_ERROR "SELF_ENABLE_GPU and SELF_ENABLE_APU are mutually exclusive. Enable only one.")
+endif()
+
 if(SELF_ENABLE_GPU)
-    message("-- SELF Build System : Enabling GPU Support")
+    message("-- SELF Build System : Enabling GPU Support (backend: ${SELF_GPU_BACKEND})")
     set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_GPU" )
     set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_GPU" )
     set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_GPU")
     set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_GPU")
     set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_GPU" )
 
-    # Check MPI for GPU awareness
     # Add SELF's cmake module directory to the search path
     set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-
-    # Include the custom module
     include(CheckMPIFeatures)
     CheckMPIFeatures()
 
-    # HIP (Optional)
-    # IF HIP is found, we prefer HIP. IF HIP is not found, then we fall back to look for CUDA
-    # only. If neither HIP nor CUDA are found, Cmake will error out.
-    find_package(hip)
-    if(hip_FOUND)
-        if(MPI_HAS_QUERY_HIP_SUPPORT)
-            find_package(hipblas REQUIRED)
-            message("-- HIP found. Enabling HIP language.")
-            enable_language(HIP)
-            set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
-            set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
-            set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
-            set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
-            set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
-
-            if(SELF_ENABLE_DOUBLE_PRECISION)
-                set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
-                set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
-                set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
-                set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
-                set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
-            endif()
-            set( BACKEND_LIBRARIES hip::device roc::hipblas)
+    # Resolve which backend to activate based on SELF_GPU_BACKEND
+    if(SELF_GPU_BACKEND STREQUAL "HIP")
+        find_package(hip REQUIRED)
+        set(_self_use_hip TRUE)
+    elseif(SELF_GPU_BACKEND STREQUAL "CUDA")
+        find_package(CUDAToolkit REQUIRED)
+        set(_self_use_hip FALSE)
+    else() # AUTO: prefer HIP, fall back to CUDA
+        find_package(hip)
+        if(hip_FOUND)
+            set(_self_use_hip TRUE)
         else()
-            message( FATAL_ERROR "MPI installation is not GPU-aware" )
+            find_package(CUDAToolkit REQUIRED)
+            set(_self_use_hip FALSE)
         endif()
-    else()
-        # CUDA
-        find_package(CUDAToolkit REQUIRED)
+    endif()
+
+    if(_self_use_hip)
+        find_package(hipblas REQUIRED)
+        message("-- HIP found. Enabling HIP language.")
+        enable_language(HIP)
+        if(NOT MPI_HAS_QUERY_HIP_SUPPORT)
+            if(SELF_REQUIRE_GPU_AWARE_MPI)
+                message(FATAL_ERROR "MPI installation is not HIP/ROCm-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
+            else()
+                message(WARNING "MPI installation is not HIP/ROCm-aware. GPU-direct communication will not be available.")
+            endif()
+        endif()
+        set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
+        set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
+        set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
+        set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
+        set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
+        if(SELF_ENABLE_DOUBLE_PRECISION)
+            set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
+            set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
+            set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
+            set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
+            set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        endif()
+        set( BACKEND_LIBRARIES hip::device roc::hipblas)
+        set( SELF_ACTIVE_GPU_BACKEND "HIP")
+
+    else() # CUDA
         message("-- CUDA found. Enabling CUDA language.")
         enable_language(CUDA)
-        if(MPI_HAS_QUERY_CUDA_SUPPORT)
-            set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
-            set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
-            set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
-            set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
-            set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
-
-            if(SELF_ENABLE_DOUBLE_PRECISION)
-                set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
-                set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
-                set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
-                set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
-                set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        if(NOT MPI_HAS_QUERY_CUDA_SUPPORT)
+            if(SELF_REQUIRE_GPU_AWARE_MPI)
+                message(FATAL_ERROR "MPI installation is not CUDA-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
+            else()
+                message(WARNING "MPI installation is not CUDA-aware. GPU-direct communication will not be available.")
             endif()
-
-            set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
-
-        else()
-            message( FATAL_ERROR "MPI installation is not GPU-aware" )
         endif()
+        set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
+        set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
+        set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
+        set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
+        set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
+        if(SELF_ENABLE_DOUBLE_PRECISION)
+            set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
+            set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
+            set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
+            set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
+            set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        endif()
+        set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
+        set( SELF_ACTIVE_GPU_BACKEND "CUDA")
+
     endif()
 endif()
 
+if(SELF_ENABLE_APU)
+    message("-- SELF Build System : Enabling APU Support")
+    set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_APU" )
+    set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_APU" )
+    set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_APU")
+    set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_APU")
+    set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_APU" )
+endif()
+
 # ----- end dependencies ----- #
 
 # Libraries

cmake/CheckMPIFeatures.cmake

@@ -98,6 +98,4 @@ function(CheckMPIFeatures)
 
     list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT)
   endif()
-endfunction()
-
-CheckMPIFeatures()
+endfunction()

src/CMakeLists.txt

@@ -30,11 +30,13 @@ if(SELF_ENABLE_GPU)
     file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.f*")
     file(GLOB SELF_BACKEND_CPPSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp*")
 
-    if(hip_FOUND)
+    if(SELF_ACTIVE_GPU_BACKEND STREQUAL "HIP")
       set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE HIP)
     else()
       set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE CUDA)
     endif()
+elseif(SELF_ENABLE_APU)
+    file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/apu/*.f*")
 else()
     file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/cpu/*.f*")
 endif()

test/CMakeLists.txt

@@ -38,6 +38,7 @@ function (add_fortran_tests)
         target_link_libraries(${TEST_NAME} self)
         target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
         add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
+	set_tests_properties(${TEST_NAME} PROPERTIES LABELS "serial")
         install(TARGETS ${TEST_NAME} DESTINATION test)
     endforeach ()
 
@@ -52,6 +53,7 @@ function (add_mpi_fortran_tests)
         target_link_libraries(${TEST_NAME} self)
         target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
         add_test(NAME ${TEST_NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SELF_MPIEXEC_NUMPROCS} ${SELF_MPIEXEC_OPTIONS} ${TEST_NAME})
+	set_tests_properties(${TEST_NAME} PROPERTIES LABELS "parallel")
 
         install(TARGETS ${TEST_NAME} DESTINATION test)
     endforeach ()
@@ -146,4 +148,4 @@ add_mpi_fortran_tests( "mappedvectordgdivergence_2d_linear_mpi.f90"
                        "advection_diffusion_2d_rk3_mpi.f90"
                        "advection_diffusion_2d_rk3_pickup_mpi.f90"
                        "advection_diffusion_3d_rk3_mpi.f90"
-                       "advection_diffusion_3d_rk3_pickup_mpi.f90" )
+                       "advection_diffusion_3d_rk3_pickup_mpi.f90" )