diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..5a685a2d6 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,54 @@ +steps: + - label: ":partyparrot: Build & Test AMD MI210 (Coverage)" + key: "build-test-mi210" + command: | + source /opt/spack-environment/activate.sh + export WORKSPACE=/workspace + mkdir -p /workspace/build + rocminfo + cd /workspace/build + FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \ + -DCMAKE_BUILD_TYPE="coverage" \ + -DSELF_ENABLE_GPU=ON \ + -DSELF_GPU_BACKEND=HIP \ + -DSELF_ENABLE_TESTING=ON \ + -DCMAKE_HIP_ARCHITECTURES="gfx90a" \ + -DGPU_TARGETS="gfx90a" \ + -DSELF_ENABLE_EXAMPLES=ON \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + /workspace/ + make -j + lcov --capture --initial \ + --directory /workspace/build/src/ \ + --output-file /workspace/initial.info + ctest --test-dir /workspace/build --output-on-failure + lcov --capture \ + --directory /workspace/build/src/ \ + --output-file /workspace/ctest-capture.info + lcov --add-tracefile /workspace/initial.info \ + --add-tracefile /workspace/ctest-capture.info \ + --output-file /workspace/coverage.info + env: + slurm_partition: "main" + slurm_gres: "gpu:mi210:2" + slurm_time: "01:00:00" + slurm_nodes: 1 + slurm_ntasks: 2 + slurm_cpus_per_task: 8 + slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a" + agents: + queue: "galapagos" + + - label: ":codecov: Upload Coverage" + key: "upload-coverage" + depends_on: "build-test-mi210" + command: | + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov -t "$${CODECOV_TOKEN}" \ + -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \ + -F buildkite-rocm-mi210 + secrets: + - CODECOV_TOKEN + agents: + queue: "galapagos" diff --git a/CMakeLists.txt b/CMakeLists.txt index e63bdea3a..d65cded32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,12 @@ option(SELF_ENABLE_MULTITHREADING "Option to enable CPU multithreading for `do c option(SELF_ENABLE_TESTING "Option to enable build of tests. (Default On)" ON) option(SELF_ENABLE_EXAMPLES "Option to enable build of examples. (Default On)" ON) option(SELF_ENABLE_GPU "Option to enable GPU backend. Requires either CUDA or HIP. (Default Off)" OFF) -option(SELF_ENABLE_APU "Option to enable APU backend. Requires either CUDA or HIP. (Default Off)" OFF) +option(SELF_ENABLE_APU "Option to enable APU backend. (Default Off)" OFF) + +set(SELF_GPU_BACKEND "AUTO" CACHE STRING "GPU backend to use when SELF_ENABLE_GPU is ON: AUTO, CUDA, or HIP. AUTO prefers HIP when available. (Default AUTO)") +set_property(CACHE SELF_GPU_BACKEND PROPERTY STRINGS AUTO CUDA HIP) + +option(SELF_REQUIRE_GPU_AWARE_MPI "Require GPU-aware MPI when building with GPU support. Set OFF to downgrade the check to a warning. (Default ON)" ON) option(SELF_ENABLE_DOUBLE_PRECISION "Option to enable double precision for floating point arithmetic. (Default On)" ON) set(SELF_MPIEXEC_NUMPROCS "2" CACHE STRING "The number of MPI ranks to use to launch MPI tests. Only used when launching test programs via ctest.") @@ -151,76 +156,103 @@ if(SELF_ENABLE_DOUBLE_PRECISION) endif() +if(SELF_ENABLE_GPU AND SELF_ENABLE_APU) + message(FATAL_ERROR "SELF_ENABLE_GPU and SELF_ENABLE_APU are mutually exclusive. Enable only one.") +endif() + if(SELF_ENABLE_GPU) - message("-- SELF Build System : Enabling GPU Support") + message("-- SELF Build System : Enabling GPU Support (backend: ${SELF_GPU_BACKEND})") set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_GPU" ) set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_GPU" ) set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_GPU") set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_GPU") set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_GPU" ) - # Check MPI for GPU awareness # Add SELF's cmake module directory to the search path set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") - - # Include the custom module include(CheckMPIFeatures) CheckMPIFeatures() - # HIP (Optional) - # IF HIP is found, we prefer HIP. IF HIP is not found, then we fall back to look for CUDA - # only. If neither HIP nor CUDA are found, Cmake will error out. - find_package(hip) - if(hip_FOUND) - if(MPI_HAS_QUERY_HIP_SUPPORT) - find_package(hipblas REQUIRED) - message("-- HIP found. Enabling HIP language.") - enable_language(HIP) - set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" ) - set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" ) - set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP") - set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP") - set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" ) - - if(SELF_ENABLE_DOUBLE_PRECISION) - set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" ) - set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" ) - set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION") - set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION") - set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" ) - endif() - set( BACKEND_LIBRARIES hip::device roc::hipblas) + # Resolve which backend to activate based on SELF_GPU_BACKEND + if(SELF_GPU_BACKEND STREQUAL "HIP") + find_package(hip REQUIRED) + set(_self_use_hip TRUE) + elseif(SELF_GPU_BACKEND STREQUAL "CUDA") + find_package(CUDAToolkit REQUIRED) + set(_self_use_hip FALSE) + else() # AUTO: prefer HIP, fall back to CUDA + find_package(hip) + if(hip_FOUND) + set(_self_use_hip TRUE) else() - message( FATAL_ERROR "MPI installation is not GPU-aware" ) + find_package(CUDAToolkit REQUIRED) + set(_self_use_hip FALSE) endif() - else() - # CUDA - find_package(CUDAToolkit REQUIRED) + endif() + + if(_self_use_hip) + find_package(hipblas REQUIRED) + message("-- HIP found. Enabling HIP language.") + enable_language(HIP) + if(NOT MPI_HAS_QUERY_HIP_SUPPORT) + if(SELF_REQUIRE_GPU_AWARE_MPI) + message(FATAL_ERROR "MPI installation is not HIP/ROCm-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.") + else() + message(WARNING "MPI installation is not HIP/ROCm-aware. GPU-direct communication will not be available.") + endif() + endif() + set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" ) + set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" ) + set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP") + set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP") + set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" ) + if(SELF_ENABLE_DOUBLE_PRECISION) + set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" ) + set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" ) + set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION") + set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION") + set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" ) + endif() + set( BACKEND_LIBRARIES hip::device roc::hipblas) + set( SELF_ACTIVE_GPU_BACKEND "HIP") + + else() # CUDA message("-- CUDA found. Enabling CUDA language.") enable_language(CUDA) - if(MPI_HAS_QUERY_CUDA_SUPPORT) - set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" ) - set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" ) - set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA") - set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA") - set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" ) - - if(SELF_ENABLE_DOUBLE_PRECISION) - set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" ) - set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" ) - set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION") - set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION") - set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" ) + if(NOT MPI_HAS_QUERY_CUDA_SUPPORT) + if(SELF_REQUIRE_GPU_AWARE_MPI) + message(FATAL_ERROR "MPI installation is not CUDA-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.") + else() + message(WARNING "MPI installation is not CUDA-aware. GPU-direct communication will not be available.") endif() - - set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas) - - else() - message( FATAL_ERROR "MPI installation is not GPU-aware" ) endif() + set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" ) + set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" ) + set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA") + set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA") + set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" ) + if(SELF_ENABLE_DOUBLE_PRECISION) + set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" ) + set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" ) + set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION") + set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION") + set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" ) + endif() + set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas) + set( SELF_ACTIVE_GPU_BACKEND "CUDA") + endif() endif() +if(SELF_ENABLE_APU) + message("-- SELF Build System : Enabling APU Support") + set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_APU" ) + set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_APU" ) + set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_APU") + set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_APU") + set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_APU" ) +endif() + # ----- end dependencies ----- # # Libraries diff --git a/cmake/CheckMPIFeatures.cmake b/cmake/CheckMPIFeatures.cmake index ea5550235..672f895bc 100644 --- a/cmake/CheckMPIFeatures.cmake +++ b/cmake/CheckMPIFeatures.cmake @@ -98,6 +98,4 @@ function(CheckMPIFeatures) list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT) endif() -endfunction() - -CheckMPIFeatures() \ No newline at end of file +endfunction() \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6dc5ca790..b4dbf925e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,11 +30,13 @@ if(SELF_ENABLE_GPU) file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.f*") file(GLOB SELF_BACKEND_CPPSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp*") - if(hip_FOUND) + if(SELF_ACTIVE_GPU_BACKEND STREQUAL "HIP") set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE HIP) else() set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE CUDA) endif() +elseif(SELF_ENABLE_APU) + file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/apu/*.f*") else() file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/cpu/*.f*") endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ea5a18e9f..19538a1d0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,6 +38,7 @@ function (add_fortran_tests) target_link_libraries(${TEST_NAME} self) target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include) add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "serial") install(TARGETS ${TEST_NAME} DESTINATION test) endforeach () @@ -52,6 +53,7 @@ function (add_mpi_fortran_tests) target_link_libraries(${TEST_NAME} self) target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include) add_test(NAME ${TEST_NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SELF_MPIEXEC_NUMPROCS} ${SELF_MPIEXEC_OPTIONS} ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "parallel") install(TARGETS ${TEST_NAME} DESTINATION test) endforeach () @@ -146,4 +148,4 @@ add_mpi_fortran_tests( "mappedvectordgdivergence_2d_linear_mpi.f90" "advection_diffusion_2d_rk3_mpi.f90" "advection_diffusion_2d_rk3_pickup_mpi.f90" "advection_diffusion_3d_rk3_mpi.f90" - "advection_diffusion_3d_rk3_pickup_mpi.f90" ) \ No newline at end of file + "advection_diffusion_3d_rk3_pickup_mpi.f90" )