Skip to content

Commit 3bfa2c3

Browse files
ScSteffenScSteffen
authored
Rocm sn solver (#67)
* add mpi cuda * added rocm solver * formatting * added singularity rocm installer * attempted fix at rocm compile; * install singularity rocm scripts * runscripts rocm * added build tests * next fix attempt * attempt to fix ci * unit tests with mpiu suppport * attempt to fix rocm build * changed rocm container * disabled cuda unit test * update readme
1 parent 0bf7770 commit 3bfa2c3

14 files changed

Lines changed: 2759 additions & 40 deletions

‎.github/workflows/c-cpp.yml‎

Lines changed: 142 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,32 +8,145 @@ on:
88

99
jobs:
1010
unit_tests:
11-
runs-on: ubuntu-latest
12-
container: kitrt/test:latest
13-
environment: coverage
14-
env:
15-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
16-
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
17-
OMP_NUM_THREADS: 1
18-
OMP_DYNAMIC: FALSE
19-
OPENBLAS_NUM_THREADS: 1
20-
MKL_NUM_THREADS: 1
21-
22-
steps:
23-
- uses: actions/checkout@v2
24-
25-
- name: Build code
26-
run: |
27-
git submodule update --init --recursive
28-
mkdir build && cd build
29-
cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_ML=OFF -DBUILD_CODE_COV=ON ..
30-
ninja
31-
32-
- name: Run unit tests
33-
run: |
34-
cd build
35-
./unit_tests
36-
37-
- name: Code coverage
38-
run: |
39-
cpp-coveralls -r . -b "build/" -i "src/" -i "include/" --exclude "ext/" --gcov-options "\-lp" --verbose
11+
runs-on: ubuntu-latest
12+
container: kitrt/test:latest
13+
defaults:
14+
run:
15+
working-directory: ${{ github.workspace }}
16+
environment: coverage
17+
env:
18+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
19+
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
20+
OMP_NUM_THREADS: 1
21+
OMP_DYNAMIC: FALSE
22+
OPENBLAS_NUM_THREADS: 1
23+
MKL_NUM_THREADS: 1
24+
25+
steps:
26+
- uses: actions/checkout@v4
27+
with:
28+
submodules: recursive
29+
30+
- name: Build code
31+
run: |
32+
mkdir build && cd build
33+
cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_ML=OFF -DBUILD_CODE_COV=ON ..
34+
ninja
35+
36+
- name: Run unit tests
37+
run: |
38+
cd build
39+
./unit_tests
40+
41+
- name: Code coverage
42+
run: |
43+
cpp-coveralls -r . -b "build/" -i "src/" -i "include/" --exclude "ext/" --gcov-options "\-lp" --verbose
44+
45+
rocm_build:
46+
runs-on: ubuntu-latest
47+
container: rocm/dev-ubuntu-22.04:7.2
48+
env:
49+
OMP_NUM_THREADS: 1
50+
OMP_DYNAMIC: FALSE
51+
OPENBLAS_NUM_THREADS: 1
52+
MKL_NUM_THREADS: 1
53+
54+
steps:
55+
- name: Install ROCm build dependencies
56+
run: |
57+
apt-get update
58+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
59+
git \
60+
libopenmpi-dev \
61+
openmpi-bin \
62+
libblas-dev \
63+
liblapack-dev \
64+
cmake \
65+
ninja-build \
66+
libvtk9-dev
67+
68+
- uses: actions/checkout@v4
69+
with:
70+
submodules: recursive
71+
72+
- name: Build ROCm target
73+
working-directory: ${{ github.workspace }}
74+
run: |
75+
ROCM_CLANGXX="/opt/rocm-7.2.0/lib/llvm/bin/clang++"
76+
if [ ! -x "${ROCM_CLANGXX}" ] && [ -x "/opt/rocm/lib/llvm/bin/clang++" ]; then
77+
ROCM_CLANGXX="/opt/rocm/lib/llvm/bin/clang++"
78+
fi
79+
if [ ! -x "${ROCM_CLANGXX}" ] && [ -x "/opt/rocm/llvm/bin/clang++" ]; then
80+
ROCM_CLANGXX="/opt/rocm/llvm/bin/clang++"
81+
fi
82+
if [ ! -x "${ROCM_CLANGXX}" ]; then
83+
echo "Could not find ROCm clang++ compiler." >&2
84+
exit 1
85+
fi
86+
HIP_DIR=""
87+
for CANDIDATE in \
88+
"/opt/rocm-7.2.0/lib/cmake/hip" \
89+
"/opt/rocm/lib/cmake/hip" \
90+
"/opt/rocm/cmake/hip"
91+
do
92+
if [ -f "${CANDIDATE}/hip-config.cmake" ] || [ -f "${CANDIDATE}/hipConfig.cmake" ]; then
93+
HIP_DIR="${CANDIDATE}"
94+
break
95+
fi
96+
done
97+
if [ -z "${HIP_DIR}" ]; then
98+
echo "Could not find hip package config directory." >&2
99+
exit 1
100+
fi
101+
cmake -G Ninja -S . -B build_rocm \
102+
-DCMAKE_BUILD_TYPE=Release \
103+
-DCMAKE_CXX_COMPILER="${ROCM_CLANGXX}" \
104+
-DCMAKE_HIP_COMPILER="${ROCM_CLANGXX}" \
105+
-Dhip_DIR="${HIP_DIR}" \
106+
-DBUILD_MPI=ON \
107+
-DBUILD_CUDA_HPC=OFF \
108+
-DBUILD_HIP_HPC=ON \
109+
-DBUILD_ML=OFF
110+
cmake --build build_rocm -j2
111+
112+
cuda_build:
113+
runs-on: ubuntu-latest
114+
container: nvidia/cuda:12.4.1-devel-ubuntu22.04
115+
env:
116+
OMP_NUM_THREADS: 1
117+
OMP_DYNAMIC: FALSE
118+
OPENBLAS_NUM_THREADS: 1
119+
MKL_NUM_THREADS: 1
120+
121+
steps:
122+
- name: Install CUDA build dependencies
123+
run: |
124+
apt-get update
125+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
126+
git \
127+
libopenmpi-dev \
128+
openmpi-bin \
129+
libblas-dev \
130+
liblapack-dev \
131+
cmake \
132+
ninja-build \
133+
libvtk9-dev
134+
135+
- uses: actions/checkout@v4
136+
with:
137+
submodules: recursive
138+
139+
- name: Build CUDA target and tests
140+
working-directory: ${{ github.workspace }}
141+
run: |
142+
cmake -G Ninja -S . -B build_cuda \
143+
-DCMAKE_BUILD_TYPE=Release \
144+
-DBUILD_TESTING=ON \
145+
-DBUILD_MPI=ON \
146+
-DBUILD_CUDA_HPC=ON \
147+
-DBUILD_HIP_HPC=OFF \
148+
-DBUILD_ML=OFF \
149+
-DCMAKE_CUDA_ARCHITECTURES=70
150+
cmake --build build_cuda -j2
151+
152+

‎CMakeLists.txt‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
cmake_minimum_required( VERSION 3.16 )
2-
project( KiT-RT VERSION 0.1.0 LANGUAGES CXX )
2+
project( KiT-RT VERSION 1.2.0 LANGUAGES CXX )
33

44
### OPTIONS #####################################
55
option( BUILD_TESTING "builds all available unit tests" OFF )
@@ -10,12 +10,16 @@ option( BUILD_CODE_COV "enables compiler option required for code coverage analy
1010
option( BUILD_ML "enables build with tensorflow backend access" OFF )
1111
option( BUILD_MPI "enables build with MPI access" OFF )
1212
option( BUILD_CUDA_HPC "enables CUDA backend for SN HPC solver (MPI rank to GPU mapping)" OFF )
13+
option( BUILD_HIP_HPC "enables HIP/ROCm backend for SN HPC solver (MPI rank to GPU mapping)" OFF )
1314
#################################################
1415

1516

1617
### COMPILER ####################################
1718
set( CMAKE_CXX_STANDARD 17 )
1819
set( CMAKE_CXX_STANDARD_REQUIRED ON )
20+
if( BUILD_CUDA_HPC AND BUILD_HIP_HPC )
21+
message( FATAL_ERROR "BUILD_CUDA_HPC and BUILD_HIP_HPC cannot both be enabled." )
22+
endif()
1923
if( BUILD_CUDA_HPC )
2024
enable_language( CUDA )
2125
if( CMAKE_VERSION VERSION_LESS 3.18 )
@@ -27,9 +31,25 @@ if( BUILD_CUDA_HPC )
2731
set( CMAKE_CUDA_STANDARD_REQUIRED ON )
2832
set( CMAKE_CUDA_EXTENSIONS OFF )
2933
endif()
34+
if( BUILD_HIP_HPC )
35+
if( CMAKE_VERSION VERSION_LESS 3.21 )
36+
message( FATAL_ERROR "BUILD_HIP_HPC requires CMake >= 3.21 for HIP language support." )
37+
endif()
38+
enable_language( HIP )
39+
set( CMAKE_HIP_STANDARD 17 )
40+
set( CMAKE_HIP_STANDARD_REQUIRED ON )
41+
set( CMAKE_HIP_EXTENSIONS OFF )
42+
endif()
3043
set( KITRT_RELEASE_OPTIONS -march=native -w )
3144
set( KITRT_RELWITHDEBINFO_OPTIONS -march=native -pg -no-pie )
3245
set( KITRT_DEBUG_OPTIONS -Wall -Wextra -Wpedantic )
46+
if( BUILD_HIP_HPC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|X86_64|amd64|AMD64" )
47+
# ROCm clang can expose AVX-512 feature macros that trigger an incompatible
48+
# Blaze SIMD floor implementation path on some hosts.
49+
list( APPEND KITRT_RELEASE_OPTIONS -mno-avx512f -mno-avx512bw -mno-avx512dq )
50+
list( APPEND KITRT_RELWITHDEBINFO_OPTIONS -mno-avx512f -mno-avx512bw -mno-avx512dq )
51+
message( STATUS "HIP host C++ compile options: disabling AVX-512 for Blaze compatibility" )
52+
endif()
3353
if( BUILD_UNITY AND NOT BUILD_CODE_COV )
3454
message( STATUS "Unity build enabled" )
3555
set( CMAKE_UNITY_BUILD ON )
@@ -57,6 +77,14 @@ else()
5777
message( STATUS "CUDA HPC solver: disabled" )
5878
endif()
5979

80+
if( BUILD_HIP_HPC )
81+
add_compile_definitions( KITRT_ENABLE_HIP_HPC )
82+
find_package( hip REQUIRED CONFIG )
83+
message( STATUS "HIP HPC solver: enabled" )
84+
else()
85+
message( STATUS "HIP HPC solver: disabled" )
86+
endif()
87+
6088
message(STATUS "MPI build flag: ${BUILD_MPI}")
6189
if( BUILD_MPI )
6290
add_definitions(-DIMPORT_MPI)
@@ -113,6 +141,16 @@ if( BUILD_CUDA_HPC )
113141
endif()
114142
endif()
115143

144+
if( BUILD_HIP_HPC )
145+
if( TARGET hip::host )
146+
# Link only the host runtime to avoid propagating HIP device compile flags
147+
# (e.g. --offload-arch=...) into non-HIP C++ translation units.
148+
list( APPEND CORE_LIBRARIES hip::host )
149+
else()
150+
message( FATAL_ERROR "Could not find HIP runtime target (hip::host)." )
151+
endif()
152+
endif()
153+
116154

117155
#################################################
118156

@@ -136,6 +174,10 @@ if( BUILD_CUDA_HPC )
136174
# imported target features inject an older fallback standard.
137175
set_source_files_properties( src/solvers/snsolver_hpc.cu PROPERTIES COMPILE_FLAGS "--std=c++17" )
138176
endif()
177+
if( BUILD_HIP_HPC )
178+
list( APPEND SRCS "src/solvers/snsolver_hpc.hip" "include/solvers/snsolver_hpc_hip.hpp" )
179+
set_source_files_properties( src/solvers/snsolver_hpc.hip PROPERTIES LANGUAGE HIP )
180+
endif()
139181
set( EXCLUDE_DIR "/gui/" )
140182
foreach( TMP_PATH ${SRCS} )
141183
string( FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND )
@@ -163,6 +205,12 @@ if( BUILD_CUDA_HPC )
163205
PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
164206
)
165207
endif()
208+
if( BUILD_HIP_HPC )
209+
set_target_properties(
210+
${CMAKE_PROJECT_NAME}
211+
PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON HIP_EXTENSIONS OFF
212+
)
213+
endif()
166214
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
167215
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
168216
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
@@ -204,6 +252,12 @@ if( BUILD_TESTING )
204252
PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
205253
)
206254
endif()
255+
if( BUILD_HIP_HPC )
256+
set_target_properties(
257+
unit_tests
258+
PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON HIP_EXTENSIONS OFF
259+
)
260+
endif()
207261
target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
208262
if( BUILD_CODE_COV )
209263
if( CMAKE_COMPILER_IS_GNUCXX )

‎README.md‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ singularity exec tools/singularity/kit_rt_MPI.sif \
143143
mpirun -np 4 ./build_singularity_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
144144
```
145145

146-
### 3. CPU + CUDA (single or multi-GPU via MPI)
146+
### 3. CPU + GPU backend (single or multi-GPU via MPI)
147147

148-
#### 3a) Singularity installation
148+
#### 3a) CUDA (Singularity installation)
149149
```bash
150150
cd tools/singularity
151151
sudo singularity build kit_rt_MPI_cuda.sif kit_rt_MPI_cuda.def
@@ -162,6 +162,23 @@ singularity exec --nv tools/singularity/kit_rt_MPI_cuda.sif \
162162

163163
When compiled with `-DBUILD_CUDA_HPC=ON`, HPC runs use the CUDA backend if a GPU is visible, and fall back to CPU if no GPU is detected.
164164

165+
#### 3b) ROCm/HIP (Singularity installation)
166+
```bash
167+
cd tools/singularity
168+
sudo singularity build kit_rt_MPI_rocm72.sif kit_rt_MPI_rocm72.def
169+
cd ../..
170+
mkdir -p build_singularity_rocm72
171+
cd build_singularity_rocm72
172+
singularity exec --rocm ../tools/singularity/kit_rt_MPI_rocm72.sif \
173+
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_HIP_HPC=ON -DBUILD_ML=OFF ..
174+
singularity exec --rocm ../tools/singularity/kit_rt_MPI_rocm72.sif make -j
175+
cd ..
176+
singularity exec --rocm tools/singularity/kit_rt_MPI_rocm72.sif \
177+
mpirun -np 2 ./build_singularity_rocm72/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cuda_order2.cfg
178+
```
179+
180+
When compiled with `-DBUILD_HIP_HPC=ON`, HPC runs use the HIP backend if a ROCm-capable GPU is visible, and fall back to CPU if no GPU is detected.
181+
165182
### 4. Build with TensorFlow backend (CPU + OpenMP only)
166183

167184
```bash

‎examples/configs/readme.md‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

0 commit comments

Comments
 (0)