Skip to content

Commit 395cbbb

Browse files
jameslehouxjameslehoux
authored
Merge pull request #188 from BASE-Laboratory/working
Major architecture overhaul: modular solvers, Fortran→C++ migration, GPU acceleration
2 parents 8a5cb92 + 743de5c commit 395cbbb

80 files changed

Lines changed: 8857 additions & 4160 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build-test.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,6 @@ jobs:
160160
-DCMAKE_BUILD_TYPE=Release \
161161
-DCMAKE_C_COMPILER=$(which mpicc) \
162162
-DCMAKE_CXX_COMPILER=$(which mpicxx) \
163-
-DCMAKE_Fortran_COMPILER=$(which mpif90) \
164163
-DCMAKE_PREFIX_PATH="/opt/amrex/25.03;/opt/hypre/v2.32.0;/opt/hdf5/1.12.3;/opt/libtiff/4.6.0" \
165164
-DBUILD_TESTING=ON
166165
@@ -207,7 +206,6 @@ jobs:
207206
-DCMAKE_BUILD_TYPE=Release \
208207
-DCMAKE_C_COMPILER=$(which mpicc) \
209208
-DCMAKE_CXX_COMPILER=$(which mpicxx) \
210-
-DCMAKE_Fortran_COMPILER=$(which mpif90) \
211209
-DPython_EXECUTABLE=/usr/bin/python3.11 \
212210
-Dpybind11_DIR=${PYBIND11_DIR} \
213211
-DCMAKE_PREFIX_PATH="/opt/amrex/25.03;/opt/hypre/v2.32.0;/opt/hdf5/1.12.3;/opt/libtiff/4.6.0" \
@@ -266,7 +264,6 @@ jobs:
266264
-DCMAKE_BUILD_TYPE=Debug \
267265
-DCMAKE_C_COMPILER=$(which mpicc) \
268266
-DCMAKE_CXX_COMPILER=$(which mpicxx) \
269-
-DCMAKE_Fortran_COMPILER=$(which mpif90) \
270267
-DCMAKE_PREFIX_PATH="/opt/amrex/25.03;/opt/hypre/v2.32.0;/opt/hdf5/1.12.3;/opt/libtiff/4.6.0" \
271268
-DBUILD_TESTING=ON \
272269
-DENABLE_COVERAGE=ON

.github/workflows/pypi-wheels.yml renamed to .github/workflows/pypi-wheels-cpu.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build and Publish OpenImpala Wheels
1+
name: "Build and Publish OpenImpala CPU Wheels"
22

33
on:
44
release:
@@ -48,7 +48,7 @@ jobs:
4848

4949
# Install all build dependencies inside the manylinux container.
5050
# Key points:
51-
# - gcc-gfortran is explicitly installed for Fortran kernel compilation
51+
# - gcc-gfortran is kept for AMReX's Fortran dependencies
5252
# - HDF5 and libtiff are built from source as static libraries so they
5353
# get linked into the wheel without auditwheel needing to vendor them
5454
# - HYPRE is built static (--enable-shared=no)
@@ -127,14 +127,13 @@ jobs:
127127
PATH="/usr/lib64/openmpi/bin:$PATH"
128128
CMAKE_C_COMPILER="mpicc"
129129
CMAKE_CXX_COMPILER="mpicxx"
130-
CMAKE_Fortran_COMPILER="mpif90"
131130
CMAKE_PREFIX_PATH="/usr/local"
132131
CMAKE_GENERATOR="Unix Makefiles"
133132
134133
# Vendor libraries into the wheel, but exclude host-specific MPI and
135134
# runtime libraries that users must provide on their system.
136135
# With all C/C++ deps statically linked, the only external shared libs
137-
# left are MPI, OpenMP runtime (libgomp), and Fortran runtime.
136+
# left are MPI and OpenMP runtime (libgomp).
138137
CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
139138
auditwheel repair -w {dest_dir} {wheel}
140139
--exclude libmpi.so

‎.github/workflows/pypi-wheels-gpu.yml‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
name: "Build and Publish OpenImpala GPU Wheels (CUDA)"
2+
3+
on:
4+
release:
5+
types:
6+
- published
7+
workflow_dispatch:
8+
9+
jobs:
10+
build_gpu_wheels:
11+
name: Build CUDA GPU wheels
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: Checkout repository
16+
uses: actions/checkout@v4
17+
with:
18+
submodules: recursive
19+
20+
- name: Set up Python
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: "3.x"
24+
25+
- name: Install cibuildwheel
26+
run: python -m pip install cibuildwheel==2.16.5
27+
28+
# Cache the compiled GPU dependencies (HDF5, libtiff, HYPRE+CUDA, AMReX+CUDA)
29+
# These take ~15-20 minutes to build from source with CUDA support.
30+
- name: Cache native GPU dependencies
31+
uses: actions/cache@v4
32+
with:
33+
path: .cibw-deps-cache
34+
key: cibw-deps-gpu-cuda12-x86_64-hdf5_1.14.6-tiff_4.6.0-hypre_2.31.0-amrex_25.03-v1
35+
36+
- name: Build GPU wheels
37+
run: python -m cibuildwheel --output-dir wheelhouse
38+
env:
39+
CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*"
40+
CIBW_SKIP: "*musllinux* *i686*"
41+
CIBW_ARCHS_LINUX: "x86_64"
42+
43+
# Use NVIDIA's CUDA-enabled manylinux image (CUDA 12.6, AlmaLinux 8)
44+
# This provides nvcc, CUDA runtime, and cuBLAS/cuSPARSE out of the box.
45+
CIBW_MANYLINUX_X86_64_IMAGE: sameli/manylinux_2_28_x86_64_cuda_12.6
46+
47+
# Build all dependencies with CUDA support.
48+
# HDF5 and libtiff are CPU-only (no GPU path needed).
49+
# HYPRE is built with --with-cuda for GPU-accelerated solves.
50+
# AMReX is built with -DAMReX_GPU_BACKEND=CUDA for device kernels.
51+
CIBW_BEFORE_ALL_LINUX: >
52+
dnf install -y epel-release &&
53+
dnf --enablerepo=powertools install -y
54+
openmpi-devel gcc-gfortran gcc-c++ wget git
55+
zlib-devel libjpeg-turbo-devel python3-pip &&
56+
pip3 install "cmake>=3.28,<4" &&
57+
export PATH=/usr/lib64/openmpi/bin:/usr/local/cuda/bin:$PATH &&
58+
export CUDA_HOME=/usr/local/cuda &&
59+
if [ -f /project/.cibw-deps-cache/deps.tar.gz ]; then
60+
echo "=== Restoring cached GPU dependencies ===" &&
61+
tar xzf /project/.cibw-deps-cache/deps.tar.gz -C / ;
62+
else
63+
echo "=== Building GPU dependencies from source ===" &&
64+
wget -q https://github.com/HDFGroup/hdf5/releases/download/hdf5_1.14.6/hdf5-1.14.6.tar.gz &&
65+
tar xzf hdf5-1.14.6.tar.gz &&
66+
cd hdf5-1.14.6 &&
67+
CC=mpicc CXX=mpicxx ./configure
68+
--prefix=/usr/local
69+
--enable-parallel
70+
--enable-cxx
71+
--enable-unsupported
72+
--disable-shared
73+
--with-pic &&
74+
make -j$(nproc) &&
75+
make install &&
76+
cd .. &&
77+
wget -q https://download.osgeo.org/libtiff/tiff-4.6.0.tar.gz &&
78+
tar xzf tiff-4.6.0.tar.gz &&
79+
cd tiff-4.6.0 &&
80+
cmake -S . -B build
81+
-DCMAKE_INSTALL_PREFIX=/usr/local
82+
-DCMAKE_BUILD_TYPE=Release
83+
-DBUILD_SHARED_LIBS=OFF
84+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON &&
85+
cmake --build build -j$(nproc) &&
86+
cmake --install build &&
87+
cd .. &&
88+
wget -q https://github.com/hypre-space/hypre/archive/v2.31.0.tar.gz &&
89+
tar xzf v2.31.0.tar.gz &&
90+
cd hypre-2.31.0/src &&
91+
./configure --prefix=/usr/local --with-MPI --with-cuda
92+
--with-cuda-home=/usr/local/cuda --enable-shared=no
93+
CC=mpicc CXX=mpicxx FC=mpif90
94+
CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" FFLAGS="-O2 -fPIC"
95+
CUDA_HOME=/usr/local/cuda &&
96+
make -j$(nproc) &&
97+
make install &&
98+
cd ../.. &&
99+
git clone --depth 1 --branch 25.03 https://github.com/AMReX-Codes/amrex.git /tmp/amrex &&
100+
cmake -S /tmp/amrex -B /tmp/amrex/build
101+
-DCMAKE_INSTALL_PREFIX=/usr/local
102+
-DCMAKE_BUILD_TYPE=Release
103+
-DBUILD_SHARED_LIBS=OFF
104+
-DAMReX_MPI=ON
105+
-DAMReX_OMP=ON
106+
-DAMReX_SPACEDIM=3
107+
-DAMReX_FORTRAN=ON
108+
-DAMReX_PARTICLES=OFF
109+
-DAMReX_GPU_BACKEND=CUDA
110+
-DAMReX_CUDA_ARCH=60;70;75;80;86;89;90
111+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
112+
-DCMAKE_CUDA_ARCHITECTURES="60;70;75;80;86;89;90" &&
113+
cmake --build /tmp/amrex/build -j$(nproc) &&
114+
cmake --install /tmp/amrex/build &&
115+
mkdir -p /project/.cibw-deps-cache &&
116+
tar czf /project/.cibw-deps-cache/deps.tar.gz /usr/local ;
117+
fi
118+
119+
CIBW_BEFORE_BUILD: pip install "cmake>=3.28,<4"
120+
121+
# Point to MPI, CUDA, and our compiled GPU dependencies.
122+
CIBW_ENVIRONMENT_LINUX: >
123+
PATH="/usr/lib64/openmpi/bin:/usr/local/cuda/bin:$PATH"
124+
CUDA_HOME="/usr/local/cuda"
125+
CMAKE_C_COMPILER="mpicc"
126+
CMAKE_CXX_COMPILER="mpicxx"
127+
CMAKE_PREFIX_PATH="/usr/local"
128+
CMAKE_GENERATOR="Unix Makefiles"
129+
CMAKE_ARGS="-DGPU_BACKEND=CUDA -DCMAKE_CUDA_ARCHITECTURES=60;70;75;80;86;89;90"
130+
131+
# Vendor libraries but exclude host-specific MPI, OpenMP, Fortran runtime,
132+
# and CUDA runtime libraries (users must have CUDA toolkit installed).
133+
CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
134+
auditwheel repair -w {dest_dir} {wheel}
135+
--exclude libmpi.so
136+
--exclude libmpi.so.12
137+
--exclude libmpi.so.40
138+
--exclude libmpi_cxx.so
139+
--exclude libmpi_cxx.so.1
140+
--exclude libmpi_cxx.so.40
141+
--exclude libopen-rte.so
142+
--exclude libopen-rte.so.40
143+
--exclude libopen-pal.so
144+
--exclude libopen-pal.so.40
145+
--exclude libmpi_mpifh.so
146+
--exclude libmpi_mpifh.so.40
147+
--exclude libgomp.so.1
148+
--exclude libgfortran.so.5
149+
--exclude libquadmath.so.0
150+
--exclude libcuda.so
151+
--exclude libcuda.so.1
152+
--exclude libcudart.so
153+
--exclude libcudart.so.12
154+
--exclude libcublas.so
155+
--exclude libcublas.so.12
156+
--exclude libcublasLt.so
157+
--exclude libcublasLt.so.12
158+
--exclude libcusparse.so
159+
--exclude libcusparse.so.12
160+
--exclude libcurand.so
161+
--exclude libcurand.so.10
162+
--exclude libnvJitLink.so
163+
--exclude libnvJitLink.so.12
164+
165+
- name: Upload wheels as artifacts
166+
uses: actions/upload-artifact@v4
167+
with:
168+
name: cibw-wheels-gpu
169+
path: ./wheelhouse/*.whl
170+
171+
publish_to_pypi:
172+
name: Publish GPU wheels to PyPI
173+
needs: build_gpu_wheels
174+
runs-on: ubuntu-latest
175+
environment: pypi
176+
permissions:
177+
id-token: write
178+
179+
steps:
180+
- name: Download wheel artifacts
181+
uses: actions/download-artifact@v4
182+
with:
183+
name: cibw-wheels-gpu
184+
path: dist/
185+
186+
- name: Publish to PyPI
187+
uses: pypa/gh-action-pypi-publish@release/v1
188+
with:
189+
skip-existing: true

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ jobs:
5555
-DCMAKE_BUILD_TYPE=Release \
5656
-DCMAKE_C_COMPILER=$(which mpicc) \
5757
-DCMAKE_CXX_COMPILER=$(which mpicxx) \
58-
-DCMAKE_Fortran_COMPILER=$(which mpif90) \
5958
-DCMAKE_PREFIX_PATH="/opt/amrex/25.03;/opt/hypre/v2.32.0;/opt/hdf5/1.12.3;/opt/libtiff/4.6.0" \
6059
-DBUILD_TESTING=OFF \
6160
-DCMAKE_INSTALL_PREFIX=/src/cmake_install && \
@@ -89,8 +88,8 @@ jobs:
8988
"$STAGING_DIR/usr/local" /usr/
9089
9190
%post
92-
# FIX #1: Install the CORRECT package that provides the gfortran runtime,
93-
# as determined by your research and diagnostic tests.
91+
# Install the gfortran runtime (required by AMReX/HYPRE dependencies,
92+
# not by OpenImpala itself which is pure C++).
9493
dnf install -y \
9594
gcc-toolset-11-gcc-gfortran \
9695
hwloc-libs \

‎CMakeLists.txt‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,29 @@ project(OpenImpala
2121
DESCRIPTION "Image-based simulation of transport properties in porous media"
2222
)
2323

24+
# ==============================================================================
25+
# GPU Acceleration
26+
# ==============================================================================
27+
# Set GPU_BACKEND to CUDA, HIP, or NONE (default) to enable GPU-accelerated
28+
# HYPRE solves. When enabled, AMReX and HYPRE must also be built with matching
29+
# GPU support. The CUDA or HIP language is added to the project automatically.
30+
set(GPU_BACKEND "NONE" CACHE STRING "GPU backend: NONE, CUDA, or HIP")
31+
set_property(CACHE GPU_BACKEND PROPERTY STRINGS NONE CUDA HIP)
32+
33+
if(GPU_BACKEND STREQUAL "CUDA")
34+
enable_language(CUDA)
35+
set(CMAKE_CUDA_STANDARD 17)
36+
set(CMAKE_CUDA_STANDARD_REQUIRED ON)
37+
add_compile_definitions(OPENIMPALA_USE_GPU OPENIMPALA_USE_CUDA)
38+
message(STATUS "GPU acceleration enabled: CUDA")
39+
elseif(GPU_BACKEND STREQUAL "HIP")
40+
enable_language(HIP)
41+
add_compile_definitions(OPENIMPALA_USE_GPU OPENIMPALA_USE_HIP)
42+
message(STATUS "GPU acceleration enabled: HIP")
43+
else()
44+
message(STATUS "GPU acceleration: disabled")
45+
endif()
46+
2447
# ==============================================================================
2548
# C++ Standard
2649
# ==============================================================================
@@ -55,10 +78,10 @@ FetchContent_Declare(
5578
FetchContent_MakeAvailable(nlohmann_json)
5679

5780
# --- MPI ---
58-
find_package(MPI REQUIRED COMPONENTS CXX Fortran)
81+
find_package(MPI REQUIRED COMPONENTS CXX)
5982

6083
# --- OpenMP ---
61-
find_package(OpenMP REQUIRED COMPONENTS CXX Fortran)
84+
find_package(OpenMP REQUIRED COMPONENTS CXX)
6285

6386
# --- AMReX ---
6487
# AMReX ships AMReXConfig.cmake when installed via CMake.

‎GNUmakefile‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ BUILD_DIRS := $(addprefix build/,$(MODULES))# build/io build/props
6969
# --- Find source files ---
7070
SOURCES_IO_ALL := $(wildcard src/io/*.cpp)
7171
SOURCES_PRP_ALL := $(wildcard src/props/*.cpp)
72-
SOURCES_F90_ALL := $(wildcard src/props/*.F90) # Assuming Fortran only in props
72+
# Fortran sources removed — all kernels are now native C++
7373

7474
# --- Test drivers (now in tests/ directory) ---
7575
SOURCES_TESTS := $(wildcard tests/t*.cpp)
@@ -80,7 +80,6 @@ SOURCES_APP_DRIVER := src/props/Diffusion.cpp
8080
# --- Identify Library Sources (excluding app driver and auxiliary files) ---
8181
SOURCES_IO_LIB := $(SOURCES_IO_ALL)
8282
SOURCES_PRP_LIB := $(filter-out $(SOURCES_APP_DRIVER) src/props/hypre_test.cpp, $(SOURCES_PRP_ALL))
83-
SOURCES_F90_LIB := $(SOURCES_F90_ALL) # Assuming all F90 are library code
8483

8584
# --- Define Object Files based on Categories ---
8685
OBJECTS_APP_DRIVER := $(patsubst src/props/%.cpp,$(PROPS_DIR)/%.o,$(SOURCES_APP_DRIVER))
@@ -89,10 +88,9 @@ TEST_EXECS := $(patsubst tests/%.cpp,$(TST_DIR)/%,$(SOURCES_TESTS))
8988

9089
OBJECTS_IO_LIB := $(patsubst src/io/%.cpp,$(IO_DIR)/%.o,$(SOURCES_IO_LIB))
9190
OBJECTS_PRP_LIB := $(patsubst src/props/%.cpp,$(PROPS_DIR)/%.o,$(SOURCES_PRP_LIB))
92-
OBJECTS_F90_LIB := $(patsubst src/props/%.F90,$(PROPS_DIR)/%.o,$(SOURCES_F90_LIB))
9391

9492
# --- Consolidate Library Objects ---
95-
OBJECTS_LIB_ALL := $(OBJECTS_IO_LIB) $(OBJECTS_PRP_LIB) $(OBJECTS_F90_LIB)
93+
OBJECTS_LIB_ALL := $(OBJECTS_IO_LIB) $(OBJECTS_PRP_LIB)
9694

9795
# Let Make search for source files in relevant directories
9896
VPATH := $(subst $(space),:,$(SRC_DIRS)):src
@@ -184,11 +182,6 @@ $(OBJECTS_APP_DRIVER): $(PROPS_DIR)/%.o: src/props/%.cpp
184182
@mkdir -p $(@D)
185183
$(CXX) $(CXXFLAGS) $(INCLUDE) -c $< -o $@
186184

187-
# Static Pattern Rule for Props Lib F90 objects
188-
$(OBJECTS_F90_LIB): $(PROPS_DIR)/%.o: src/props/%.F90
189-
@echo "Compiling (Props Fortran) $< ..."
190-
@mkdir -p $(@D) $(INC_DIR) # Ensure both obj and mod dirs exist
191-
$(F90) $(F90FLAGS) $(INCLUDE) -J$(INC_DIR) -c $< -o $@
192185

193186
# ============================================================
194187
# Linking Executables (Revised Rules - v4 style)

‎cmake/FindHYPRE.cmake‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#
88
# The following variables influence the search:
99
# HYPRE_ROOT / HYPRE_HOME / ENV{HYPRE_HOME}
10+
# GPU_BACKEND (from parent CMakeLists.txt) — selects GPU variant if available
1011
#
1112
# This module defines:
1213
# HYPRE_FOUND - True if HYPRE was found
@@ -28,9 +29,21 @@ find_path(HYPRE_INCLUDE_DIR
2829
PATH_SUFFIXES include
2930
)
3031

32+
# When a GPU backend is enabled, try the GPU-specific HYPRE library first.
33+
# HYPRE builds with --with-cuda produce libHYPRE.so that is GPU-enabled,
34+
# but some installations provide a separate libHYPRE_cuda or libHYPRE_hip.
35+
set(_hypre_lib_names HYPRE)
36+
if(DEFINED GPU_BACKEND)
37+
if(GPU_BACKEND STREQUAL "CUDA")
38+
list(PREPEND _hypre_lib_names HYPRE_cuda)
39+
elseif(GPU_BACKEND STREQUAL "HIP")
40+
list(PREPEND _hypre_lib_names HYPRE_hip)
41+
endif()
42+
endif()
43+
3144
# Look for the library
3245
find_library(HYPRE_LIBRARY
33-
NAMES HYPRE
46+
NAMES ${_hypre_lib_names}
3447
HINTS
3548
${HYPRE_ROOT}
3649
${HYPRE_HOME}

0 commit comments

Comments
 (0)