diff --git a/.github/hubcast.yml b/.github/hubcast.yml index bf7f35e28f..8989f9e6e8 100644 --- a/.github/hubcast.yml +++ b/.github/hubcast.yml @@ -1,21 +1,25 @@ -Repo: - # Required: organization or user that owns the repo on CZ GitLab - dest_org: sundials - - # Required: name of the destination repository on CZ GitLab - dest_name: sundials - - # Optional: name of the CI check as reported back to GitHub (default: gitlab-ci) - check_name: llnl-lc-gitlab-ci - - # Optional: granularity of CI statuses reported to GitHub (default: [pipeline]) - # Set to [pipeline] for overall pipeline status only - # Set to [jobs] for individual job statuses only - # Set to [pipeline, jobs] to report both - check_types: [pipeline, jobs] - - # Optional: delete branches from destination when source PR is closed (default: true) - delete_closed: true - - # Optional: sync draft PRs/MRs (default: true) - sync_drafts: true +# Hubcast configuration is disabled; CI runs on GitHub-hosted resources only. +# To re-enable mirroring to LLNL CZ GitLab (and the llnl-lc-gitlab-ci checks +# on LLNL clusters), uncomment the block below. +# +# Repo: +# # Required: organization or user that owns the repo on CZ GitLab +# dest_org: sundials +# +# # Required: name of the destination repository on CZ GitLab +# dest_name: sundials +# +# # Optional: name of the CI check as reported back to GitHub (default: gitlab-ci) +# check_name: llnl-lc-gitlab-ci +# +# # Optional: granularity of CI statuses reported to GitHub (default: [pipeline]) +# # Set to [pipeline] for overall pipeline status only +# # Set to [jobs] for individual job statuses only +# # Set to [pipeline, jobs] to report both +# check_types: [pipeline, jobs] +# +# # Optional: delete branches from destination when source PR is closed (default: true) +# delete_closed: true +# +# # Optional: sync draft PRs/MRs (default: true) +# sync_drafts: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed1a1fde8..25f2e720a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ ### New Features and Enhancements +Added a interface for the Re::Solve library (https://github.com/ORNL/ReSolve) and the +SUNMatrix_ReSolve class to use ReSolve matrices. + Added the function `SUNLogger_SetQueueAndFlushMsgFns` to allow for user-defined functions to queue and flush log messages. diff --git a/bindings/sundials4py/sundials/sundials_matrix_generated.hpp b/bindings/sundials4py/sundials/sundials_matrix_generated.hpp index 672fac632d..2eb2347aae 100644 --- a/bindings/sundials4py/sundials/sundials_matrix_generated.hpp +++ b/bindings/sundials4py/sundials/sundials_matrix_generated.hpp @@ -16,6 +16,7 @@ auto pyEnumSUNMatrix_ID = .value("SUNMATRIX_GINKGO", SUNMATRIX_GINKGO, "") .value("SUNMATRIX_GINKGOBATCH", SUNMATRIX_GINKGOBATCH, "") .value("SUNMATRIX_KOKKOSDENSE", SUNMATRIX_KOKKOSDENSE, "") + .value("SUNMATRIX_RESOLVE", SUNMATRIX_RESOLVE, "") .value("SUNMATRIX_CUSTOM", SUNMATRIX_CUSTOM, "") .export_values(); // #ifndef SWIG diff --git a/cmake/SundialsBuildOptionsPost.cmake b/cmake/SundialsBuildOptionsPost.cmake index 23df261a95..8ee00f746c 100644 --- a/cmake/SundialsBuildOptionsPost.cmake +++ b/cmake/SundialsBuildOptionsPost.cmake @@ -201,6 +201,13 @@ sundials_option( ADVANCED DEPRECATED_NAMES BUILD_SUNMATRIX_ONEMKLDENSE) list(APPEND SUNDIALS_BUILD_LIST "SUNDIALS_ENABLE_SUNMATRIX_ONEMKLDENSE") +sundials_option( + SUNDIALS_ENABLE_SUNMATRIX_RESOLVE BOOL + "Enable the SUNMATRIX_RESOLVE module (requires Re::Solve)" ON + DEPENDS_ON SUNDIALS_ENABLE_RESOLVE + ADVANCED DEPRECATED_NAMES BUILD_SUNMATRIX_RESOLVE) +list(APPEND SUNDIALS_BUILD_LIST "SUNDIALS_ENABLE_SUNMATRIX_RESOLVE") + sundials_option( SUNDIALS_ENABLE_SUNMATRIX_SLUNRLOC BOOL "Enable the SUNMATRIX_SLUNRLOC module (requires SuperLU_DIST)" ON diff --git a/cmake/SundialsSetupCompilers.cmake b/cmake/SundialsSetupCompilers.cmake index 3a82d9b841..58ffca0d6d 100644 --- a/cmake/SundialsSetupCompilers.cmake +++ b/cmake/SundialsSetupCompilers.cmake @@ -438,7 +438,8 @@ if(SUNDIALS_ENABLE_BENCHMARKS OR SUNDIALS_ENABLE_MAGMA OR SUNDIALS_ENABLE_GINKGO OR SUNDIALS_ENABLE_KOKKOS - OR SUNDIALS_ENABLE_ADIAK) + OR SUNDIALS_ENABLE_ADIAK + OR SUNDIALS_ENABLE_RESOLVE) include(SundialsSetupCXX) endif() diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 1bf49bee9d..21316890d8 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -87,6 +87,11 @@ foreach(backend ${SUNDIALS_MAGMA_BACKENDS}) set(SUNDIALS_MAGMA_BACKENDS_${backend} TRUE) endforeach() +# prepare substitution variable(s) SUNDIALS_RESOLVE_BACKENDS_* +foreach(backend ${SUNDIALS_RESOLVE_BACKENDS}) + set(SUNDIALS_RESOLVE_BACKENDS_${backend} TRUE) +endforeach() + # prepare substitution variable SUNDIALS_HAVE_POSIX_TIMERS for sundials_config.h if(SUNDIALS_POSIX_TIMERS) # set in SundialsPOSIXTimers.cmake set(SUNDIALS_HAVE_POSIX_TIMERS TRUE) diff --git a/cmake/SundialsSetupTPLs.cmake b/cmake/SundialsSetupTPLs.cmake index 44b60d26ee..ab7f7c75e5 100644 --- a/cmake/SundialsSetupTPLs.cmake +++ b/cmake/SundialsSetupTPLs.cmake @@ -170,6 +170,15 @@ if(SUNDIALS_ENABLE_RAJA) list(APPEND SUNDIALS_TPL_LIST "RAJA") endif() +# --------------------------------------------------------------- +# Find (and test) the ReSolve libraries +# --------------------------------------------------------------- + +if(SUNDIALS_ENABLE_RESOLVE) + include(SundialsReSolve) + list(APPEND SUNDIALS_TPL_LIST "RESOLVE") +endif() + # --------------------------------------------------------------- # Find (and test) the SuperLUDIST libraries # --------------------------------------------------------------- diff --git a/cmake/SundialsTPLOptions.cmake b/cmake/SundialsTPLOptions.cmake index f5c5f2360d..f92bbef89d 100644 --- a/cmake/SundialsTPLOptions.cmake +++ b/cmake/SundialsTPLOptions.cmake @@ -560,3 +560,29 @@ sundials_option( DEPRECATED_NAMES KOKKOS_KERNELS_WORKS NEGATE_DEPRECATED) + +# --------------------------------------------------------------- +# Enable ReSolve support? +# --------------------------------------------------------------- + +sundials_option(SUNDIALS_ENABLE_RESOLVE BOOL "Enable ReSolve support" OFF) + +sundials_option( + SUNDIALS_RESOLVE_BACKENDS + STRING + "Which ReSolve backend to use under the SUNDIALS ReSolve interfaces (CPU, CUDA, HIP)" + "CPU" + OPTIONS "CPU;CUDA;HIP" + DEPENDS_ON SUNDIALS_ENABLE_RESOLVE) + +sundials_option(ReSolve_DIR PATH "Path to the root of a ReSolve installation" + "${ReSolve_DIR}") + +sundials_option(ReSolve_INCLUDE_DIR PATH "ReSolve include directory" + "${ReSolve_INCLUDE_DIR}" ADVANCED) + +sundials_option(ReSolve_LIBRARY_DIR PATH "ReSolve library directory" + "${ReSolve_LIBRARY_DIR}" ADVANCED) + +sundials_option(SUNDIALS_ENABLE_RESOLVE_CHECKS BOOL + "Enable ReSolve compatibility checks" ON ADVANCED) diff --git a/cmake/tpl/FindReSolve.cmake b/cmake/tpl/FindReSolve.cmake new file mode 100644 index 0000000000..7d3ffc738d --- /dev/null +++ b/cmake/tpl/FindReSolve.cmake @@ -0,0 +1,127 @@ +# ------------------------------------------------------------------------------ +# Programmer(s): Slaven Peles @ ORNL +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2025-2026, Lawrence Livermore National Security, +# University of Maryland Baltimore County, and the SUNDIALS contributors. +# Copyright (c) 2013-2025, Lawrence Livermore National Security +# and Southern Methodist University. +# Copyright (c) 2002-2013, Lawrence Livermore National Security. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ +# ReSolve find module that creates an imported target for ReSolve. +# The target is SUNDIALS::ReSolve. +# +# The variable ReSolve_DIR can be used to control where the module +# looks for the library (path to the root of a ReSolve installation or +# to a directory containing ReSolveConfig.cmake). +# +# The variable ReSolve_INCLUDE_DIR can be used to set the include path. +# The variable ReSolve_LIBRARY_DIR can be used to set the library path. +# +# This module also defines variables, but it is best to use the defined +# target to ensure includes and compile/link options are correctly passed +# to consumers. +# +# ReSolve_FOUND - system has the ReSolve library +# ReSolve_LIBRARY - the ReSolve library +# ReSolve_INCLUDE_DIR - the ReSolve include path +# ReSolve_LIBRARIES - all libraries needed for ReSolve +# ------------------------------------------------------------------------------ + +# Prefer the upstream CMake config file if the user did not point to a specific +# include/library directory. + +if(NOT + (ReSolve_INCLUDE_DIR + OR ReSolve_LIBRARY_DIR + OR ReSolve_LIBRARY)) + + find_package( + ReSolve + CONFIG + QUIET + PATHS + "${ReSolve_DIR}" + PATH_SUFFIXES + lib/cmake/ReSolve + cmake/ReSolve) + + if(ReSolve_FOUND AND TARGET ReSolve::ReSolve) + if(NOT TARGET SUNDIALS::ReSolve) + add_library(SUNDIALS::ReSolve ALIAS ReSolve::ReSolve) + endif() + + # Check for CUDA backend + if(TARGET ReSolve::CUDA) + set(RESOLVE_CUDA_FOUND + TRUE + CACHE BOOL "ReSolve CUDA backend found") + if(NOT TARGET SUNDIALS::ReSolve_CUDA) + add_library(SUNDIALS::ReSolve_CUDA ALIAS ReSolve::resolve_backend_cuda) + endif() + endif() + + # Check for HIP backend + if(TARGET ReSolve::HIP) + set(RESOLVE_HIP_FOUND + TRUE + CACHE BOOL "ReSolve HIP backend found") + if(NOT TARGET SUNDIALS::ReSolve_HIP) + add_library(SUNDIALS::ReSolve_HIP ALIAS ReSolve::resolve_backend_hip) + endif() + endif() + return() + endif() + +endif() + +# Fall back to manual detection using ReSolve_DIR, ReSolve_INCLUDE_DIR, and +# ReSolve_LIBRARY_DIR. + +# Find the include directory +find_path( + ReSolve_INCLUDE_DIR resolve/SystemSolver.hpp + PATHS "${ReSolve_DIR}" + PATH_SUFFIXES include + DOC "ReSolve include directory") + +if(ReSolve_LIBRARY) + get_filename_component(ReSolve_LIBRARY_DIR "${ReSolve_LIBRARY}" PATH) + set(ReSolve_LIBRARY_DIR + "${ReSolve_LIBRARY_DIR}" + CACHE PATH "" FORCE) +else() + find_library( + ReSolve_LIBRARY resolve + PATHS "${ReSolve_DIR}" "${ReSolve_LIBRARY_DIR}" + PATH_SUFFIXES lib lib64 + DOC "ReSolve library") +endif() +mark_as_advanced(ReSolve_LIBRARY) + +set(ReSolve_LIBRARIES "${ReSolve_LIBRARY}") + +# Set package variables including ReSolve_FOUND +find_package_handle_standard_args( + ReSolve REQUIRED_VARS ReSolve_LIBRARY ReSolve_LIBRARIES ReSolve_INCLUDE_DIR) + +# Create the SUNDIALS::ReSolve imported target +if(ReSolve_FOUND) + + if(NOT TARGET SUNDIALS::ReSolve) + add_library(SUNDIALS::ReSolve UNKNOWN IMPORTED) + endif() + + set_target_properties( + SUNDIALS::ReSolve + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ReSolve_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${ReSolve_LIBRARIES}" + IMPORTED_LOCATION "${ReSolve_LIBRARY}") + +endif() diff --git a/cmake/tpl/SundialsReSolve.cmake b/cmake/tpl/SundialsReSolve.cmake new file mode 100644 index 0000000000..0e553df60d --- /dev/null +++ b/cmake/tpl/SundialsReSolve.cmake @@ -0,0 +1,90 @@ +# ----------------------------------------------------------------------------- +# Programmer(s): Slaven Peles @ ORNL +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2025-2026, Lawrence Livermore National Security, +# University of Maryland Baltimore County, and the SUNDIALS contributors. +# Copyright (c) 2013-2025, Lawrence Livermore National Security +# and Southern Methodist University. +# Copyright (c) 2002-2013, Lawrence Livermore National Security. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- +# Module to find and setup ReSolve. +# ReSolve is a GPU-friendly linear solver library developed at ORNL: +# https://github.com/ORNL/ReSolve +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# Section 1: Include guard +# ----------------------------------------------------------------------------- + +include_guard(GLOBAL) + +# ----------------------------------------------------------------------------- +# Section 2: Check to make sure options are compatible +# ----------------------------------------------------------------------------- + +# ReSolve is a C++ library; CXX must be enabled (SundialsSetupCompilers.cmake +# gates include(SundialsSetupCXX) on SUNDIALS_ENABLE_RESOLVE). +if(NOT CMAKE_CXX_COMPILER_LOADED) + message(FATAL_ERROR "ReSolve requires C++ but no C++ compiler was found. " + "Enable a C++ compiler or set CMAKE_CXX_COMPILER.") +endif() + +if(CMAKE_CXX_STANDARD LESS "14") + message(FATAL_ERROR "CMAKE_CXX_STANDARD must be >= 14 when using ReSolve") +endif() + +# ----------------------------------------------------------------------------- +# Section 3: Find the TPL +# ----------------------------------------------------------------------------- + +find_package(ReSolve REQUIRED) + +message(STATUS "ReSolve_LIBRARIES: ${ReSolve_LIBRARIES}") +message(STATUS "ReSolve_INCLUDE_DIR: ${ReSolve_INCLUDE_DIR}") +message(STATUS "SUNDIALS_RESOLVE_BACKENDS: ${SUNDIALS_RESOLVE_BACKENDS}") + +# ----------------------------------------------------------------------------- +# Section 4: Test the TPL +# ----------------------------------------------------------------------------- + +if(SUNDIALS_ENABLE_RESOLVE_CHECKS) + + message(CHECK_START "Testing ReSolve") + + set(TEST_DIR ${PROJECT_BINARY_DIR}/RESOLVE_TEST) + + # Use the self-contained Common.hpp rather than SystemSolver.hpp; the latter + # has missing internal includes in some ReSolve versions. + file( + WRITE ${TEST_DIR}/test.cpp + "\#include \n" "int main(void) {\n" + " ReSolve::real_type x = ReSolve::constants::ONE;\n" " (void)x;\n" + " return 0;\n" "}\n") + + try_compile( + COMPILE_OK ${TEST_DIR} + ${TEST_DIR}/test.cpp + LINK_LIBRARIES ReSolve::ReSolve + OUTPUT_VARIABLE COMPILE_OUTPUT) + + if(COMPILE_OK) + message(CHECK_PASS "success") + else() + message(CHECK_FAIL "failed") + file(WRITE ${TEST_DIR}/compile.out "${COMPILE_OUTPUT}") + message( + FATAL_ERROR + "Could not compile ReSolve test. Check output in ${TEST_DIR}/compile.out" + ) + endif() + +else() + message(STATUS "Skipped ReSolve checks.") +endif() diff --git a/doc/shared/RecentChanges.rst b/doc/shared/RecentChanges.rst index 39bc58019a..c4b3b580c0 100644 --- a/doc/shared/RecentChanges.rst +++ b/doc/shared/RecentChanges.rst @@ -5,6 +5,9 @@ **New Features and Enhancements** +Added a interface for the Re::Solve library (https://github.com/ORNL/ReSolve) +and the SUNMatrix_ReSolve class to use ReSolve matrices. + Added the function :c:func:`SUNLogger_SetQueueAndFlushMsgFns` to allow for user-defined functions to queue and flush log messages. diff --git a/doc/shared/sundials/Install.rst b/doc/shared/sundials/Install.rst index b08d2627ba..e4022ae6a1 100644 --- a/doc/shared/sundials/Install.rst +++ b/doc/shared/sundials/Install.rst @@ -2127,6 +2127,65 @@ the CUDA backend (targeting Ampere GPUs): Default: ``CUDA`` +.. _Installation.Options.ReSolve: + +Building with Re\:\:Solve +^^^^^^^^^^^^^^^^^^^ + +The `Re\:\:Solve library `__ is a library of +GPU-resident linear solvers. It contains iterative and direct solvers designed to run +on NVIDIA and AMD GPUs, as well as on CPU devices. The library is +developed by Oak Ridge National Laboratory and is available from the `Re::Solve GitHub +repository `__. + +When Re::Solve support is enabled, :ref:`Re::Solve sparse SUNMatrix +` will be built (see section +:numref:`Installation.LibrariesAndHeaders.Matrix.ReSolve` for the corresponding +header files and libraries). + +To enable Re::Solve support, set :cmakeop:`SUNDIALS_ENABLE_RESOLVE` to ``ON``, +:cmakeop:`ReSolve_DIR` to the root path of a Re::Solve installation, and +:cmakeop:`SUNDIALS_RESOLVE_BACKENDS` to the desired Re::Solve backend to use. + +.. code-block:: bash + + cmake \ + -S SOLVER_DIR \ + -B BUILD_DIR \ + -D CMAKE_INSTALL_PREFIX=INSTALL_DIR \ + -D SUNDIALS_ENABLE_RESOLVE=ON \ + -D ReSolve_DIR=/path/to/resolve/installation \ + -D SUNDIALS_RESOLVE_BACKENDS=CUDA \ + -D SUNDIALS_ENABLE_CUDA=ON + +Optionally, a specific CUDA architecture can be set using `-D CMAKE_CUDA_ARCHITECTURES="80"`. + +.. cmakeoption:: SUNDIALS_ENABLE_RESOLVE + + Enable Re\:\:Solve support + + Default: ``OFF`` + +.. cmakeoption:: ReSolve_DIR + + Path to the Re\:\:Solve installation + + Default: None + +.. cmakeoption:: SUNDIALS_RESOLVE_BACKENDS + + Which Re\:\:Solve backend to use under the SUNDIALS Re\:\:Solve interface: ``CPU`` ``CUDA`` or + ``HIP`` + + Default: ``CPU`` + +.. cmakeoption:: SUNDIALS_ENABLE_RESOLVE_CHECKS + + Perform Re::Solve compatibility checks + + Default: ``OFF`` + + .. _Installation.Options.SuperLU_DIST: Building with SuperLU_DIST @@ -3681,6 +3740,25 @@ header file and link to the library given below. | CMake target | ``SUNDIALS::sunmatrixonemkldense`` | +--------------+----------------------------------------------+ +.. _Installation.LibrariesAndHeaders.Matrix.ReSolve: + +Re::Solve +"""""""""""" + +To use the :ref:`Re\:\:Solve sparse SUNMatrix `, include the +header file and link to the library given below. + +.. table:: The sparse Re::Solve SUNMatrix library, header file, and CMake target + :align: center + + +--------------+----------------------------------------------+ + | Libraries | ``libsundials_sunmatrixresolve.LIB`` | + +--------------+----------------------------------------------+ + | Headers | ``sunmatrix/sunmatrix_resolve.h`` | + +--------------+----------------------------------------------+ + | CMake target | ``SUNDIALS::sunmatrixresolve`` | + +--------------+----------------------------------------------+ + .. _Installation.LibrariesAndHeaders.Matrix.Sparse: Sparse diff --git a/doc/shared/sunmatrix/SUNMatrix_ReSolve.rst b/doc/shared/sunmatrix/SUNMatrix_ReSolve.rst new file mode 100644 index 0000000000..901d0b9cb2 --- /dev/null +++ b/doc/shared/sunmatrix/SUNMatrix_ReSolve.rst @@ -0,0 +1,179 @@ +.. + Programmer(s): Jeffery Zhang + ---------------------------------------------------------------- + SUNDIALS Copyright Start + Copyright (c) 2025-2026, Lawrence Livermore National Security, + University of Maryland Baltimore County, and the SUNDIALS contributors. + Copyright (c) 2013-2025, Lawrence Livermore National Security + and Southern Methodist University. + Copyright (c) 2002-2013, Lawrence Livermore National Security. + All rights reserved. + + See the top-level LICENSE and NOTICE files for details. + + SPDX-License-Identifier: BSD-3-Clause + SUNDIALS Copyright End + ---------------------------------------------------------------- + +.. _SUNMatrix.ReSolve: + +The SUNMATRIX_RESOLVE Module +====================================== + +The SUNMATRIX_RESOLVE module interfaces to the +`Re::Solve `_ library of GPU-resident linear +solvers designed to run on NVIDIA and AMD GPUs as well as on CPU devices. The module stores +matrices in the *compressed-sparse-row* (CSR) sparse matrix format. + +More general information about sparse matrices can be found in the SUNMatrix_Sparse documentation +described in :numref:`SUNMatrix.Sparse`. + + +The header file to include when using this module is ``sunmatrix/sunmatrix_resolve.h``. +The installed library to link to is ``libsundials_sunmatrixresolve.lib`` where ``lib`` is +typically ``.so`` for shared libraries and ``.a`` for static libraries. + + +.. _SUNMatrix.ReSolve.functions: + +SUNMATRIX_RESOLVE Functions +----------------------------------- + + +The SUNMATRIX_RESOLVE module currently defines the following implementations +of matrix operations listed in :numref:`SUNMatrix.Ops`. + +* ``SUNMatGetID_ReSolve`` -- returns ``SUNMATRIX_RESOLVE`` +* ``SUNMatClone_ReSolve`` +* ``SUNMatDestroy_ReSolve`` +* ``SUNMatZero_ReSolve`` + +In addition, the SUNMATRIX_RESOLVE module defines the following +implementation specific functions: + +.. cpp:function:: SUNMatrix SUNMatrix_ReSolve(sunindextype M, sunindextype N, sunindextype NNZ, ReSolve::memory::MemorySpace memspace, SUNContext sunctx) + + This constructor function creates and allocates memory for an + :math:`M \times N` SUNMATRIX_RESOLVE ``SUNMatrix``. + + **Arguments:** + * *M* -- the number of matrix rows. + * *N* -- the number of matrix columns. + * *NNZ* -- the number of non-zeros. + * *memspace* -- A Re::Solve variable used to choose between performing an operation on + either the HOST and DEVICE. Choose ``ReSolve::memory::HOST`` to use CPU devices or + ``ReSolve::memory::DEVICE`` to use NVIDIA or AMD GPUs. + * *sunctx* -- the :c:type:`SUNContext` object (see :numref:`SUNDIALS.SUNContext`) + + **Return value:** + If successful, a ``SUNMatrix`` object otherwise ``NULL``. + +.. cpp:function:: sunindextype SUNMatrix_ReSolve_Rows(SUNMatrix A) + + This function returns the number of rows in the ``SUNMatrix`` object. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + + **Return value:** + The number of rows in the ``SUNMatrix`` object. + + +.. cpp:function:: sunindextype SUNMatrix_ReSolve_Columns(SUNMatrix A) + + This function returns the number of columns in the ``SUNMatrix`` object. + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + + **Return value:** + The number of columns in the ``SUNMatrix`` object. + + +.. cpp:function:: sunindextype SUNMatrix_ReSolve_NNZ(SUNMatrix A) + + This function returns the number of nonzeros in a ``SUNMatrix`` + object. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + + **Return value:** + The number of nonzeros in the ``SUNMatrix`` object. + + +.. cpp:function:: sunrealtype* SUNMatrix_ReSolve_Data(SUNMatrix A, ReSolve::memory::MemorySpace memspace) + + This function returns a pointer to the nonzero entries of the ``SUNMatrix`` + object on either the HOST or the DEVICE. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *memspace* -- Either ``ReSolve::memory::HOST`` or ``ReSolve::memory::DEVICE`` + + **Return value:** + A pointer to the array of the nonzero entries of the ``SUNMatrix`` object on either + the HOST or DEVICE. + +.. cpp:function:: sunindextype* SUNMatrix_ReSolve_IndexValues(SUNMatrix A, ReSolve::memory::MemorySpace memspace) + + This function returns a pointer to the array of column indices of the ``SUNMatrix`` + object on either the HOST or the DEVICE. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *memspace* -- Either ``ReSolve::memory::HOST`` or ``ReSolve::memory::DEVICE`` + + **Return value:** + A pointer to the array of column indices of the ``SUNMatrix`` object on either + the HOST or DEVICE. + +.. cpp:function:: sunindextype* SUNMatrix_ReSolve_IndexPointers(SUNMatrix A, ReSolve::memory::MemorySpace memspace) + + This function returns a pointer to the array of row pointers of the ``SUNMatrix`` + object on either the HOST or the DEVICE. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *memspace* -- Either ``ReSolve::memory::HOST`` or ``ReSolve::memory::DEVICE`` + + **Return value:** + A pointer to the array of row pointers of the ``SUNMatrix`` object on either + the HOST or DEVICE. + +.. cpp:function:: SUNErrCode SUNMatrix_ReSolve_SetUpdated(SUNMatrix A, ReSolve::memory::MemorySpace memspace) + + This function is used to set a Re::Solve memory space to "updated". The other data + mirror is also automatically set as non-updated. + This function should be used if matrix data is updated through access of the raw pointers + as the matrix has no way of knowing which data is most recent otherwise. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *memspace* -- Either ``ReSolve::memory::HOST`` or ``ReSolve::memory::DEVICE`` + + **Return value:** + ``SUN_SUCCESS`` if the update is successful. + +.. cpp:function:: SUNErrCode SUNMatrix_ReSolve_SyncData(SUNMatrix A, ReSolve::memory::MemorySpace memspace) + + This function is used to sync data in the given memory space with the updated memory space. + + The memory space other than the the given memory space must be up-to-date. Otherwise, + the function will return an error. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *memspace* -- Either ``ReSolve::memory::HOST`` or ``ReSolve::memory::DEVICE`` + + **Return value:** + * ``SUN_SUCCESS`` if the sync is successful. + * ``SUN_ERR_ARG_INCOMPATIBLE`` if the given memory space is not updated. + +.. cpp:function:: void SUNMatrix_ReSolve_Print(SUNMatrix A, std::ostream& out, sunindextype indexing_base) + + This function can be used to print the ``SUNMatrix`` object. + + **Arguments:** + * *A* -- a ``SUNMatrix`` object. + * *out* -- reference specifying where output should be written + * *indexing_base* -- value specifying the indexing base \ No newline at end of file diff --git a/doc/superbuild/source/sunmatrix/SUNMatrix_links.rst b/doc/superbuild/source/sunmatrix/SUNMatrix_links.rst index 187ed6b53d..68de41c22f 100644 --- a/doc/superbuild/source/sunmatrix/SUNMatrix_links.rst +++ b/doc/superbuild/source/sunmatrix/SUNMatrix_links.rst @@ -23,4 +23,5 @@ .. include:: ../../../shared/sunmatrix/SUNMatrix_Ginkgo.rst .. include:: ../../../shared/sunmatrix/SUNMatrix_GinkgoBatch.rst .. include:: ../../../shared/sunmatrix/SUNMatrix_KokkosDense.rst +.. include:: ../../../shared/sunmatrix/SUNMatrix_ReSolve.rst .. include:: ../../../shared/sunmatrix/SUNMatrix_Examples.rst diff --git a/docker/sundials-ci/spack-nightly/int32-double/spack.yaml b/docker/sundials-ci/spack-nightly/int32-double/spack.yaml index 4291a71a27..388484cf52 100644 --- a/docker/sundials-ci/spack-nightly/int32-double/spack.yaml +++ b/docker/sundials-ci/spack-nightly/int32-double/spack.yaml @@ -20,6 +20,7 @@ spack: - superlu-dist~int64 ^parmetis~int64 arch=x86_64 %gcc@9.4.0 - trilinos+tpetra gotype=int arch=x86_64 %gcc@9.4.0 - xbraid arch=x86_64 %gcc@9.4.0 + # - resolve~cuda~rocm arch=x86_64 %gcc@9.4.0 config: install_tree: /opt/software mirrors: diff --git a/docker/sundials-ci/spack-nightly/int64-double/spack.yaml b/docker/sundials-ci/spack-nightly/int64-double/spack.yaml index 2e0f859aa6..d434efec89 100644 --- a/docker/sundials-ci/spack-nightly/int64-double/spack.yaml +++ b/docker/sundials-ci/spack-nightly/int64-double/spack.yaml @@ -20,6 +20,7 @@ spack: - superlu-dist+int64 ^parmetis+int64 ^netlib-lapack arch=x86_64 %gcc@9.4.0 - trilinos+tpetra gotype=long_long arch=x86_64 %gcc@9.4.0 - xbraid arch=x86_64 %gcc@9.4.0 + # - resolve~cuda~rocm arch=x86_64 %gcc@9.4.0 config: install_tree: /opt/software mirrors: diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 662807c61d..75adbbedca 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -236,6 +236,11 @@ #cmakedefine SUNDIALS_MAGMA_BACKENDS_CUDA #cmakedefine SUNDIALS_MAGMA_BACKENDS_HIP +/* ReSolve backends */ +#cmakedefine SUNDIALS_RESOLVE_BACKENDS_CPU +#cmakedefine SUNDIALS_RESOLVE_BACKENDS_CUDA +#cmakedefine SUNDIALS_RESOLVE_BACKENDS_HIP + /* Set if SUNDIALS is built with MPI support, then * #define SUNDIALS_MPI_ENABLED 1 * otherwise diff --git a/include/sundials/sundials_matrix.h b/include/sundials/sundials_matrix.h index e89dede7c0..6dedeff29e 100644 --- a/include/sundials/sundials_matrix.h +++ b/include/sundials/sundials_matrix.h @@ -72,6 +72,7 @@ enum SUNMatrix_ID SUNMATRIX_GINKGO, SUNMATRIX_GINKGOBATCH, SUNMATRIX_KOKKOSDENSE, + SUNMATRIX_RESOLVE, SUNMATRIX_CUSTOM }; diff --git a/include/sunmatrix/sunmatrix_resolve.h b/include/sunmatrix/sunmatrix_resolve.h new file mode 100644 index 0000000000..931d9d98f3 --- /dev/null +++ b/include/sunmatrix/sunmatrix_resolve.h @@ -0,0 +1,96 @@ +/* + * ----------------------------------------------------------------- + * Programmer(s): Jeffery Zhang + * ----------------------------------------------------------------- + * SUNDIALS Copyright Start + * Copyright (c) 2025-2026, Lawrence Livermore National Security, + * University of Maryland Baltimore County, and the SUNDIALS contributors. + * Copyright (c) 2013-2025, Lawrence Livermore National Security + * and Southern Methodist University. + * Copyright (c) 2002-2013, Lawrence Livermore National Security. + * All rights reserved. + * + * See the top-level LICENSE and NOTICE files for details. + * + * SPDX-License-Identifier: BSD-3-Clause + * SUNDIALS Copyright End + * ----------------------------------------------------------------- + * This is the header file for the sparse implementation of the + * SUNMATRIX module, SUNMATRIX_RESOLVE. + * ----------------------------------------------------------------- + */ + +#ifndef _SUNMATRIX_RESOLVE_H +#define _SUNMATRIX_RESOLVE_H + +#include +#include +#include +#include + +#ifdef __cplusplus /* wrapper to enable C++ usage */ +extern "C" { +#endif + +struct _SUNMatrixContent_ReSolve +{ + sunindextype M; + sunindextype N; + sunindextype NNZ; + ReSolve::memory::MemorySpace memspace; + ReSolve::matrix::Csr* mat; +}; + +typedef struct _SUNMatrixContent_ReSolve* SUNMatrixContent_ReSolve; + +/* --------------------------------------- + * Implementation specific functions + * ---------------------------------------*/ + +SUNDIALS_EXPORT SUNMatrix SUNMatrix_ReSolve(sunindextype M, sunindextype N, + sunindextype NNZ, + ReSolve::memory::MemorySpace memspace, + SUNContext sunctx); + +SUNDIALS_EXPORT sunindextype SUNMatrix_ReSolve_Rows(SUNMatrix A); + +SUNDIALS_EXPORT sunindextype SUNMatrix_ReSolve_Columns(SUNMatrix A); + +SUNDIALS_EXPORT sunindextype SUNMatrix_ReSolve_NNZ(SUNMatrix A); + +SUNDIALS_EXPORT sunrealtype* SUNMatrix_ReSolve_Data( + SUNMatrix A, ReSolve::memory::MemorySpace memspace); + +SUNDIALS_EXPORT sunindextype* SUNMatrix_ReSolve_IndexValues( + SUNMatrix A, ReSolve::memory::MemorySpace memspace); + +SUNDIALS_EXPORT sunindextype* SUNMatrix_ReSolve_IndexPointers( + SUNMatrix A, ReSolve::memory::MemorySpace memspace); + +SUNDIALS_EXPORT SUNErrCode +SUNMatrix_ReSolve_SetUpdated(SUNMatrix A, ReSolve::memory::MemorySpace memspace); + +SUNDIALS_EXPORT SUNErrCode +SUNMatrix_ReSolve_SyncData(SUNMatrix A, ReSolve::memory::MemorySpace memspace); + +SUNDIALS_EXPORT void SUNMatrix_ReSolve_Print(SUNMatrix A, std::ostream& out, + sunindextype indexing_base); + +/* --------------------------------------- + * SUNMatrix API functions + * ---------------------------------------*/ + +static inline SUNMatrix_ID SUNMatGetID_ReSolve(SUNMatrix A) +{ + return SUNMATRIX_RESOLVE; +} + +SUNDIALS_EXPORT void SUNMatDestroy_ReSolve(SUNMatrix A); +SUNDIALS_EXPORT SUNErrCode SUNMatZero_ReSolve(SUNMatrix A); +SUNDIALS_EXPORT SUNMatrix SUNMatClone_ReSolve(SUNMatrix A); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/sundials/fmod_int32/fsundials_core_mod.f90 b/src/sundials/fmod_int32/fsundials_core_mod.f90 index 9f392ded5e..d010e7c605 100644 --- a/src/sundials/fmod_int32/fsundials_core_mod.f90 +++ b/src/sundials/fmod_int32/fsundials_core_mod.f90 @@ -340,11 +340,12 @@ module fsundials_core_mod enumerator :: SUNMATRIX_GINKGO enumerator :: SUNMATRIX_GINKGOBATCH enumerator :: SUNMATRIX_KOKKOSDENSE + enumerator :: SUNMATRIX_RESOLVE enumerator :: SUNMATRIX_CUSTOM end enum integer, parameter, public :: SUNMatrix_ID = kind(SUNMATRIX_DENSE) public :: SUNMATRIX_DENSE, SUNMATRIX_MAGMADENSE, SUNMATRIX_ONEMKLDENSE, SUNMATRIX_BAND, SUNMATRIX_SPARSE, SUNMATRIX_SLUNRLOC, & - SUNMATRIX_CUSPARSE, SUNMATRIX_GINKGO, SUNMATRIX_GINKGOBATCH, SUNMATRIX_KOKKOSDENSE, SUNMATRIX_CUSTOM + SUNMATRIX_CUSPARSE, SUNMATRIX_GINKGO, SUNMATRIX_GINKGOBATCH, SUNMATRIX_KOKKOSDENSE, SUNMATRIX_RESOLVE, SUNMATRIX_CUSTOM ! struct struct _generic_SUNMatrix_Ops type, bind(C), public :: SUNMatrix_Ops type(C_FUNPTR), public :: getid diff --git a/src/sundials/fmod_int64/fsundials_core_mod.f90 b/src/sundials/fmod_int64/fsundials_core_mod.f90 index b3037e6dd5..778e391c0f 100644 --- a/src/sundials/fmod_int64/fsundials_core_mod.f90 +++ b/src/sundials/fmod_int64/fsundials_core_mod.f90 @@ -340,11 +340,12 @@ module fsundials_core_mod enumerator :: SUNMATRIX_GINKGO enumerator :: SUNMATRIX_GINKGOBATCH enumerator :: SUNMATRIX_KOKKOSDENSE + enumerator :: SUNMATRIX_RESOLVE enumerator :: SUNMATRIX_CUSTOM end enum integer, parameter, public :: SUNMatrix_ID = kind(SUNMATRIX_DENSE) public :: SUNMATRIX_DENSE, SUNMATRIX_MAGMADENSE, SUNMATRIX_ONEMKLDENSE, SUNMATRIX_BAND, SUNMATRIX_SPARSE, SUNMATRIX_SLUNRLOC, & - SUNMATRIX_CUSPARSE, SUNMATRIX_GINKGO, SUNMATRIX_GINKGOBATCH, SUNMATRIX_KOKKOSDENSE, SUNMATRIX_CUSTOM + SUNMATRIX_CUSPARSE, SUNMATRIX_GINKGO, SUNMATRIX_GINKGOBATCH, SUNMATRIX_KOKKOSDENSE, SUNMATRIX_RESOLVE, SUNMATRIX_CUSTOM ! struct struct _generic_SUNMatrix_Ops type, bind(C), public :: SUNMatrix_Ops type(C_FUNPTR), public :: getid diff --git a/src/sunmatrix/CMakeLists.txt b/src/sunmatrix/CMakeLists.txt index e61aed9513..6618c0ff88 100644 --- a/src/sunmatrix/CMakeLists.txt +++ b/src/sunmatrix/CMakeLists.txt @@ -76,6 +76,10 @@ if(SUNDIALS_ENABLE_SUNMATRIX_ONEMKLDENSE) add_subdirectory(onemkldense) endif() +if(SUNDIALS_ENABLE_SUNMATRIX_RESOLVE) + add_subdirectory(resolve) +endif() + if(SUNDIALS_ENABLE_SUNMATRIX_SLUNRLOC) add_subdirectory(slunrloc) endif() diff --git a/src/sunmatrix/resolve/CMakeLists.txt b/src/sunmatrix/resolve/CMakeLists.txt new file mode 100644 index 0000000000..61def5b1e1 --- /dev/null +++ b/src/sunmatrix/resolve/CMakeLists.txt @@ -0,0 +1,39 @@ +# --------------------------------------------------------------- +# Programmer(s): Jeffery Zhang +# --------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2025-2026, Lawrence Livermore National Security, +# University of Maryland Baltimore County, and the SUNDIALS contributors. +# Copyright (c) 2013-2025, Lawrence Livermore National Security +# and Southern Methodist University. +# Copyright (c) 2002-2013, Lawrence Livermore National Security. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# --------------------------------------------------------------- + +install( + CODE "MESSAGE(\"\nInstall SUNMATRIX_RESOLVE with ${SUNDIALS_RESOLVE_BACKENDS} backend(s)\n\")" +) + +# Add the sunmatrix_resolve library +sundials_add_library( + sundials_sunmatrixresolve + SOURCES sunmatrix_resolve.cpp + HEADERS ${SUNDIALS_SOURCE_DIR}/include/sunmatrix/sunmatrix_resolve.h + INCLUDE_SUBDIR sunmatrix + LINK_LIBRARIES PUBLIC sundials_core + OBJECT_LIBRARIES + INCLUDE_DIRECTORIES PUBLIC ${ReSolve_INCLUDE_DIR} + LINK_LIBRARIES PUBLIC SUNDIALS::ReSolve ${_libs_needed} + OUTPUT_NAME sundials_sunmatrixresolve + VERSION ${sunmatrixlib_VERSION} + SOVERSION ${sunmatrixlib_SOVERSION}) + +message( + STATUS + "Added SUNMATRIX_RESOLVE module with ${SUNDIALS_RESOLVE_BACKENDS} backend(s)" +) diff --git a/src/sunmatrix/resolve/sunmatrix_resolve.cpp b/src/sunmatrix/resolve/sunmatrix_resolve.cpp new file mode 100644 index 0000000000..e25f5ebd63 --- /dev/null +++ b/src/sunmatrix/resolve/sunmatrix_resolve.cpp @@ -0,0 +1,325 @@ +/* --------------------------------------------------------------------------- + * Programmer(s): Jeffery Zhang + * --------------------------------------------------------------------------- + * SUNDIALS Copyright Start + * Copyright (c) 2025-2026, Lawrence Livermore National Security, + * University of Maryland Baltimore County, and the SUNDIALS contributors. + * Copyright (c) 2013-2025, Lawrence Livermore National Security + * and Southern Methodist University. + * Copyright (c) 2002-2013, Lawrence Livermore National Security. + * All rights reserved. + * + * See the top-level LICENSE and NOTICE files for details. + * + * SPDX-License-Identifier: BSD-3-Clause + * SUNDIALS Copyright End + * --------------------------------------------------------------------------- + * This is the implementation file for the dense implementation of the + * SUNMATRIX class using the Re::Solve library + * ---------------------------------------------------------------------------*/ + +#include +#include + +// SUNDIALS headers +#include +#include +#include +#include "sundials/sundials_errors.h" +#include "sundials_debug.h" +#include "sundials_macros.h" + +// Check for a valid precision +#if defined(SUNDIALS_EXTENDED_PRECISION) +#error \ + "Re::Solve set precision does not match SUNDIALS precision for floating type" +#endif + +#if defined(SUNDIALS_INT64_T) +#error "Re::Solve set precision does not match SUNDIALS precision for indices" +#endif + +// Constants +#define ZERO SUN_RCONST(0.0) +#define ONE SUN_RCONST(1.0) + +// Content accessor macro +#define RESOLVE_CONTENT(A) \ + ((SUNMatrixContent_ReSolve)(A)->content) // ASSUMES CSR FORMAT DEFAULT +#define RESOLVE_MAT(A) (RESOLVE_CONTENT(A)->mat) +#define RESOLVE_M(A) (RESOLVE_CONTENT(A)->M) +#define RESOLVE_N(A) (RESOLVE_CONTENT(A)->N) +#define RESOLVE_NNZ(A) (RESOLVE_CONTENT(A)->NNZ) +#define RESOLVE_MEMSPACE(A) (RESOLVE_CONTENT(A)->memspace) + +/* -------------------------------------------------------------------------- + * Constructor + * -------------------------------------------------------------------------- */ + +SUNMatrix SUNMatrix_ReSolve(sunindextype m, sunindextype n, sunindextype nnz, + ReSolve::memory::MemorySpace memspace, + SUNContext sunctx) +{ + SUNFunctionBegin(sunctx); + SUNMatrixContent_ReSolve content; + + // Check inputs + if ((m <= 0) || (n <= 0) || (nnz < 0)) + { + SUNDIALS_DEBUG_ERROR("Illegal input\n"); + return NULL; + } + + // Create an empty matrix object + SUNMatrix A = SUNMatNewEmpty(sunctx); + if (!A) + { + SUNDIALS_DEBUG_ERROR("SUNMatNewEmpty returned NULL\n"); + return NULL; + } + + // Attach operations + A->ops->getid = SUNMatGetID_ReSolve; + A->ops->destroy = SUNMatDestroy_ReSolve; + A->ops->zero = SUNMatZero_ReSolve; + A->ops->clone = SUNMatClone_ReSolve; + + // Create ReSolve matrix + ReSolve::matrix::Csr* mat = new ReSolve::matrix::Csr(m, n, nnz); + mat->allocateMatrixData(ReSolve::memory::HOST); + // Allocate matrix on device if necessary + if (memspace == ReSolve::memory::DEVICE) + { + mat->allocateMatrixData(memspace); + } + + /* Create content */ + content = NULL; + content = (SUNMatrixContent_ReSolve)malloc(sizeof *content); + SUNAssertNull(content, SUN_ERR_MALLOC_FAIL); + + /* Attach content */ + A->content = content; + + /* Fill content */ + content->M = m; + content->N = n; + content->NNZ = nnz; + content->mat = mat; + content->memspace = memspace; + + if (!(A->content)) + { + SUNDIALS_DEBUG_ERROR("Content allocation failed\n"); + SUNMatDestroy(A); + return NULL; + } + + return A; +} + +/* -------------------------------------------------------------------------- + * Implementation specific functions + * -------------------------------------------------------------------------- */ + +sunindextype SUNMatrix_ReSolve_Rows(SUNMatrix A) +{ + SUNFunctionBegin(A->sunctx); + SUNAssertNoRet(SUNMatGetID(A) == SUNMATRIX_RESOLVE, SUN_ERR_ARG_WRONGTYPE); + return RESOLVE_M(A); +} + +sunindextype SUNMatrix_ReSolve_Columns(SUNMatrix A) +{ + SUNFunctionBegin(A->sunctx); + SUNAssertNoRet(SUNMatGetID(A) == SUNMATRIX_RESOLVE, SUN_ERR_ARG_WRONGTYPE); + return RESOLVE_N(A); +} + +sunindextype SUNMatrix_ReSolve_NNZ(SUNMatrix A) +{ + SUNFunctionBegin(A->sunctx); + SUNAssertNoRet(SUNMatGetID(A) == SUNMATRIX_RESOLVE, SUN_ERR_ARG_WRONGTYPE); + return RESOLVE_NNZ(A); +} + +/** + Get the pointer to the ReSolve matrix data array + + @param[in] A The SUNMatrix object +*/ +sunrealtype* SUNMatrix_ReSolve_Data(SUNMatrix A, + ReSolve::memory::MemorySpace memspace) +{ + return RESOLVE_MAT(A)->getValues(memspace); +} + +/** + Get the pointer to the ReSolve matrix offsets array + + @param[in] A The SUNMatrix object +*/ +sunindextype* SUNMatrix_ReSolve_IndexPointers(SUNMatrix A, + ReSolve::memory::MemorySpace memspace) +{ + return RESOLVE_MAT(A)->getRowData(memspace); +} + +/** + Get the pointer to the ReSolve matrix indices array + + @param[in] A The SUNMatrix object +*/ +sunindextype* SUNMatrix_ReSolve_IndexValues(SUNMatrix A, + ReSolve::memory::MemorySpace memspace) +{ + return RESOLVE_MAT(A)->getColData(memspace); +} + +/** + * @brief Tags `memspace` as updated. + * + * @param[in] memspace - memory space (HOST or DEVICE) to set to "updated" + * + * @return 0 if successful, -1 if not. + * + * The method sets the boolean flag indicating that the `memspace` is updated. + * It automatically sets the other data mirror to non-updated. You would + * use this function if you update matrix data by accessing its raw pointers. + * In such case, the matrix has no way of knowing which data is most recent, so + * you have to tell it. + * + * @warning This is an expert-level function. Use only if you know what you are + * doing. + * + * @note If you want to set both DEVICE and HOST memory to the same value + * use syncData function. +*/ +SUNErrCode SUNMatrix_ReSolve_SetUpdated(SUNMatrix A, + ReSolve::memory::MemorySpace memspace) +{ + RESOLVE_MAT(A)->setUpdated(memspace); + return SUN_SUCCESS; +} + +/** + * @brief Sync data in memspace with the updated memory space. + * + * @param A - The SUNMatrix_ReSolve object + * @param memspace - memory space to be synced up (HOST or DEVICE) + * + * @pre The memory space other than `memspace` must be up-to-date. Otherwise, + * this function will return an error. + * + * @see Sparse::setUpdated in the Re::Solve library +*/ +SUNErrCode SUNMatrix_ReSolve_SyncData(SUNMatrix A, + ReSolve::memory::MemorySpace memspace) +{ + int result = RESOLVE_MAT(A)->syncData(memspace); + if (result == 0) { return SUN_SUCCESS; } + else { return SUN_ERR_ARG_INCOMPATIBLE; } +} + +/** + Print the Re::Solve matrix +*/ +void SUNMatrix_ReSolve_Print(SUNMatrix A, std::ostream& out, + sunindextype indexing_base) +{ + RESOLVE_MAT(A)->print(out, indexing_base); +} + +/* -------------------------------------------------------------------------- + * Implementation of generic SUNMatrix operations. + * -------------------------------------------------------------------------- */ + +void SUNMatDestroy_ReSolve(SUNMatrix A) +{ + if (!A) + { + SUNDIALS_DEBUG_ERROR("Input matrix is NULL\n"); + return; + } + + if (SUNMatGetID(A) != SUNMATRIX_RESOLVE) + { + SUNDIALS_DEBUG_ERROR("Invalid matrix ID\n"); + return; + } + + // Free ReSolve matrix + delete RESOLVE_MAT(A); + + /* free content struct */ + free(A->content); + A->content = NULL; + + /* free ops and matrix */ + if (A->ops) + { + free(A->ops); + A->ops = NULL; + } + + // Free matrix + SUNMatFreeEmpty(A); + A = NULL; + + return; +} + +// This function automatically syncs data between host and device +SUNErrCode SUNMatZero_ReSolve(SUNMatrix A) +{ + if (!A) + { + SUNDIALS_DEBUG_ERROR("Input matrix is NULL\n"); + return SUN_ERR_ARG_INCOMPATIBLE; + } + + if (SUNMatGetID(A) != SUNMATRIX_RESOLVE) + { + SUNDIALS_DEBUG_ERROR("Invalid matrix ID\n"); + return SUN_ERR_ARG_INCOMPATIBLE; + } + + sunindextype i; + + // Get pointers to the data, indexvalues and indexpointers arrays on host in ReSolve + sunrealtype* values = RESOLVE_MAT(A)->getValues(ReSolve::memory::HOST); + + sunindextype* index_pointers = RESOLVE_MAT(A)->getRowData(ReSolve::memory::HOST); + + sunindextype* index_values = RESOLVE_MAT(A)->getColData(ReSolve::memory::HOST); + + // Zero out the values of these arrays + for (i = 0; i < RESOLVE_NNZ(A); i++) + { + values[i] = ZERO; + index_values[i] = 0; + } + + for (i = 0; i < RESOLVE_M(A); i++) { index_pointers[i] = ZERO; } + + (index_pointers)[RESOLVE_M(A)] = 0; + + SUNMatrix_ReSolve_SetUpdated(A, ReSolve::memory::HOST); + + // Sync to device if necessary + if (RESOLVE_MEMSPACE(A) != ReSolve::memory::HOST) + { + SUNMatrix_ReSolve_SyncData(A, RESOLVE_MEMSPACE(A)); + } + + return SUN_SUCCESS; +} + +SUNMatrix SUNMatClone_ReSolve(SUNMatrix A) +{ + SUNFunctionBegin(A->sunctx); + SUNMatrix B = SUNMatrix_ReSolve(RESOLVE_M(A), RESOLVE_N(A), RESOLVE_NNZ(A), + RESOLVE_MEMSPACE(A), A->sunctx); + SUNCheckLastErrNull(); + return (B); +} diff --git a/test/config_cmake.py b/test/config_cmake.py index a208260299..bb02013388 100644 --- a/test/config_cmake.py +++ b/test/config_cmake.py @@ -1156,6 +1156,30 @@ def main(): dependson="--xbraid", ) + # ReSolve + group = parser.add_argument_group("ReSolve Options") + + add_arg( + group, + "--resolve", + "SUNDIALS_RESOLVE", + "SUNDIALS_ENABLE_RESOLVE", + "OFF", + "BOOL", + "SUNDIALS ReSolve support", + ) + + add_arg( + group, + "--resolve-dir", + "RESOLVE_ROOT", + "ReSolve_DIR", + None, + "PATH", + "ReSolve install directory", + dependson="--resolve", + ) + # -------- # Testing # -------- diff --git a/test/env/docker.sh b/test/env/docker.sh index b26abc1cd1..465f3f850a 100644 --- a/test/env/docker.sh +++ b/test/env/docker.sh @@ -360,3 +360,15 @@ else export SUNDIALS_XBRAID=OFF unset XBRAID_ROOT fi + +# ------- +# resolve +# ------- + +# if [ "$SUNDIALS_PRECISION" == "double" ] && [ "$SUNDIALS_INDEX_SIZE" == "32" ]; then +# export SUNDIALS_RESOLVE=ON +# export RESOLVE_ROOT=/opt/view +# else +# export SUNDIALS_RESOLVE=OFF +# unset RESOLVE_ROOT +# fi diff --git a/test/unit_tests/sunmatrix/CMakeLists.txt b/test/unit_tests/sunmatrix/CMakeLists.txt index 5a828f6222..27117805ef 100644 --- a/test/unit_tests/sunmatrix/CMakeLists.txt +++ b/test/unit_tests/sunmatrix/CMakeLists.txt @@ -60,6 +60,10 @@ if(SUNDIALS_ENABLE_SUNMATRIX_ONEMKLDENSE) add_subdirectory(onemkldense) endif() +if(SUNDIALS_ENABLE_SUNMATRIX_RESOLVE) + add_subdirectory(resolve) +endif() + if(SUNDIALS_ENABLE_SUNMATRIX_SLUNRLOC) add_subdirectory(slunrloc) endif() diff --git a/test/unit_tests/sunmatrix/resolve/CMakeLists.txt b/test/unit_tests/sunmatrix/resolve/CMakeLists.txt new file mode 100644 index 0000000000..4dc5a40172 --- /dev/null +++ b/test/unit_tests/sunmatrix/resolve/CMakeLists.txt @@ -0,0 +1,65 @@ +# ------------------------------------------------------------------------------ +# Programmer(s): Jeffery Zhang +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2025-2026, Lawrence Livermore National Security, +# University of Maryland Baltimore County, and the SUNDIALS contributors. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +set(examples_list "test_sunmatrix_resolve.cpp\;\;") + +# Add source directory to include directories +include_directories(..) + +foreach(example_tuple ${examples_list}) + # parse the example tuple + list(GET example_tuple 0 example) + list(GET example_tuple 1 example_args) + list(GET example_tuple 2 example_type) + + # extract the file name without extension + get_filename_component(example_target ${example} NAME_WE) + + if(NOT TARGET ${example_target}) + # example source files + sundials_add_executable(${example_target} ${example} ../test_sunmatrix.c + ../test_sunmatrix.h) + + # folder for IDEs + set_target_properties(${example_target} PROPERTIES FOLDER "Examples") + + # libraries to link against + target_link_libraries( + ${example_target} PRIVATE sundials_core sundials_sunmatrixresolve + SUNDIALS::ReSolve ${EXE_EXTRA_LINK_LIBS}) + + # Target GPU backends if necessary + if(SUNDIALS_RESOLVE_BACKENDS MATCHES "CUDA") + target_link_libraries(${example_target} PRIVATE SUNDIALS::ReSolve_CUDA) + elseif(SUNDIALS_RESOLVE_BACKENDS MATCHES "HIP") + target_link_libraries(${example_target} PRIVATE SUNDIALS::ReSolve_HIP) + endif() + + endif() + + # set test name + if("${example_args}" STREQUAL "") + set(test_name ${example_target}) + else() + string(REGEX REPLACE " " "_" test_name ${example_target}_${example_args}) + endif() + + # add to regression tests + sundials_add_test( + ${test_name} ${example_target} + TEST_ARGS ${example_args} + EXAMPLE_TYPE ${example_type} + NODIFF) + +endforeach() diff --git a/test/unit_tests/sunmatrix/resolve/test_sunmatrix_resolve.cpp b/test/unit_tests/sunmatrix/resolve/test_sunmatrix_resolve.cpp new file mode 100644 index 0000000000..e03e115c90 --- /dev/null +++ b/test/unit_tests/sunmatrix/resolve/test_sunmatrix_resolve.cpp @@ -0,0 +1,286 @@ +/* + * ----------------------------------------------------------------- + * Programmer(s): Jeffery Zhang + * ----------------------------------------------------------------- + * SUNDIALS Copyright Start + * Copyright (c) 2025-2026, Lawrence Livermore National Security, + * University of Maryland Baltimore County, and the SUNDIALS contributors. + * Copyright (c) 2013-2025, Lawrence Livermore National Security + * and Southern Methodist University. + * Copyright (c) 2002-2013, Lawrence Livermore National Security. + * All rights reserved. + * + * See the top-level LICENSE and NOTICE files for details. + * + * SPDX-License-Identifier: BSD-3-Clause + * SUNDIALS Copyright End + * ----------------------------------------------------------------- + * This is the testing routine to check the SUNMatrix ReSolve module + * implementation. + * ----------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "test_sunmatrix.h" + +/* ---------------------------------------------------------------------- + * Main SUNMatrix Testing Routine + * --------------------------------------------------------------------*/ +int main(int argc, char* argv[]) +{ + int fails = 0; /* counter for test failures */ + SUNMatrix A; + sunindextype i, j, k, kstart, kend, N, uband, lband; + int print_timing, square; + SUNContext sunctx; + + // Default backend + std::string hwbackend = "CPU"; + + if (SUNContext_Create(SUN_COMM_NULL, &sunctx)) + { + printf("ERROR: SUNContext_Create failed\n"); + return (-1); + } + + // Initialize a ReSolve HOST memory space. + ReSolve::memory::MemorySpace memspace = ReSolve::memory::HOST; +// Check if a GPU backend is enabled +#ifdef SUNDIALS_RESOLVE_BACKENDS_CUDA + hwbackend = "CUDA"; + memspace = ReSolve::memory::DEVICE; +#elif defined(SUNDIALS_RESOLVE_BACKENDS_HIP) + hwbackend = "HIP"; + memspace = ReSolve::memory::DEVICE; +#endif + + // Create a SUNMatrix_ReSolve object + A = NULL; + A = SUNMatrix_ReSolve(5, 5, 13, memspace, sunctx); + + // Get pointers to content arrays + sunrealtype* data = SUNMatrix_ReSolve_Data(A, ReSolve::memory::HOST); + sunindextype* index_values = + SUNMatrix_ReSolve_IndexValues(A, ReSolve::memory::HOST); + sunindextype* index_pointers = + SUNMatrix_ReSolve_IndexPointers(A, ReSolve::memory::HOST); + + // Fill the matrix as a 5x5 second difference matrix + for (i = 0; i < SUNMatrix_ReSolve_NNZ(A); i++) + { + if (i % 3 == 0) { data[i] = 2; } + else { data[i] = -1; } + } + + // Row pointers + index_pointers[0] = 0; // row 0 starts at 0 (2 non-zeros: diag + right) + index_pointers[1] = 2; // row 1 starts at 2 (3 non-zeros: left + diag + right) + index_pointers[2] = 5; // row 2 starts at 5 (3 non-zeros) + index_pointers[3] = 8; // row 3 starts at 8 (3 non-zeros) + index_pointers[4] = 11; // row 4 starts at 11 (2 non-zeros: left + diag) + index_pointers[5] = 13; // total non-zeros + + // Column indices + index_values[0] = 0; + index_values[1] = 1; // row 0: cols 0, 1 + index_values[2] = 0; + index_values[3] = 1; + index_values[4] = 2; // row 1: cols 0, 1, 2 + index_values[5] = 1; + index_values[6] = 2; + index_values[7] = 3; // row 2: cols 1, 2, 3 + index_values[8] = 2; + index_values[9] = 3; + index_values[10] = 4; // row 3: cols 2, 3, 4 + index_values[11] = 3; + index_values[12] = 4; // row 4: cols 3, 4 + + SUNMatrix_ReSolve_SetUpdated(A, ReSolve::memory::HOST); + // Sync host to device if necessary + if (memspace != ReSolve::memory::HOST) + { + SUNMatrix_ReSolve_SyncData(A, memspace); + } + + // Print the matrix + printf("\n The SUNMatrix object\n"); + SUNMatrix_ReSolve_Print(A, std::cout, 0); + printf("\n"); + + std::cout << "Running SUNMatrix generic unit tests\n"; + /* SUNMatrix Tests */ + fails += Test_SUNMatGetID(A, SUNMATRIX_RESOLVE, 0); + fails += Test_SUNMatZero(A, 0); + + if (fails) + { + std::cout << "FAIL: SUNMatrix module failed " << fails << " tests on the " + << hwbackend << " hardware backend\n \n"; + printf("\nA =\n"); + SUNMatrix_ReSolve_Print(A, std::cout, 0); + } + else + { + std::cout << "\nSUCCESS: SUNMatrix module passed all tests on the " + << hwbackend << " hardware backend\n\n"; + } + + // Destroy the SUNMatrix_ReSolve object + SUNMatDestroy_ReSolve(A); + + return 0; +} + +/* ---------------------------------------------------------------------- + * Check matrix + * --------------------------------------------------------------------*/ + +// Assumes already synced +int check_matrix(SUNMatrix A, SUNMatrix B, sunrealtype tol) +{ + int failure = 0; + sunindextype i, A_NP, B_NP, A_nnz, B_nnz; + sunrealtype *A_data, *B_data; + sunindextype *A_index_values, *A_index_pointers, *B_index_values, + *B_index_pointers; + + // Get pointers to the data, pointer and value arrays + A_data = SUNMatrix_ReSolve_Data(A, ReSolve::memory::HOST); + A_index_values = SUNMatrix_ReSolve_IndexValues(A, ReSolve::memory::HOST); + A_index_pointers = SUNMatrix_ReSolve_IndexPointers(A, ReSolve::memory::HOST); + + B_data = SUNMatrix_ReSolve_Data(B, ReSolve::memory::HOST); + B_index_values = SUNMatrix_ReSolve_IndexValues(B, ReSolve::memory::HOST); + B_index_pointers = SUNMatrix_ReSolve_IndexPointers(B, ReSolve::memory::HOST); + + // Get nnz and np + A_nnz = SUNMatrix_ReSolve_NNZ(A); + A_NP = SUNMatrix_ReSolve_Rows(A); + + B_nnz = SUNMatrix_ReSolve_NNZ(B); + B_NP = SUNMatrix_ReSolve_Rows(B); + + // Check same storage type + if (SUNMatGetID(A) != SUNMatGetID(B)) + { + printf(">>> ERROR: check_matrix: Different storage types (%d vs %d)\n", + SUNMatGetID(A), SUNMatGetID(B)); + return (1); + } + + // Check shape + if (SUNMatrix_ReSolve_Rows(A) != SUNMatrix_ReSolve_Rows(B)) + { + printf(">>> ERROR: check_matrix: Different numbers of rows (%ld vs %ld)\n", + (long int)SUNMatrix_ReSolve_Rows(A), + (long int)SUNMatrix_ReSolve_Rows(B)); + return (1); + } + if (SUNMatrix_ReSolve_Columns(A) != SUNMatrix_ReSolve_Columns(B)) + { + printf(">>> ERROR: check_matrix: Different numbers of columns (%ld vs " + "%ld)\n", + (long int)SUNMatrix_ReSolve_Columns(A), + (long int)SUNMatrix_ReSolve_Columns(B)); + return (1); + } + + // Check non zeros + if (A_nnz != B_nnz) + { + printf(">>> ERROR: check_matrix: Different numbers of nonzeros (%ld vs " + "%ld)\n", + (long int)A_nnz, (long int)B_nnz); + return (1); + } + + /* compare sparsity patterns */ + for (i = 0; i < A_NP; i++) + { + failure += (A_index_pointers[i] != B_index_pointers[i]); + } + + if (failure > ZERO) + { + printf(">>> ERROR: check_matrix: Different indexptrs \n"); + return (1); + } + + for (i = 0; i < A_nnz; i++) + { + failure += (A_index_values[i] != B_index_values[i]); + } + + if (failure > ZERO) + { + printf(">>> ERROR: check_matrix: Different indexvals \n"); + return (1); + } + + /* compare matrix values */ + for (i = 0; i < A_nnz; i++) + { + failure += SUNRCompareTol(A_data[i], B_data[i], tol); + } + if (failure > ZERO) + { + printf(">>> ERROR: check_matrix: Different entries \n"); + return (1); + } + + return (0); +} + +int check_matrix_entry(SUNMatrix A, sunrealtype val, sunrealtype tol) +{ + int failure = 0; + sunrealtype* Adata; + sunindextype* indexptrs; + sunindextype i, NP; + + /* get data pointer */ + Adata = SUNMatrix_ReSolve_Data(A, ReSolve::memory::HOST); + + /* compare data */ + indexptrs = SUNMatrix_ReSolve_IndexPointers(A, ReSolve::memory::HOST); + NP = SUNMatrix_ReSolve_Rows(A); + for (i = 0; i < indexptrs[NP]; i++) + { + failure += SUNRCompareTol(Adata[i], val, tol); + } + + if (failure > ZERO) { return (1); } + else { return (0); } +} + +int check_vector(N_Vector actual, N_Vector expected, sunrealtype tol) +{ + return 0; +} + +sunbooleantype has_data(SUNMatrix A) +{ + sunrealtype* Adata = SUNMatrix_ReSolve_Data(A, ReSolve::memory::HOST); + if (Adata == NULL) { return SUNFALSE; } + else { return SUNTRUE; } +} + +sunbooleantype is_square(SUNMatrix A) +{ + if (SUNMatrix_ReSolve_Rows(A) == SUNMatrix_ReSolve_Columns(A)) + { + return SUNTRUE; + } + else { return SUNFALSE; } +} + +void sync_device(SUNMatrix A) +{ + /* sync is performed by functions for now */ + return; +} \ No newline at end of file