From 67898dc0297cc9446f1ee068cd1f997ca591b2e0 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Wed, 29 Apr 2026 13:19:58 -0500 Subject: [PATCH 1/3] First cut of support for kokkos-tools! --- CMakeLists.txt | 1 + cmake/tpls/EkatBuildKokkosTools.cmake | 19 ++++++++++++++ src/testing-support/CMakeLists.txt | 25 +++++++++++++++++++ src/testing-support/ekat_catch_main.cpp | 6 +++++ src/testing-support/ekat_test_session.cpp | 6 +++++ .../ekat_testing_support.hpp.in | 10 ++++++++ 6 files changed, 67 insertions(+) create mode 100644 cmake/tpls/EkatBuildKokkosTools.cmake create mode 100644 src/testing-support/ekat_testing_support.hpp.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 3127e4fb..53741033 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ message(STATUS "Installation prefix: ${CMAKE_INSTALL_PREFIX}") # TODO: this is never used in EKAT src folder, so it should be removed. # Downstream projects should handle bfb logic autonomously option (EKAT_DEFAULT_BFB "Whether EKAT should default to BFB behavior whenever possible/appropriate." ${EKAT_IS_DEBUG_BUILD}) +option (EKAT_ENABLE_KOKKOS_TOOLS "Whether to build kokkos-tools and use it to instrument tests at runtime" OFF) option (EKAT_ENABLE_VALGRIND "Whether to run tests with valgrind" OFF) option (EKAT_ENABLE_CUDA_MEMCHECK "Whether to run tests with cuda-memcheck" OFF) option (EKAT_ENABLE_COMPUTE_SANITIZER "Whether to run tests with nvidia's compute-sanitizer" OFF) diff --git a/cmake/tpls/EkatBuildKokkosTools.cmake b/cmake/tpls/EkatBuildKokkosTools.cmake new file mode 100644 index 00000000..446c5d2c --- /dev/null +++ b/cmake/tpls/EkatBuildKokkosTools.cmake @@ -0,0 +1,19 @@ +# Fetch kokkos-tools via FetchContent +include (FetchContent) +include (EkatUtils) + +message (STATUS " Fetching kokkos-tools via FetchContent") + +set(KOKKOS_TOOLS_GIT_TAG e222e7b6cc7faae5d096e3520d95ca832fb9f889) + +# Fixed options +set(KokkosTools_ENABLE_MPI ON CACHE BOOL "Enable MPI for kokkos-tools" FORCE) +set(KokkosTools_ENABLE_EXAMPLES OFF CACHE BOOL "Disable kokkos-tools examples" FORCE) +set(KokkosTools_ENABLE_TESTS OFF CACHE BOOL "Disable kokkos-tools tests" FORCE) + +# If TPL is already present with correct sha simply adds subdir, otherwise uses FetchContent first +ekat_fetch_content(KokkosTools + GIT_REPOSITORY https://github.com/kokkos/kokkos-tools.git + GIT_TAG ${KOKKOS_TOOLS_GIT_TAG} + SOURCE_DIR ${EKAT_SOURCE_DIR}/extern/kokkos-tools +) diff --git a/src/testing-support/CMakeLists.txt b/src/testing-support/CMakeLists.txt index d09cd246..622b3028 100644 --- a/src/testing-support/CMakeLists.txt +++ b/src/testing-support/CMakeLists.txt @@ -1,3 +1,21 @@ +### Kokkos Tools for kernel profiling ### + +if (EKAT_ENABLE_KOKKOS_TOOLS) + set (KokkosTools_FOUND FALSE) + if (TARGET kokkostools) + set(KokkosTools_FOUND TRUE) + message(STATUS "EKAT: Using kokkos-tools provided by parent project") + else() # not found -- we fetch it ourselves + include (EkatBuildKokkosTools) + endif() + + # Build a list of kokkos-tools libraries + set(profiling_libs simple-kernel-timer) + foreach (lib_path ${profile_libs}) + list(APPEND KOKKOS_TOOLS_LIBS ${kokkostools_BINARY_DIR}/profiling/${lib_path}) + endforeach() +endif() + ### CATCH MAIN ### set (Catch2_FOUND FALSE) @@ -84,6 +102,13 @@ set_target_properties(ekat_testsession PROPERTIES EXPORT_NAME DefaultTestSession ) +# Write out a configuration header for testing support. +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ekat_testing_support.hpp.in + ${CMAKE_CURRENT_BINARY_DIR}/ekat_testing_support.hpp) +set_target_properties(ekat_testsession PROPERTIES + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR} +) + # Install the packages install (TARGETS ekat_testsession EXPORT EkatTargets diff --git a/src/testing-support/ekat_catch_main.cpp b/src/testing-support/ekat_catch_main.cpp index 82c2ae60..36d8e178 100644 --- a/src/testing-support/ekat_catch_main.cpp +++ b/src/testing-support/ekat_catch_main.cpp @@ -1,5 +1,7 @@ #define CATCH_CONFIG_RUNNER +#include "ekat_testing_support.hpp" + #include "catch2/catch.hpp" #include "ekat_test_utils.hpp" @@ -49,6 +51,10 @@ Args split_args (int argc, char** argv) int main (int argc, char **argv) { +#ifdef EKAT_ENABLE_KOKKOS_TOOLS + setenv("KOKKOS_TOOLS_LIBS", KOKKOS_TOOLS_LIBS, 1); +#endif + #ifdef EKAT_ENABLE_MPI // Initialize MPI MPI_Init(&argc,&argv); diff --git a/src/testing-support/ekat_test_session.cpp b/src/testing-support/ekat_test_session.cpp index c751d185..aa7c2733 100644 --- a/src/testing-support/ekat_test_session.cpp +++ b/src/testing-support/ekat_test_session.cpp @@ -1,3 +1,5 @@ +#include "ekat_testing_support.hpp" + #ifdef EKAT_HAS_KOKKOS #include "ekat_kokkos_session.hpp" #endif @@ -6,6 +8,10 @@ #include "ekat_arch.hpp" void ekat_initialize_test_session (int argc, char** argv, const bool print_config) { +#ifdef EKAT_ENABLE_KOKKOS_TOOLS + setenv("KOKKOS_TOOLS_LIBS", KOKKOS_TOOLS_LIBS, 1); +#endif + #ifdef EKAT_HAS_KOKKOS ekat::initialize_kokkos_session (argc,argv,print_config); #endif diff --git a/src/testing-support/ekat_testing_support.hpp.in b/src/testing-support/ekat_testing_support.hpp.in new file mode 100644 index 00000000..a0944ead --- /dev/null +++ b/src/testing-support/ekat_testing_support.hpp.in @@ -0,0 +1,10 @@ +#ifndef EKAT_TESTING_SUPPORT_HPP +#define EKAT_TESTING_SUPPORT_HPP + +#cmakedefine EKAT_ENABLE_KOKKOS_TOOLS + +#ifdef EKAT_ENABLE_KOKKOS_TOOLS +#define KOKKOS_TOOLS_LIBS @KOKKOS_TOOLS_LIBS@ +#endif + +#endif From 1fb5216733e3337cfdd9b74ea3881ed05cae853c Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Wed, 29 Apr 2026 13:34:36 -0500 Subject: [PATCH 2/3] Some fixups. --- README.md | 1 + src/testing-support/CMakeLists.txt | 14 +++++++------- src/testing-support/ekat_testing_support.hpp.in | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d1a1bc0c..73539794 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ ctest --output-on-failure | `EKAT_ENABLE_ALGORITHM` | `OFF` | Enable the Algorithm sub-package. | | `EKAT_ENABLE_EXPRESSION` | `OFF` | Enable the Expression sub-package. | | `EKAT_ENABLE_KOKKOS` | `OFF` | Enable the KokkosUtils sub-package. | +| `EKAT_ENABLE_KOKKOS_TOOLS` | `OFF` | Enable support for kokkos-tools. | | `EKAT_ENABLE_LOGGING` | `OFF` | Enable the Logging sub-package. | | `EKAT_ENABLE_PACK` | `OFF` | Enable the Pack sub-package. | | `EKAT_ENABLE_YAML_PARSER` | `OFF` | Enable the Parser sub-package. | diff --git a/src/testing-support/CMakeLists.txt b/src/testing-support/CMakeLists.txt index 622b3028..c453acc4 100644 --- a/src/testing-support/CMakeLists.txt +++ b/src/testing-support/CMakeLists.txt @@ -11,7 +11,7 @@ if (EKAT_ENABLE_KOKKOS_TOOLS) # Build a list of kokkos-tools libraries set(profiling_libs simple-kernel-timer) - foreach (lib_path ${profile_libs}) + foreach (lib_path ${profiling_libs}) list(APPEND KOKKOS_TOOLS_LIBS ${kokkostools_BINARY_DIR}/profiling/${lib_path}) endforeach() endif() @@ -52,17 +52,23 @@ else() set (EKAT_BUILDS_CATCH2 OFF CACHE BOOL "Whether Ekat builds Catch2" FORCE) endif() +# Write out a configuration header for testing support. +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ekat_testing_support.hpp.in + ${CMAKE_CURRENT_BINARY_DIR}/ekat_testing_support.hpp) + # A small lib for Catch2 main executable, so we don't rebuild it # for every single unit test add_library(ekat_catchmain ekat_catch_main.cpp) target_link_libraries(ekat_catchmain PUBLIC ekat::Core) target_link_libraries(ekat_catchmain PUBLIC Catch2::Catch2) +target_link_libraries(ekat_catchmain PUBLIC ekat_testsession) # Set the name to be used when exportin the target # This, together with the NAMESPACE property set in the main CMakeLists.txt # install call, will force user to link ekat::CatchMain set_target_properties(ekat_catchmain PROPERTIES EXPORT_NAME CatchMain + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR} ) # Install the packages @@ -100,12 +106,6 @@ endif() # install call, will force user to link ekat::CatchMain set_target_properties(ekat_testsession PROPERTIES EXPORT_NAME DefaultTestSession -) - -# Write out a configuration header for testing support. -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ekat_testing_support.hpp.in - ${CMAKE_CURRENT_BINARY_DIR}/ekat_testing_support.hpp) -set_target_properties(ekat_testsession PROPERTIES INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/src/testing-support/ekat_testing_support.hpp.in b/src/testing-support/ekat_testing_support.hpp.in index a0944ead..e1495f38 100644 --- a/src/testing-support/ekat_testing_support.hpp.in +++ b/src/testing-support/ekat_testing_support.hpp.in @@ -4,7 +4,7 @@ #cmakedefine EKAT_ENABLE_KOKKOS_TOOLS #ifdef EKAT_ENABLE_KOKKOS_TOOLS -#define KOKKOS_TOOLS_LIBS @KOKKOS_TOOLS_LIBS@ +#define KOKKOS_TOOLS_LIBS "@KOKKOS_TOOLS_LIBS@" #endif #endif From 039f0160f6f097825bdb48bec2d1455fe595ccd2 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Wed, 29 Apr 2026 15:08:03 -0500 Subject: [PATCH 3/3] Addressing review comments. --- src/testing-support/CMakeLists.txt | 19 ++++++++++++------- src/testing-support/ekat_catch_main.cpp | 2 -- src/testing-support/ekat_test_session.cpp | 2 -- .../ekat_testing_support.hpp.in | 10 ---------- 4 files changed, 12 insertions(+), 21 deletions(-) delete mode 100644 src/testing-support/ekat_testing_support.hpp.in diff --git a/src/testing-support/CMakeLists.txt b/src/testing-support/CMakeLists.txt index c453acc4..feadb737 100644 --- a/src/testing-support/CMakeLists.txt +++ b/src/testing-support/CMakeLists.txt @@ -52,23 +52,17 @@ else() set (EKAT_BUILDS_CATCH2 OFF CACHE BOOL "Whether Ekat builds Catch2" FORCE) endif() -# Write out a configuration header for testing support. -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ekat_testing_support.hpp.in - ${CMAKE_CURRENT_BINARY_DIR}/ekat_testing_support.hpp) - # A small lib for Catch2 main executable, so we don't rebuild it # for every single unit test add_library(ekat_catchmain ekat_catch_main.cpp) target_link_libraries(ekat_catchmain PUBLIC ekat::Core) target_link_libraries(ekat_catchmain PUBLIC Catch2::Catch2) -target_link_libraries(ekat_catchmain PUBLIC ekat_testsession) # Set the name to be used when exportin the target # This, together with the NAMESPACE property set in the main CMakeLists.txt # install call, will force user to link ekat::CatchMain set_target_properties(ekat_catchmain PROPERTIES EXPORT_NAME CatchMain - INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR} ) # Install the packages @@ -106,9 +100,20 @@ endif() # install call, will force user to link ekat::CatchMain set_target_properties(ekat_testsession PROPERTIES EXPORT_NAME DefaultTestSession - INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR} ) +# If we're using kokkos-tools, define some symbols for ekat_catchmain and ekat_testsession +if (EKAT_ENABLE_KOKKOS_TOOLS) + target_compile_definitions(ekat_catchmain PRIVATE + EKAT_ENABLE_KOKKOS_TOOLS + KOKKOS_TOOLS_LIBS="${KOKKOS_TOOLS_LIBS}" + ) + target_compile_definitions(ekat_testsession PRIVATE + EKAT_ENABLE_KOKKOS_TOOLS + KOKKOS_TOOLS_LIBS="${KOKKOS_TOOLS_LIBS}" + ) +endif() + # Install the packages install (TARGETS ekat_testsession EXPORT EkatTargets diff --git a/src/testing-support/ekat_catch_main.cpp b/src/testing-support/ekat_catch_main.cpp index 36d8e178..cc943112 100644 --- a/src/testing-support/ekat_catch_main.cpp +++ b/src/testing-support/ekat_catch_main.cpp @@ -1,7 +1,5 @@ #define CATCH_CONFIG_RUNNER -#include "ekat_testing_support.hpp" - #include "catch2/catch.hpp" #include "ekat_test_utils.hpp" diff --git a/src/testing-support/ekat_test_session.cpp b/src/testing-support/ekat_test_session.cpp index aa7c2733..ec1e89dd 100644 --- a/src/testing-support/ekat_test_session.cpp +++ b/src/testing-support/ekat_test_session.cpp @@ -1,5 +1,3 @@ -#include "ekat_testing_support.hpp" - #ifdef EKAT_HAS_KOKKOS #include "ekat_kokkos_session.hpp" #endif diff --git a/src/testing-support/ekat_testing_support.hpp.in b/src/testing-support/ekat_testing_support.hpp.in deleted file mode 100644 index e1495f38..00000000 --- a/src/testing-support/ekat_testing_support.hpp.in +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef EKAT_TESTING_SUPPORT_HPP -#define EKAT_TESTING_SUPPORT_HPP - -#cmakedefine EKAT_ENABLE_KOKKOS_TOOLS - -#ifdef EKAT_ENABLE_KOKKOS_TOOLS -#define KOKKOS_TOOLS_LIBS "@KOKKOS_TOOLS_LIBS@" -#endif - -#endif