From 6951d3a9387928a43d579c03826eaf81e4109877 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 1 Aug 2022 15:30:32 -0700 Subject: [PATCH 01/37] Prototype for NNFDM implementation in AXL It basically compiles and cannot be fully tested until we have an operational server. --- CMakeLists.txt | 13 +++ cmake/FindNNFDM.cmake | 41 ++++++++ cmake/config.h.in | 1 + src/CMakeLists.txt | 4 + src/axl.c | 59 ++++++++++- src/axl.h | 1 + src/axl_async_nnfdm.c | 224 ++++++++++++++++++++++++++++++++++++++++++ src/axl_async_nnfdm.h | 16 +++ 8 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 cmake/FindNNFDM.cmake create mode 100644 src/axl_async_nnfdm.c create mode 100644 src/axl_async_nnfdm.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e34a3c..e1c7430 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,9 @@ MESSAGE(STATUS "ENABLE_IBM_BBAPI: ${ENABLE_IBM_BBAPI}") OPTION(ENABLE_CRAY_DW "Whether to enable Cray Datawarp support" OFF) MESSAGE(STATUS "ENABLE_CRAY_DW: ${ENABLE_CRAY_DW}") +OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" ON) +MESSAGE(STATUS "ENABLE_HPE_NNFDM: ${ENABLE_HPE_NNFDM}") + OPTION(ENABLE_TESTS "Whether to build tests" ON) MESSAGE(STATUS "ENABLE_TESTS: ${ENABLE_TESTS}") @@ -80,6 +83,16 @@ IF(ENABLE_CRAY_DW) # LIST(APPEND AXL_LINK_LINE " -L${WITH_DATAWARP_PREFIX}/lib64 -ldatawarp") ENDIF(ENABLE_CRAY_DW) +## DataMover +IF(ENABLE_HPE_NNFDM) + FIND_PACKAGE(NNFDM) + IF(NNFDM_FOUND) + SET(HAVE_NNFDM TRUE) + INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIRS}) + LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) + ENDIF(NNFDM_FOUND) +ENDIF(ENABLE_HPE_NNFDM) + ## IBM Burst Buffer API IF(ENABLE_IBM_BBAPI) FIND_PACKAGE(BBAPI) diff --git a/cmake/FindNNFDM.cmake b/cmake/FindNNFDM.cmake new file mode 100644 index 0000000..b600037 --- /dev/null +++ b/cmake/FindNNFDM.cmake @@ -0,0 +1,41 @@ +# - Try to find libnnfdm +# +# Once done this will define +# NNFDM_FOUND - System has libdatawarp +# NNFDM_INCLUDE_DIRS - The libdatawarp include directories +# NNFDM_LIBRARIES - The libraries needed to use libdatawarp +# +# This is early days for this library. For now, we assume that there is +# a the following directory and contents pointed to by WITH_NNFDM_PREFIX: +# +# path/nnfdm/ +# lib64/ +# libnnfdm.a +# include/ +# datamovement.pb-c.h +# nnfdm.h +# +# So, the following cmake line will cause this to be found assuming the prefix exists: +# cmake -DWITH_NNFDM_PREFIX="/usr/WS2/martymcf/scr/dm/nnfdm" -DMPI=ON .. + +FIND_LIBRARY(NNFDM_LIBRARIES + NAMES nnfdm + HINTS ${WITH_NNFDM_PREFIX}/lib64 +) + +FIND_PATH(NNFDM_INCLUDE_DIRS + NAMES nnfdm.h + HINTS ${WITH_NNFDM_PREFIX}/include +) + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(NNFDM DEFAULT_MSG + NNFDM_LIBRARIES + NNFDM_INCLUDE_DIRS +) + +# Hide these vars from ccmake GUI +MARK_AS_ADVANCED( + NNFDM_LIBRARIES + NNFDM_INCLUDE_DIRS +) diff --git a/cmake/config.h.in b/cmake/config.h.in index cb1da59..33f015a 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -1,5 +1,6 @@ // Machine Specific Libs #cmakedefine HAVE_PTHREADS #cmakedefine HAVE_DATAWARP +#cmakedefine HAVE_NNFDM #cmakedefine HAVE_BBAPI #cmakedefine HAVE_BBAPI_FALLBACK diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fda3f4b..7af81b3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,10 @@ IF(HAVE_DATAWARP) LIST(APPEND libaxl_srcs axl_async_datawarp.c) ENDIF(HAVE_DATAWARP) +IF(HAVE_NNFDM) + LIST(APPEND libaxl_srcs axl_async_nnfdm.c) +ENDIF(HAVE_NNFDM) + # Default AXL library is withOUT MPI ADD_LIBRARY(axl_o OBJECT ${libaxl_srcs}) diff --git a/src/axl.c b/src/axl.c index f674e74..656e12d 100644 --- a/src/axl.c +++ b/src/axl.c @@ -38,6 +38,10 @@ #include "axl_async_bbapi.h" #endif /* HAVE_BBAPI */ +#ifdef HAVE_NNFDM +#include "axl_async_nnfdm.h" +#endif /* HAVE_NNFDM */ + #ifdef HAVE_DATAWARP #include "axl_async_datawarp.h" #endif /* HAVE_DATAWARP */ @@ -91,6 +95,7 @@ static unsigned int axl_kvtrees_count = 0; static int bbapi_is_loaded = 0; #endif + /* Allocate a new kvtree and return the AXL ID for it. If state_file is * specified, then populate the kvtree with it's data. */ static int axl_alloc_id(const char* state_file) @@ -205,10 +210,12 @@ static axl_xfer_t axl_detect_native_xfer(void) * DataWarp libraries. In the real world, our supercomputer is only going * to have one of those libraries, so just use whatever we find at * build time. */ -#ifdef HAVE_BBAPI +#if defined(HAVE_BBAPI) xtype = AXL_XFER_ASYNC_BBAPI; -#elif HAVE_DATAWARP +#elif defined(HAVE_DATAWARP) xtype = AXL_XFER_ASYNC_DW; +#elif defined(HAVE_NNFDM) + xtype = AXL_XFER_ASYNC_NNFDM; #else xtype = AXL_XFER_SYNC; #endif @@ -293,6 +300,11 @@ int AXL_Finalize (void) } #endif +#ifdef HAVE_NNFDM + axl_async_finalize_nnfdm(); +#endif + + /* decrement reference count and free data structures on last call */ axl_init_count--; if (axl_init_count == 0) { @@ -688,6 +700,15 @@ int AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file) #endif /* HAVE_DATAWARP */ break; + case AXL_XFER_ASYNC_NNFDM: +#ifndef HAVE_NNFDM + AXL_ERR("NNFDM requested but not enabled during build"); + rc = AXL_FAILURE; +#else + axl_async_init_nnfdm(); +#endif /* HAVE_NNFDM */ + break; + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -826,6 +847,11 @@ static int __AXL_Add (int id, const char* src, const char* dest) break; #endif /* HAVE_DATAWARP */ +#ifdef HAVE_NNFDM + case AXL_XFER_ASYNC_NNFDM: + break; +#endif /* HAVE_NNFDM */ + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -1146,6 +1172,17 @@ int __AXL_Dispatch (int id, int resume) break; #endif /* HAVE_DATAWARP */ +#ifdef HAVE_NNFDM + case AXL_XFER_ASYNC_NNFDM: + if (resume) { + AXL_ERR("AXL_Resume() isn't supported yet for NNFDM"); + rc = AXL_FAILURE; + break; + } + rc = axl_async_start_nnfdm(id); + break; +#endif /* HAVE_NNFDM */ + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -1297,6 +1334,12 @@ int AXL_Test (int id) break; #endif /* HAVE_DATAWARP */ +#ifdef HAVE_NNFDM + case AXL_XFER_ASYNC_NNFDM: + rc = axl_async_test_nnfdm(id); + break; +#endif /* HAVE_NNFDM */ + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -1367,6 +1410,12 @@ int AXL_Wait (int id) break; #endif /* HAVE_DATAWARP */ +#ifdef HAVE_NNFDM + case AXL_XFER_ASYNC_NNFDM: + rc = axl_async_wait_nnfdm(id); + break; +#endif /* HAVE_NNDFM */ + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -1465,6 +1514,12 @@ int AXL_Cancel (int id) break; #endif +#if 0 + case AXL_XFER_ASYNC_NNFDM: + rc = axl_async_cancel_nnfdm(id); + break; +#endif + default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; diff --git a/src/axl.h b/src/axl.h index 261eac6..6f2e1e9 100644 --- a/src/axl.h +++ b/src/axl.h @@ -42,6 +42,7 @@ typedef enum { AXL_XFER_SYNC, /* synchronous copy */ AXL_XFER_ASYNC_DAEMON, /* async daemon process (not used, but kept to maintain enum values) */ AXL_XFER_ASYNC_DW, /* Cray Datawarp */ + AXL_XFER_ASYNC_NNFDM, /* HPE DataMover */ AXL_XFER_ASYNC_BBAPI, /* IBM Burst Buffer API */ AXL_XFER_NATIVE, /* Autodetect and use the native API (BBAPI, DW, * etc) for this node type. It may or may not diff --git a/src/axl_async_nnfdm.c b/src/axl_async_nnfdm.c new file mode 100644 index 0000000..4224229 --- /dev/null +++ b/src/axl_async_nnfdm.c @@ -0,0 +1,224 @@ +#include /* For: sleep */ +#include "axl_internal.h" +#include "datamovement.pb-c.h" +#include "kvtree.h" +#include "nnfdm.h" + +static GoInt32 nnfdm_connection_ref = 0; +static int nnfdm_initialized = 0; + +/* TODO: Move to appropriate header file for key/value keys */ +#define AXL_KEY_FILE_SESSION_UID "NNFDM_Session_ID" + + +void axl_async_init_nnfdm() { + if (!nnfdm_initialized) { + nnfdm_initialized++; + char *socketAddr = "/var/run/nnf-dm.sock"; + + struct OpenConnection_return conn; + conn = OpenConnection(socketAddr); + if (conn.r1 != 0) { + axl_abort(-1, + "NNFDM OpenConnection(%s) failed with error %d @ %s:%d", + socketAddr, conn.r1, __FILE__, __LINE__ + ); + } + nnfdm_connection_ref = conn.r0; + } +} + +void axl_async_finalize_nnfdm() { + if (nnfdm_initialized) { + nnfdm_initialized = 0; + + CloseConnection(nnfdm_connection_ref); + /* TODO: No return value from here? */ + } +} + +static int axl_async_stat_nnfdm(char* uid) { + struct Status_return statusResponse = Status(uid); + int state = statusResponse.r0; + int status = statusResponse.r1; + int rc = statusResponse.r2; + + if (rc != 0) { + axl_abort(-1, + "Data Mover Status(%s) failed with error %d @ %s:%d", + uid, rc, __FILE__, __LINE__ + ); + } + + switch (state) { + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__PENDING: + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__STARTING: + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__RUNNING: + status = AXL_STATUS_INPROG; + break; + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__COMPLETED: + switch (status) { + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__SUCCESS: + status = AXL_STATUS_DEST; + break; + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__INVALID: + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__NOT_FOUND: + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__FAILED: + case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__UNKNOWN_STATUS: + status = AXL_STATUS_ERROR; + break; + default: + axl_abort(-1, + "Data Mover Status(%s) Unknown status: %d @ %s:%d", + uid, status, __FILE__, __LINE__ + ); + /*NOTREACHED*/ + break; + } + break; + default: + axl_abort(-1, + "Data Mover Status(%s) Unknown state: %d @ %s:%d", + uid, state, __FILE__, __LINE__ + ); + /*NOTREACHED*/ + break; + } + return status; +} + +int axl_async_start_nnfdm(int id) { + /* Record that we started transfer of this file list */ + kvtree* file_list = axl_kvtrees[id]; + kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); + + /* iterate over files */ + kvtree_elem* elem; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + /* get the filename */ + char* file = kvtree_elem_key(elem); + + /* get the hash for this file */ + kvtree* elem_hash = kvtree_elem_hash(elem); + + /* get the destination for this file */ + char* dest_file; + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dest_file); + + struct Create_return createResponse; + createResponse = Create(file, dest_file); + if (createResponse.r1 != 0) { + axl_abort(-1, + "Data Mover Create(%s, %s) failed with error %d @ %s:%d", + file, dest_file, createResponse.r1, __FILE__, __LINE__ + ); + } + + // Get status of the data movement task. + struct Status_return statusResponse; + statusResponse = Status(createResponse.r0); + if (statusResponse.r2 != 0) { + axl_abort(-1, + "Data Mover Status(%s) failed with error %d @ %s:%d", + createResponse.r0, statusResponse.r2, __FILE__, __LINE__ + ); + } + + /* record that the file is in progress */ + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, axl_async_stat_nnfdm(createResponse.r0)); + + /* record the session Id for this file */ + kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, createResponse.r0); + } + + return AXL_SUCCESS; +} + +int axl_async_test_nnfdm(int id) { + kvtree* file_list = axl_kvtrees[id]; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + + /* iterate/wait over in-progress files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + int status; + char* uid; + + kvtree* elem_hash = kvtree_elem_hash(elem); + kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); + + if (status == AXL_STATUS_DEST) { + continue; /* This one is done */ + } + + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + status = axl_async_stat_nnfdm(uid); + + if (status != AXL_STATUS_DEST) { + return AXL_FAILURE; /* At least one file is not done */ + } + + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); + } + + return AXL_SUCCESS; +} + +int axl_async_wait_nnfdm(int id) { + kvtree* file_list = axl_kvtrees[id]; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + + /* iterate/wait over in-progress files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + int status; + char* uid; + + kvtree* elem_hash = kvtree_elem_hash(elem); + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + + for (status = axl_async_stat_nnfdm(uid); status == AXL_STATUS_INPROG; status = axl_async_stat_nnfdm(uid)) { + sleep(1); + } + + /* Delete the request */ + struct Delete_return deleteResponse; + deleteResponse = Delete(uid); + if (deleteResponse.r1 != 0) { + char* file = kvtree_elem_key(elem); + + axl_abort(-1, + "Delete(uid=%s, filename=%s) for %s failed with %d @ %s:%d", + uid, file, __FILE__, __LINE__ + ); + } + Free(uid); + + + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); + } + + /* iterate over files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + kvtree* elem_hash = kvtree_elem_hash(elem); + + int status; + kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); + + switch (status) { + case AXL_STATUS_DEST: + break; + case AXL_STATUS_SOURCE: + case AXL_STATUS_ERROR: + default: + axl_abort(-1, + "Wait operation called on file with invalid status @ %s:%d", + __FILE__, __LINE__ + ); + } + } + + /* record transfer complete */ + kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_DEST); + return AXL_SUCCESS; +} + diff --git a/src/axl_async_nnfdm.h b/src/axl_async_nnfdm.h new file mode 100644 index 0000000..3c9b6ea --- /dev/null +++ b/src/axl_async_nnfdm.h @@ -0,0 +1,16 @@ +#ifndef AXL_ASYNC_NNFDM_H +#define AXL_ASYNC_NNFDM_H + +/** \file axl_async_nnfdm.h + * \ingroup axl + * \brief implementation of axl for Hpe datamoveer */ + +/** \name nnfdm */ +///@{ +void axl_async_init_nnfdm(); +void axl_async_finalize_nnfdm(); +int axl_async_start_nnfdm(int id); +int axl_async_test_nnfdm(int id); +int axl_async_wait_nnfdm(int id); +///@} +#endif //AXL_ASYNC_NNFDM_H From ffb4dcc43fe97ef6eb4b91bd637250b64dad81a0 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 4 Oct 2022 16:39:10 -0700 Subject: [PATCH 02/37] Use new HPE C++ NNFDM library --- CMakeLists.txt | 10 +- cmake/FindNNFDM.cmake | 41 -------- src/CMakeLists.txt | 2 +- src/axl.c | 12 +-- src/axl_async_nnfdm.c | 224 --------------------------------------- src/axl_async_nnfdm.h | 16 --- src/nnfdm.cpp | 237 ++++++++++++++++++++++++++++++++++++++++++ src/nnfdm.h | 18 ++++ 8 files changed, 266 insertions(+), 294 deletions(-) delete mode 100644 cmake/FindNNFDM.cmake delete mode 100644 src/axl_async_nnfdm.c delete mode 100644 src/axl_async_nnfdm.h create mode 100644 src/nnfdm.cpp create mode 100644 src/nnfdm.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e1c7430..9297570 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,12 +85,10 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) - FIND_PACKAGE(NNFDM) - IF(NNFDM_FOUND) - SET(HAVE_NNFDM TRUE) - INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIRS}) - LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) - ENDIF(NNFDM_FOUND) + FIND_PACKAGE(nnfdm REQUIRED) + SET(HAVE_NNFDM TRUE) + INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) + LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) ENDIF(ENABLE_HPE_NNFDM) ## IBM Burst Buffer API diff --git a/cmake/FindNNFDM.cmake b/cmake/FindNNFDM.cmake deleted file mode 100644 index b600037..0000000 --- a/cmake/FindNNFDM.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# - Try to find libnnfdm -# -# Once done this will define -# NNFDM_FOUND - System has libdatawarp -# NNFDM_INCLUDE_DIRS - The libdatawarp include directories -# NNFDM_LIBRARIES - The libraries needed to use libdatawarp -# -# This is early days for this library. For now, we assume that there is -# a the following directory and contents pointed to by WITH_NNFDM_PREFIX: -# -# path/nnfdm/ -# lib64/ -# libnnfdm.a -# include/ -# datamovement.pb-c.h -# nnfdm.h -# -# So, the following cmake line will cause this to be found assuming the prefix exists: -# cmake -DWITH_NNFDM_PREFIX="/usr/WS2/martymcf/scr/dm/nnfdm" -DMPI=ON .. - -FIND_LIBRARY(NNFDM_LIBRARIES - NAMES nnfdm - HINTS ${WITH_NNFDM_PREFIX}/lib64 -) - -FIND_PATH(NNFDM_INCLUDE_DIRS - NAMES nnfdm.h - HINTS ${WITH_NNFDM_PREFIX}/include -) - -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(NNFDM DEFAULT_MSG - NNFDM_LIBRARIES - NNFDM_INCLUDE_DIRS -) - -# Hide these vars from ccmake GUI -MARK_AS_ADVANCED( - NNFDM_LIBRARIES - NNFDM_INCLUDE_DIRS -) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7af81b3..7419de6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,7 +35,7 @@ IF(HAVE_DATAWARP) ENDIF(HAVE_DATAWARP) IF(HAVE_NNFDM) - LIST(APPEND libaxl_srcs axl_async_nnfdm.c) + LIST(APPEND libaxl_srcs nnfdm.cpp) ENDIF(HAVE_NNFDM) # Default AXL library is withOUT MPI diff --git a/src/axl.c b/src/axl.c index 656e12d..5b45058 100644 --- a/src/axl.c +++ b/src/axl.c @@ -39,7 +39,7 @@ #endif /* HAVE_BBAPI */ #ifdef HAVE_NNFDM -#include "axl_async_nnfdm.h" +#include "nnfdm.h" #endif /* HAVE_NNFDM */ #ifdef HAVE_DATAWARP @@ -301,7 +301,7 @@ int AXL_Finalize (void) #endif #ifdef HAVE_NNFDM - axl_async_finalize_nnfdm(); + nnfdm_finalize(); #endif @@ -705,7 +705,7 @@ int AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file) AXL_ERR("NNFDM requested but not enabled during build"); rc = AXL_FAILURE; #else - axl_async_init_nnfdm(); + nnfdm_init("socketname", "workflowname", "namespace"); #endif /* HAVE_NNFDM */ break; @@ -1179,7 +1179,7 @@ int __AXL_Dispatch (int id, int resume) rc = AXL_FAILURE; break; } - rc = axl_async_start_nnfdm(id); + rc = nnfdm_start(id); break; #endif /* HAVE_NNFDM */ @@ -1336,7 +1336,7 @@ int AXL_Test (int id) #ifdef HAVE_NNFDM case AXL_XFER_ASYNC_NNFDM: - rc = axl_async_test_nnfdm(id); + rc = nnfdm_test(id); break; #endif /* HAVE_NNFDM */ @@ -1412,7 +1412,7 @@ int AXL_Wait (int id) #ifdef HAVE_NNFDM case AXL_XFER_ASYNC_NNFDM: - rc = axl_async_wait_nnfdm(id); + rc = nnfdm_wait(id); break; #endif /* HAVE_NNDFM */ diff --git a/src/axl_async_nnfdm.c b/src/axl_async_nnfdm.c deleted file mode 100644 index 4224229..0000000 --- a/src/axl_async_nnfdm.c +++ /dev/null @@ -1,224 +0,0 @@ -#include /* For: sleep */ -#include "axl_internal.h" -#include "datamovement.pb-c.h" -#include "kvtree.h" -#include "nnfdm.h" - -static GoInt32 nnfdm_connection_ref = 0; -static int nnfdm_initialized = 0; - -/* TODO: Move to appropriate header file for key/value keys */ -#define AXL_KEY_FILE_SESSION_UID "NNFDM_Session_ID" - - -void axl_async_init_nnfdm() { - if (!nnfdm_initialized) { - nnfdm_initialized++; - char *socketAddr = "/var/run/nnf-dm.sock"; - - struct OpenConnection_return conn; - conn = OpenConnection(socketAddr); - if (conn.r1 != 0) { - axl_abort(-1, - "NNFDM OpenConnection(%s) failed with error %d @ %s:%d", - socketAddr, conn.r1, __FILE__, __LINE__ - ); - } - nnfdm_connection_ref = conn.r0; - } -} - -void axl_async_finalize_nnfdm() { - if (nnfdm_initialized) { - nnfdm_initialized = 0; - - CloseConnection(nnfdm_connection_ref); - /* TODO: No return value from here? */ - } -} - -static int axl_async_stat_nnfdm(char* uid) { - struct Status_return statusResponse = Status(uid); - int state = statusResponse.r0; - int status = statusResponse.r1; - int rc = statusResponse.r2; - - if (rc != 0) { - axl_abort(-1, - "Data Mover Status(%s) failed with error %d @ %s:%d", - uid, rc, __FILE__, __LINE__ - ); - } - - switch (state) { - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__PENDING: - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__STARTING: - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__RUNNING: - status = AXL_STATUS_INPROG; - break; - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATE__COMPLETED: - switch (status) { - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__SUCCESS: - status = AXL_STATUS_DEST; - break; - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__INVALID: - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__NOT_FOUND: - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__FAILED: - case DATAMOVEMENT__DATA_MOVEMENT_STATUS_RESPONSE__STATUS__UNKNOWN_STATUS: - status = AXL_STATUS_ERROR; - break; - default: - axl_abort(-1, - "Data Mover Status(%s) Unknown status: %d @ %s:%d", - uid, status, __FILE__, __LINE__ - ); - /*NOTREACHED*/ - break; - } - break; - default: - axl_abort(-1, - "Data Mover Status(%s) Unknown state: %d @ %s:%d", - uid, state, __FILE__, __LINE__ - ); - /*NOTREACHED*/ - break; - } - return status; -} - -int axl_async_start_nnfdm(int id) { - /* Record that we started transfer of this file list */ - kvtree* file_list = axl_kvtrees[id]; - kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); - - /* iterate over files */ - kvtree_elem* elem; - kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); - for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { - /* get the filename */ - char* file = kvtree_elem_key(elem); - - /* get the hash for this file */ - kvtree* elem_hash = kvtree_elem_hash(elem); - - /* get the destination for this file */ - char* dest_file; - kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dest_file); - - struct Create_return createResponse; - createResponse = Create(file, dest_file); - if (createResponse.r1 != 0) { - axl_abort(-1, - "Data Mover Create(%s, %s) failed with error %d @ %s:%d", - file, dest_file, createResponse.r1, __FILE__, __LINE__ - ); - } - - // Get status of the data movement task. - struct Status_return statusResponse; - statusResponse = Status(createResponse.r0); - if (statusResponse.r2 != 0) { - axl_abort(-1, - "Data Mover Status(%s) failed with error %d @ %s:%d", - createResponse.r0, statusResponse.r2, __FILE__, __LINE__ - ); - } - - /* record that the file is in progress */ - kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, axl_async_stat_nnfdm(createResponse.r0)); - - /* record the session Id for this file */ - kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, createResponse.r0); - } - - return AXL_SUCCESS; -} - -int axl_async_test_nnfdm(int id) { - kvtree* file_list = axl_kvtrees[id]; - kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); - - /* iterate/wait over in-progress files */ - for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { - int status; - char* uid; - - kvtree* elem_hash = kvtree_elem_hash(elem); - kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); - - if (status == AXL_STATUS_DEST) { - continue; /* This one is done */ - } - - kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); - status = axl_async_stat_nnfdm(uid); - - if (status != AXL_STATUS_DEST) { - return AXL_FAILURE; /* At least one file is not done */ - } - - kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); - } - - return AXL_SUCCESS; -} - -int axl_async_wait_nnfdm(int id) { - kvtree* file_list = axl_kvtrees[id]; - kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); - - /* iterate/wait over in-progress files */ - for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { - int status; - char* uid; - - kvtree* elem_hash = kvtree_elem_hash(elem); - kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); - - for (status = axl_async_stat_nnfdm(uid); status == AXL_STATUS_INPROG; status = axl_async_stat_nnfdm(uid)) { - sleep(1); - } - - /* Delete the request */ - struct Delete_return deleteResponse; - deleteResponse = Delete(uid); - if (deleteResponse.r1 != 0) { - char* file = kvtree_elem_key(elem); - - axl_abort(-1, - "Delete(uid=%s, filename=%s) for %s failed with %d @ %s:%d", - uid, file, __FILE__, __LINE__ - ); - } - Free(uid); - - - kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); - } - - /* iterate over files */ - for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { - kvtree* elem_hash = kvtree_elem_hash(elem); - - int status; - kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); - - switch (status) { - case AXL_STATUS_DEST: - break; - case AXL_STATUS_SOURCE: - case AXL_STATUS_ERROR: - default: - axl_abort(-1, - "Wait operation called on file with invalid status @ %s:%d", - __FILE__, __LINE__ - ); - } - } - - /* record transfer complete */ - kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_DEST); - return AXL_SUCCESS; -} - diff --git a/src/axl_async_nnfdm.h b/src/axl_async_nnfdm.h deleted file mode 100644 index 3c9b6ea..0000000 --- a/src/axl_async_nnfdm.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef AXL_ASYNC_NNFDM_H -#define AXL_ASYNC_NNFDM_H - -/** \file axl_async_nnfdm.h - * \ingroup axl - * \brief implementation of axl for Hpe datamoveer */ - -/** \name nnfdm */ -///@{ -void axl_async_init_nnfdm(); -void axl_async_finalize_nnfdm(); -int axl_async_start_nnfdm(int id); -int axl_async_test_nnfdm(int id); -int axl_async_wait_nnfdm(int id); -///@} -#endif //AXL_ASYNC_NNFDM_H diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp new file mode 100644 index 0000000..d539c31 --- /dev/null +++ b/src/nnfdm.cpp @@ -0,0 +1,237 @@ +#include "nnfdm/client.h" + +extern "C" { +#include "nnfdm.h" +#include "axl_internal.h" +#include "kvtree.h" +} + +near_node_flash::data_movement::DataMoverClient* nnfdm_client{nullptr}; +near_node_flash::data_movement::Workflow* nnfdm_workflow{nullptr}; + +extern "C" { + +// TODO: Need to move this where other keys are stored +#define AXL_KEY_FILE_SESSION_UID "NNFDM_Session_ID" + +static int nnfdm_stat(const char* uid, int64_t max_seconds_to_wait) +{ + int rval = 0; + near_node_flash::data_movement::StatusRequest request{std::string{uid}, max_seconds_to_wait}; + near_node_flash::data_movement::StatusResponse response; + + near_node_flash::data_movement::RPCStatus status = nnfdm_client->Status(*nnfdm_workflow, request, &response); + if (!status.ok()) { + axl_abort(-1, + "NNFDM Status RPC FAILED %d: %s @ %s:%d", + status.error_code(), status.error_message().c_str(), __FILE__, __LINE__ + ); + /*NOTREACHED*/ + } + + switch (response.state()) { + case near_node_flash::data_movement::StatusResponse::STATE_PENDING: + case near_node_flash::data_movement::StatusResponse::STATE_STARTING: + case near_node_flash::data_movement::StatusResponse::STATE_RUNNING: + rval = AXL_STATUS_INPROG; + break; + case near_node_flash::data_movement::StatusResponse::STATE_COMPLETED: + switch (response.status()) { + case near_node_flash::data_movement::StatusResponse::STATUS_SUCCESS: + rval = AXL_STATUS_DEST; + break; + default: + rval = AXL_STATUS_ERROR; + axl_err("NNFDM Offload Status UNSUCCESSFUL: %d %s", response.status(), response.message().c_str()); + break; + } + default: + axl_abort(-1, "NNFDM Offload State STATE UNKNOWN: %d %s", response.status(), response.message().c_str()); + /*NOTREACHED*/ + break; + } + + return rval; +} + +void nnfdm_init(const char* sname, const char* workflow, const char* name_space) +{ + if (nnfdm_client == nullptr) { + nnfdm_client = new near_node_flash::data_movement::DataMoverClient{std::string{sname}}; + if (nnfdm_client == nullptr) { + axl_abort(-1, + "NNFDM init: Failed to create data movement client instance for %s@ %s:%d", + sname, __FILE__, __LINE__ + ); + } + + nnfdm_workflow = new near_node_flash::data_movement::Workflow{std::string{workflow}, std::string{name_space}}; + if (nnfdm_workflow == nullptr) { + axl_abort(-1, + "nnfdm_init: Failed to create data movement workflow instance for %s/%s@ %s:%d", + workflow, name_space, __FILE__, __LINE__ + ); + } + } +} + +void nnfdm_finalize() +{ + if (nnfdm_client) { + delete nnfdm_workflow; + delete nnfdm_client; + + nnfdm_workflow = nullptr; + nnfdm_client = nullptr; + } +} + +int nnfdm_start(int id) +{ + /* Record that we started transfer of this file list */ + kvtree* file_list = axl_kvtrees[id]; + kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); + + /* iterate over files */ + kvtree_elem* elem; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + /* get the filename */ + char* file = kvtree_elem_key(elem); + + /* get the hash for this file */ + kvtree* elem_hash = kvtree_elem_hash(elem); + + /* get the destination for this file */ + char* dest_file; + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dest_file); + + near_node_flash::data_movement::CreateRequest createRequest(std::string{file}, std::string{dest_file}); + near_node_flash::data_movement::CreateResponse createResponse; + + near_node_flash::data_movement::RPCStatus status = nnfdm_client->Create(*nnfdm_workflow, createRequest, &createResponse); + if (!status.ok()) { + axl_abort(-1, + "NNFDM Create(%s, %s) failed with error %d (%s) @ %s:%d", + file, dest_file, status.error_code(), status.error_message().c_str(), __FILE__, __LINE__ + ); + /*NOTREACHED*/ + } + + switch (createResponse.status()) { + case near_node_flash::data_movement::CreateResponse::Status::STATUS_SUCCESS: + /* record that the file is in progress */ + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_INPROG); + + /* record the session Id for this file */ + kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, createResponse.uid().c_str()); + break; + default: + axl_abort(-1, + "NNFDM Offload Create FAILED for %s: error %d (%s) ", + file, createResponse.status(), createResponse.message().c_str() + ); + /*NOTREACHED*/ + break; + } + } + + return AXL_SUCCESS; +} + +int nnfdm_test(int id) { + kvtree* file_list = axl_kvtrees[id]; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + + /* iterate/wait over in-progress files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + int status; + char* uid; + + kvtree* elem_hash = kvtree_elem_hash(elem); + kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); + + if (status == AXL_STATUS_DEST) { + continue; /* This one is done */ + } + + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + status = nnfdm_stat(uid, 0); + + if (status != AXL_STATUS_DEST) { + return AXL_FAILURE; /* At least one file is not done */ + } + + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); + } + + return AXL_SUCCESS; +} + +int nnfdm_wait(int id) +{ + const int64_t max_seconds_to_wait{1}; + kvtree* file_list = axl_kvtrees[id]; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + + /* iterate/wait over in-progress files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + int status; + char* uid; + + kvtree* elem_hash = kvtree_elem_hash(elem); + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + + do { + status = nnfdm_stat(uid, max_seconds_to_wait); + } while (status == AXL_STATUS_INPROG); + + near_node_flash::data_movement::DeleteRequest deleteRequest(std::string{uid}); + near_node_flash::data_movement::DeleteResponse deleteResponse; + + near_node_flash::data_movement::RPCStatus delete_status = nnfdm_client->Delete(*nnfdm_workflow, deleteRequest, &deleteResponse); + if (!delete_status.ok()) { + axl_abort(-1, "NNFDM Delete RPC FAILED: %d (%s)", delete_status.error_code(), delete_status.error_message().c_str()); + /*NOTEACHED*/ + } + + /* Delete the request */ + switch (deleteResponse.status()) { + case near_node_flash::data_movement::DeleteResponse::STATUS_SUCCESS: + break; + default: + char* file = kvtree_elem_key(elem); + axl_abort(-1, + "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s)", + file, deleteResponse.status(), deleteResponse.message().c_str()); + return 1; + } + + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); + } + + /* iterate over files */ + for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + kvtree* elem_hash = kvtree_elem_hash(elem); + + int status; + kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); + + switch (status) { + case AXL_STATUS_DEST: + break; + case AXL_STATUS_SOURCE: + case AXL_STATUS_ERROR: + default: + axl_abort(-1, + "Wait operation called on file with invalid status @ %s:%d", + __FILE__, __LINE__ + ); + } + } + + /* record transfer complete */ + kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_DEST); + return AXL_SUCCESS; +} +} \ No newline at end of file diff --git a/src/nnfdm.h b/src/nnfdm.h new file mode 100644 index 0000000..f79b706 --- /dev/null +++ b/src/nnfdm.h @@ -0,0 +1,18 @@ +#ifndef AXL_ASYNC_NNFDM_H +#define AXL_ASYNC_NNFDM_H + +#ifdef __cplusplus +extern "C" { +#endif + +void nnfdm_init(const char* sname, const char* workflow, const char* name_space); +void nnfdm_finalize(); +int nnfdm_start(int id); +int nnfdm_test(int id); +int nnfdm_wait(int id); + +#ifdef __cplusplus +}; +#endif + +#endif //AXL_ASYNC_NNFDM_H From 18b0cfb3aec98f1ba95170bd6e926a16c7e9d052 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 4 Oct 2022 16:48:38 -0700 Subject: [PATCH 03/37] Leave NNFDM off by default for now --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9297570..c31843b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ MESSAGE(STATUS "ENABLE_IBM_BBAPI: ${ENABLE_IBM_BBAPI}") OPTION(ENABLE_CRAY_DW "Whether to enable Cray Datawarp support" OFF) MESSAGE(STATUS "ENABLE_CRAY_DW: ${ENABLE_CRAY_DW}") -OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" ON) +OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" OFF) MESSAGE(STATUS "ENABLE_HPE_NNFDM: ${ENABLE_HPE_NNFDM}") OPTION(ENABLE_TESTS "Whether to build tests" ON) From 2dc695ca77237e63aaa24adeb659ada8e2cc1fc5 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 17 Oct 2022 15:23:45 -0700 Subject: [PATCH 04/37] Add NNFDM Cancel --- src/axl.c | 24 ++-- src/axl_internal.h | 2 + src/nnfdm.cpp | 317 ++++++++++++++++++++++++++++++++------------- src/nnfdm.h | 1 + 4 files changed, 241 insertions(+), 103 deletions(-) diff --git a/src/axl.c b/src/axl.c index 5b45058..2ae0af8 100644 --- a/src/axl.c +++ b/src/axl.c @@ -826,6 +826,16 @@ static int __AXL_Add (int id, const char* src, const char* dest) case AXL_XFER_SYNC: break; +#ifdef HAVE_DATAWARP + case AXL_XFER_ASYNC_DW: + break; +#endif /* HAVE_DATAWARP */ + +#ifdef HAVE_NNFDM + case AXL_XFER_ASYNC_NNFDM: + break; +#endif /* HAVE_NNFDM */ + #ifdef HAVE_PTHREADS case AXL_XFER_PTHREAD: break; @@ -842,16 +852,6 @@ static int __AXL_Add (int id, const char* src, const char* dest) break; #endif /* HAVE_BBAPI */ -#ifdef HAVE_DATAWARP - case AXL_XFER_ASYNC_DW: - break; -#endif /* HAVE_DATAWARP */ - -#ifdef HAVE_NNFDM - case AXL_XFER_ASYNC_NNFDM: - break; -#endif /* HAVE_NNFDM */ - default: AXL_ERR("Unknown transfer type (%d)", (int) xtype); rc = AXL_FAILURE; @@ -1514,9 +1514,9 @@ int AXL_Cancel (int id) break; #endif -#if 0 +#ifdef HAVE_NNFDM case AXL_XFER_ASYNC_NNFDM: - rc = axl_async_cancel_nnfdm(id); + rc = nnfdm_cancel(id); break; #endif diff --git a/src/axl_internal.h b/src/axl_internal.h index 10d97c6..776b12d 100644 --- a/src/axl_internal.h +++ b/src/axl_internal.h @@ -54,6 +54,8 @@ extern int axl_rank; #define AXL_KEY_FILE_STATUS ("STATUS") #define AXL_KEY_FILE_CRC ("CRC") #define AXL_KEY_STATE_FILE ("STATE_FILE") +#define AXL_KEY_FILE_SESSION_UID ("NNFDM_Session_ID") + /* TRANSFER STATUS */ #define AXL_STATUS_SOURCE (1) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index d539c31..c852f5b 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -6,71 +6,87 @@ extern "C" { #include "kvtree.h" } -near_node_flash::data_movement::DataMoverClient* nnfdm_client{nullptr}; -near_node_flash::data_movement::Workflow* nnfdm_workflow{nullptr}; - -extern "C" { - -// TODO: Need to move this where other keys are stored -#define AXL_KEY_FILE_SESSION_UID "NNFDM_Session_ID" +namespace nnfdm = near_node_flash::data_movement; + +namespace { + nnfdm::DataMoverClient* nnfdm_client{nullptr}; + nnfdm::Workflow* nnfdm_workflow{nullptr}; + + int nnfdm_stat(const char* uid, int64_t max_seconds_to_wait) + { + int rval = 0; + nnfdm::StatusRequest status_request{std::string{uid}, + max_seconds_to_wait}; + nnfdm::StatusResponse status_response; + nnfdm::RPCStatus rpc_status = nnfdm_client->Status( *nnfdm_workflow + , status_request + , &status_response); + if (!rpc_status.ok()) { + axl_abort( -1 + , "NNFDM Status RPC FAILED %d: %s @ %s:%d" + , rpc_status.error_code() + , rpc_status.error_message().c_str() + , __FILE__ + , __LINE__); + /*NOTREACHED*/ + } -static int nnfdm_stat(const char* uid, int64_t max_seconds_to_wait) -{ - int rval = 0; - near_node_flash::data_movement::StatusRequest request{std::string{uid}, max_seconds_to_wait}; - near_node_flash::data_movement::StatusResponse response; - - near_node_flash::data_movement::RPCStatus status = nnfdm_client->Status(*nnfdm_workflow, request, &response); - if (!status.ok()) { - axl_abort(-1, - "NNFDM Status RPC FAILED %d: %s @ %s:%d", - status.error_code(), status.error_message().c_str(), __FILE__, __LINE__ - ); - /*NOTREACHED*/ - } + switch (status_response.state()) { + case nnfdm::StatusResponse::STATE_PENDING: + case nnfdm::StatusResponse::STATE_STARTING: + case nnfdm::StatusResponse::STATE_RUNNING: + rval = AXL_STATUS_INPROG; + break; + case nnfdm::StatusResponse::STATE_COMPLETED: + switch (status_response.status()) { + case nnfdm::StatusResponse::STATUS_SUCCESS: + rval = AXL_STATUS_DEST; + break; + default: + rval = AXL_STATUS_ERROR; + axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d %s" + , status_response.status() + , status_response.message().c_str()); + break; + } + default: + axl_abort( -1 + , "NNFDM Offload State STATE UNKNOWN: %d %s" + , status_response.status() + , status_response.message().c_str()); + /*NOTREACHED*/ + break; + } - switch (response.state()) { - case near_node_flash::data_movement::StatusResponse::STATE_PENDING: - case near_node_flash::data_movement::StatusResponse::STATE_STARTING: - case near_node_flash::data_movement::StatusResponse::STATE_RUNNING: - rval = AXL_STATUS_INPROG; - break; - case near_node_flash::data_movement::StatusResponse::STATE_COMPLETED: - switch (response.status()) { - case near_node_flash::data_movement::StatusResponse::STATUS_SUCCESS: - rval = AXL_STATUS_DEST; - break; - default: - rval = AXL_STATUS_ERROR; - axl_err("NNFDM Offload Status UNSUCCESSFUL: %d %s", response.status(), response.message().c_str()); - break; - } - default: - axl_abort(-1, "NNFDM Offload State STATE UNKNOWN: %d %s", response.status(), response.message().c_str()); - /*NOTREACHED*/ - break; + return rval; } +} // End of empty namepace - return rval; -} +extern "C" { void nnfdm_init(const char* sname, const char* workflow, const char* name_space) { if (nnfdm_client == nullptr) { - nnfdm_client = new near_node_flash::data_movement::DataMoverClient{std::string{sname}}; + nnfdm_client = new nnfdm::DataMoverClient{std::string{sname}}; if (nnfdm_client == nullptr) { - axl_abort(-1, - "NNFDM init: Failed to create data movement client instance for %s@ %s:%d", - sname, __FILE__, __LINE__ - ); + axl_abort(-1 + , "NNFDM init: Failed to create data movement client " + "instance for %s@ %s:%d" + , sname + , __FILE__ + , __LINE__); } - nnfdm_workflow = new near_node_flash::data_movement::Workflow{std::string{workflow}, std::string{name_space}}; + nnfdm_workflow = new nnfdm::Workflow{ std::string{workflow} + , std::string{name_space}}; if (nnfdm_workflow == nullptr) { - axl_abort(-1, - "nnfdm_init: Failed to create data movement workflow instance for %s/%s@ %s:%d", - workflow, name_space, __FILE__, __LINE__ - ); + axl_abort( -1 + , "nnfdm_init: Failed to create data movement workflow " + "instance for %s/%s@ %s:%d" + , workflow + , name_space + , __FILE__ + , __LINE__); } } } @@ -92,45 +108,53 @@ int nnfdm_start(int id) kvtree* file_list = axl_kvtrees[id]; kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); - /* iterate over files */ kvtree_elem* elem; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); - for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { - /* get the filename */ - char* file = kvtree_elem_key(elem); - - /* get the hash for this file */ + for ( kvtree_elem* elem = kvtree_elem_first(files); + elem != NULL; + elem = kvtree_elem_next(elem)) + { + char* src_filename = kvtree_elem_key(elem); kvtree* elem_hash = kvtree_elem_hash(elem); /* get the destination for this file */ - char* dest_file; - kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dest_file); - - near_node_flash::data_movement::CreateRequest createRequest(std::string{file}, std::string{dest_file}); - near_node_flash::data_movement::CreateResponse createResponse; - - near_node_flash::data_movement::RPCStatus status = nnfdm_client->Create(*nnfdm_workflow, createRequest, &createResponse); - if (!status.ok()) { - axl_abort(-1, - "NNFDM Create(%s, %s) failed with error %d (%s) @ %s:%d", - file, dest_file, status.error_code(), status.error_message().c_str(), __FILE__, __LINE__ - ); + char* dst_filename; + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); + + nnfdm::CreateRequest create_request( std::string{src_filename} + , std::string{dst_filename}); + nnfdm::CreateResponse create_response; + nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow + , create_request + , &create_response); + if (!rpc_status.ok()) { + axl_abort( -1 + , "NNFDM Create(%s, %s) failed with error %d (%s) @ %s:%d" + , src_filename + , dst_filename + , rpc_status.error_code() + , rpc_status.error_message().c_str() + , __FILE__ + , __LINE__); /*NOTREACHED*/ } - switch (createResponse.status()) { - case near_node_flash::data_movement::CreateResponse::Status::STATUS_SUCCESS: + switch (create_response.status()) { + case nnfdm::CreateResponse::Status::STATUS_SUCCESS: /* record that the file is in progress */ - kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_INPROG); - - /* record the session Id for this file */ - kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, createResponse.uid().c_str()); + kvtree_util_set_int( elem_hash + , AXL_KEY_FILE_STATUS + , AXL_STATUS_INPROG); + kvtree_util_set_str( elem_hash + , AXL_KEY_FILE_SESSION_UID + , create_response.uid().c_str()); break; default: - axl_abort(-1, - "NNFDM Offload Create FAILED for %s: error %d (%s) ", - file, createResponse.status(), createResponse.message().c_str() - ); + axl_abort( -1 + , "NNFDM Offload Create FAILED for %s: error %d (%s) " + , src_filename + , create_response.status() + , create_response.message().c_str()); /*NOTREACHED*/ break; } @@ -156,7 +180,7 @@ int nnfdm_test(int id) { } kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); - status = nnfdm_stat(uid, 0); + status = nnfdm_stat(uid, -1); // -1 means to wait forever if (status != AXL_STATUS_DEST) { return AXL_FAILURE; /* At least one file is not done */ @@ -168,9 +192,120 @@ int nnfdm_test(int id) { return AXL_SUCCESS; } +int nnfdm_cancel(int id) { + kvtree* file_list = axl_kvtrees[id]; + kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); + + /* iterate/wait over in-progress files */ + for ( kvtree_elem* elem = kvtree_elem_first(files); + elem != NULL; + elem = kvtree_elem_next(elem)) + { + int status; + char* uid; + char* src_filename = kvtree_elem_key(elem); + kvtree* elem_hash = kvtree_elem_hash(elem); + kvtree_util_get_int(elem_hash, AXL_KEY_FILE_STATUS, &status); + + if (status == AXL_STATUS_DEST) { + continue; /* This one is done */ + } + + kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + + nnfdm::CancelRequest cancel_request(uid); + nnfdm::CancelResponse cancel_response; + nnfdm::RPCStatus rpc_status = nnfdm_client->Cancel( *nnfdm_workflow + , cancel_request + , &cancel_response); + if (!rpc_status.ok()) { + axl_abort(-1, + "NNFDM Cancel(uid=%s, file=%s) failed with error %d (%s) @ %s:%d" + , uid + , src_filename + , rpc_status.error_code() + , rpc_status.error_message().c_str() + , __FILE__ + , __LINE__ + ); + /*NOTREACHED*/ + } + + switch (cancel_response.status()) { + case nnfdm::CancelResponse::STATUS_NOT_FOUND: + // + // Assume that unfound uids simply completed previously + // + axl_dbg( 0 + , "NNFDM Cancel(uid=%s, file=%s) NOTFOUND - " + "IGNORING @ %s:%d" + , uid + , src_filename + , __FILE__ + , __LINE__); + continue; + case nnfdm::CancelResponse::STATUS_SUCCESS: + axl_dbg( 0 + , "NNFDM Cancel(uid=%s, file=%s) Canceled @ %s:%d" + , uid + , src_filename + , __FILE__ + , __LINE__); + break; + default: + axl_abort( -1 + , "NNFDM Cancel(uid=%s, file=%s) " + "Failed with error %d - IGNORING @ %s:%d" + , uid + , src_filename + , cancel_response.status() + , __FILE__ + , __LINE__); + /*NOTEACHED*/ + return 1; + } + + // Now delete the associated uid + nnfdm::DeleteRequest delete_request(std::string{uid}); + nnfdm::DeleteResponse deleteResponse; + rpc_status = nnfdm_client->Delete( *nnfdm_workflow + , delete_request + , &deleteResponse); + + if (!rpc_status.ok()) { + axl_abort( -1 + , "NNFDM Delete(uid=%s, file=%s) RPC FAILED: " + "%d (%s) @ %s:%d" + , uid + , src_filename + , rpc_status.error_code() + , rpc_status.error_message().c_str() + , __FILE__ + , __LINE__); + /*NOTEACHED*/ + } + + /* Delete the request */ + switch (deleteResponse.status()) { + case nnfdm::DeleteResponse::STATUS_SUCCESS: + break; + default: + axl_abort(-1, + "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s) @ %s:%d", + src_filename, deleteResponse.status(), deleteResponse.message().c_str(), + __FILE__, __LINE__); + return 1; + } + + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_DEST); + } + + return AXL_SUCCESS; +} + int nnfdm_wait(int id) { - const int64_t max_seconds_to_wait{1}; + const int64_t max_seconds_to_wait{-1}; // Wait forever kvtree* file_list = axl_kvtrees[id]; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); @@ -186,24 +321,24 @@ int nnfdm_wait(int id) status = nnfdm_stat(uid, max_seconds_to_wait); } while (status == AXL_STATUS_INPROG); - near_node_flash::data_movement::DeleteRequest deleteRequest(std::string{uid}); - near_node_flash::data_movement::DeleteResponse deleteResponse; + nnfdm::DeleteRequest delete_request(std::string{uid}); + nnfdm::DeleteResponse deleteResponse; - near_node_flash::data_movement::RPCStatus delete_status = nnfdm_client->Delete(*nnfdm_workflow, deleteRequest, &deleteResponse); - if (!delete_status.ok()) { - axl_abort(-1, "NNFDM Delete RPC FAILED: %d (%s)", delete_status.error_code(), delete_status.error_message().c_str()); + nnfdm::RPCStatus rpc_status = nnfdm_client->Delete(*nnfdm_workflow, delete_request, &deleteResponse); + if (!rpc_status.ok()) { + axl_abort(-1, "NNFDM Delete RPC FAILED: %d (%s)", rpc_status.error_code(), rpc_status.error_message().c_str()); /*NOTEACHED*/ } /* Delete the request */ switch (deleteResponse.status()) { - case near_node_flash::data_movement::DeleteResponse::STATUS_SUCCESS: + case nnfdm::DeleteResponse::STATUS_SUCCESS: break; default: - char* file = kvtree_elem_key(elem); + char* src_filename = kvtree_elem_key(elem); axl_abort(-1, "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s)", - file, deleteResponse.status(), deleteResponse.message().c_str()); + src_filename, deleteResponse.status(), deleteResponse.message().c_str()); return 1; } diff --git a/src/nnfdm.h b/src/nnfdm.h index f79b706..f85b3cf 100644 --- a/src/nnfdm.h +++ b/src/nnfdm.h @@ -10,6 +10,7 @@ void nnfdm_finalize(); int nnfdm_start(int id); int nnfdm_test(int id); int nnfdm_wait(int id); +int nnfdm_cancel(int id); #ifdef __cplusplus }; From bae2a8da5a5d2e939180ae80fd737287e47c7ceb Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 5 Jan 2023 15:45:14 -0800 Subject: [PATCH 05/37] Turn HPE data mover back on --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c31843b..d236d60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ MESSAGE(STATUS "ENABLE_IBM_BBAPI: ${ENABLE_IBM_BBAPI}") OPTION(ENABLE_CRAY_DW "Whether to enable Cray Datawarp support" OFF) MESSAGE(STATUS "ENABLE_CRAY_DW: ${ENABLE_CRAY_DW}") -OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" OFF) +OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" ON) MESSAGE(STATUS "ENABLE_HPE_NNFDM: ${ENABLE_HPE_NNFDM}") OPTION(ENABLE_TESTS "Whether to build tests" ON) @@ -85,6 +85,7 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) + message(STATUS "Looking for NNFDM") FIND_PACKAGE(nnfdm REQUIRED) SET(HAVE_NNFDM TRUE) INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) From d52d9a9ea8cc55cedb3cf07c07c5bd3985f0caa5 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 6 Jan 2023 10:18:35 -0800 Subject: [PATCH 06/37] make nnfdm inclusion automatic --- CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d236d60..cf217e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,11 +85,12 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) - message(STATUS "Looking for NNFDM") - FIND_PACKAGE(nnfdm REQUIRED) - SET(HAVE_NNFDM TRUE) - INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) - LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) + FIND_PACKAGE(NNFDM) + IF(NNFDM_FOUND) + SET(HAVE_NNFDM TRUE) + INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) + LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) + ENDIF(NNFDM_FOUND) ENDIF(ENABLE_HPE_NNFDM) ## IBM Burst Buffer API From e71984927de0e75cabd7606b6707bdfdba28d052 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 14 Feb 2023 15:36:50 -0800 Subject: [PATCH 07/37] Added basic test of nnf-dm library --- test/CMakeLists.txt | 6 ++ test/axl_test_nnfdm.cpp | 163 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 test/axl_test_nnfdm.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c13821f..da4cbb7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,6 +22,12 @@ ADD_EXECUTABLE(test_config test_config.c) TARGET_LINK_LIBRARIES(axl_cp ${axl_lib}) TARGET_LINK_LIBRARIES(test_config ${axl_lib}) +IF(HAVE_NNFDM) + ADD_EXECUTABLE(axl_test_nnfdm axl_test_nnfdm.cpp) + TARGET_LINK_LIBRARIES(axl_test_nnfdm ${axl_lib}) + ADD_TEST(axl_test_nnfdm axl_test_nnfdm) +ENDIF(HAVE_NNFDM) + ################ # Add tests to ctest ################ diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp new file mode 100644 index 0000000..9f04385 --- /dev/null +++ b/test/axl_test_nnfdm.cpp @@ -0,0 +1,163 @@ +#include +#include +#include +#include + +#include "nnfdm/client.h" + +using namespace near_node_flash::data_movement; + +int main(int argc, char** argv) +{ + std::string uid; + + DataMoverClient client("unix:///var/run/nnf-dm.sock"); + + { + // Retrieve the version information + + VersionResponse versionResponse; + + RPCStatus status = client.Version(&versionResponse); + if (!status.ok()) { + std::cerr << "Version RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + std::cout << "Data Movement Version: " << versionResponse.version() << std::endl; + std::cout << "Supported API Versions:" << std::endl; + for (auto version : versionResponse.apiversions()) { + std::cout << "\t" << version << std::endl; + } + + return 0; + } + + const char* workflow_name{getenv("DW_WORKFLOW_NAME")}; + const char* workflow_namespace{getenv("DW_WORKFLOW_NAMESPACE")}; + + if (workflow_name == nullptr) { + std::cerr << "DW_WORKFLOW_NAME environment variableis not set, make sure you are running within an allocation" << std::endl; + } + + if (workflow_namespace == nullptr) { + std::cerr << "DW_WORKFLOW_NAMESPACE environment variableis not set, make sure you are running within an allocation" << std::endl; + } + + Workflow workflow(workflow_name, workflow_namespace); + + { + // Create an offload request + CreateRequest createRequest("/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo", "/p/lslide/martymcf/bar"); + CreateResponse createResponse; + + RPCStatus status = client.Create(workflow, createRequest, &createResponse); + if (!status.ok()) { + std::cout << "Create RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + switch (createResponse.status()) { + case CreateResponse::Status::STATUS_SUCCESS: + uid = createResponse.uid(); + std::cout << "Offload Created: UID: " << createResponse.uid() << std::endl; + break; + default: + std::cout << "Offload Create FAILED: " << createResponse.status() << ": " << createResponse.message() << std::endl; + return 1; + } + } + + { + ListRequest listRequest; + ListResponse listResponse; + + RPCStatus status = client.List(workflow, listRequest, &listResponse); + if (!status.ok()) { + std::cout << "List RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + std::cout << "Offload List: " << listResponse.uids().size() << std::endl; + for (auto uid : listResponse.uids()) { + std::cout << "\t" << uid << std::endl; + } + } + + { + StatusRequest statusRequest(uid, 0); + StatusResponse statusResponse; + +RequestStatus: + RPCStatus status = client.Status(workflow, statusRequest, &statusResponse); + if (!status.ok()) { + std::cout << "Status RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + switch (statusResponse.state()) { + case StatusResponse::STATE_PENDING: + case StatusResponse::STATE_STARTING: + case StatusResponse::STATE_RUNNING: + std::cout << "Offload State Pending/Starting/Running" << std::endl; + goto RequestStatus; + case StatusResponse::STATE_COMPLETED: + std::cout << "Offload State Completed" << std::endl; + break; + default: + std::cout << "Offload State STATE UNKNOWN: " << statusResponse.state() << " Status: " << statusResponse.status() << std::endl; + return 1; + } + + switch (statusResponse.status()) { + case StatusResponse::STATUS_SUCCESS: + std::cout << "Offload Status Successful" << std::endl; + break; + default: + std::cout << "Offload Status UNSUCCESSFUL: " << statusResponse.status() << " " << statusResponse.message() << std::endl; + return 1; + } + } + + { + CancelRequest cancelRequest(uid); + CancelResponse cancelResponse; + + RPCStatus status = client.Cancel(workflow, cancelRequest, &cancelResponse); + if (!status.ok()) { + std::cout << "Cancel RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + switch (cancelResponse.status()) { + case CancelResponse::STATUS_SUCCESS: + std::cout << "Offload Cancel Successful" << std::endl; + break; + default: + std::cout << "Offload Cancel UNSUCCESSFUL: " << cancelResponse.status() << " " << cancelResponse.message() << std::endl; + return 1; + } + } + + { + DeleteRequest deleteRequest(uid); + DeleteResponse deleteResponse; + + RPCStatus status = client.Delete(workflow, deleteRequest, &deleteResponse); + if (!status.ok()) { + std::cout << "Delete RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; + return 1; + } + + switch (deleteResponse.status()) { + case DeleteResponse::STATUS_SUCCESS: + std::cout << "Offload Delete Successful" << std::endl; + break; + default: + std::cout << "Offload Delete UNSUCCESSFUL: " << deleteResponse.status() << " " << deleteResponse.message() << std::endl; + return 1; + } + } + + return 0; +} From 0277f5999986d85b04a18708677baedd9a458b35 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 17 Feb 2023 16:18:28 -0800 Subject: [PATCH 08/37] Checkpoint: SCR now successfully is using nnfdm to copy files. The nnfdm library currently reports conflicting error messages that will be investigated next week --- src/axl.c | 2 +- src/nnfdm.cpp | 78 +++++++++++++++++++++++++++++---------------------- src/nnfdm.h | 2 +- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/axl.c b/src/axl.c index 2ae0af8..93a5d25 100644 --- a/src/axl.c +++ b/src/axl.c @@ -705,7 +705,7 @@ int AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file) AXL_ERR("NNFDM requested but not enabled during build"); rc = AXL_FAILURE; #else - nnfdm_init("socketname", "workflowname", "namespace"); + nnfdm_init(); #endif /* HAVE_NNFDM */ break; diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index c852f5b..aa261b7 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "nnfdm/client.h" extern "C" { @@ -14,13 +17,12 @@ namespace { int nnfdm_stat(const char* uid, int64_t max_seconds_to_wait) { + AXL_DBG(0, "Status(uid=%s, max_seconds_to_wait=%" PRId64 ")", uid, max_seconds_to_wait); int rval = 0; - nnfdm::StatusRequest status_request{std::string{uid}, - max_seconds_to_wait}; + nnfdm::StatusRequest status_request{std::string{uid}, max_seconds_to_wait}; nnfdm::StatusResponse status_response; - nnfdm::RPCStatus rpc_status = nnfdm_client->Status( *nnfdm_workflow - , status_request - , &status_response); + nnfdm::RPCStatus rpc_status{ nnfdm_client->Status(*nnfdm_workflow, status_request, &status_response) }; + if (!rpc_status.ok()) { axl_abort( -1 , "NNFDM Status RPC FAILED %d: %s @ %s:%d" @@ -33,11 +35,19 @@ namespace { switch (status_response.state()) { case nnfdm::StatusResponse::STATE_PENDING: + AXL_DBG(0, "nnfdm::StatusResponse::STATE_PENDING"); + rval = AXL_STATUS_INPROG; + break; case nnfdm::StatusResponse::STATE_STARTING: + AXL_DBG(0, "nnfdm::StatusResponse::STATE_STARTING"); + rval = AXL_STATUS_INPROG; + break; case nnfdm::StatusResponse::STATE_RUNNING: + AXL_DBG(0, "nnfdm::StatusResponse::STATE_RUNNING"); rval = AXL_STATUS_INPROG; break; case nnfdm::StatusResponse::STATE_COMPLETED: + AXL_DBG(0, "nnfdm::StatusResponse::STATE_COMPLETED"); switch (status_response.status()) { case nnfdm::StatusResponse::STATUS_SUCCESS: rval = AXL_STATUS_DEST; @@ -64,27 +74,30 @@ namespace { extern "C" { -void nnfdm_init(const char* sname, const char* workflow, const char* name_space) +void nnfdm_init() { if (nnfdm_client == nullptr) { - nnfdm_client = new nnfdm::DataMoverClient{std::string{sname}}; + const std::string socket_name{"unix:///var/run/nnf-dm.sock"}; + const std::string workflow_name{getenv("DW_WORKFLOW_NAME")}; + const std::string workflow_namespace{getenv("DW_WORKFLOW_NAMESPACE")}; + + nnfdm_client = new nnfdm::DataMoverClient{std::string{socket_name}}; if (nnfdm_client == nullptr) { axl_abort(-1 , "NNFDM init: Failed to create data movement client " "instance for %s@ %s:%d" - , sname + , socket_name.c_str() , __FILE__ , __LINE__); } - nnfdm_workflow = new nnfdm::Workflow{ std::string{workflow} - , std::string{name_space}}; + nnfdm_workflow = new nnfdm::Workflow{workflow_name, workflow_namespace}; if (nnfdm_workflow == nullptr) { axl_abort( -1 , "nnfdm_init: Failed to create data movement workflow " "instance for %s/%s@ %s:%d" - , workflow - , name_space + , workflow_name.c_str() + , workflow_namespace.c_str() , __FILE__ , __LINE__); } @@ -104,6 +117,8 @@ void nnfdm_finalize() int nnfdm_start(int id) { + int rc = AXL_SUCCESS; + /* Record that we started transfer of this file list */ kvtree* file_list = axl_kvtrees[id]; kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); @@ -139,28 +154,23 @@ int nnfdm_start(int id) /*NOTREACHED*/ } - switch (create_response.status()) { - case nnfdm::CreateResponse::Status::STATUS_SUCCESS: - /* record that the file is in progress */ - kvtree_util_set_int( elem_hash - , AXL_KEY_FILE_STATUS - , AXL_STATUS_INPROG); - kvtree_util_set_str( elem_hash - , AXL_KEY_FILE_SESSION_UID - , create_response.uid().c_str()); - break; - default: - axl_abort( -1 - , "NNFDM Offload Create FAILED for %s: error %d (%s) " - , src_filename - , create_response.status() - , create_response.message().c_str()); - /*NOTREACHED*/ - break; + if (create_response.status() == nnfdm::CreateResponse::Status::STATUS_SUCCESS) { + /* record that the file is in progress */ + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_INPROG); + kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, create_response.uid().c_str()); + } + else { + AXL_DBG(0, "Create(%s, %s) FAILED:\n error=%d (%s)" + , src_filename + , dst_filename + , create_response.status() + , create_response.message().c_str()); + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_ERROR); + rc = AXL_FAILURE; } } - return AXL_SUCCESS; + return rc; } int nnfdm_test(int id) { @@ -180,7 +190,7 @@ int nnfdm_test(int id) { } kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); - status = nnfdm_stat(uid, -1); // -1 means to wait forever + status = nnfdm_stat(uid, 1); if (status != AXL_STATUS_DEST) { return AXL_FAILURE; /* At least one file is not done */ @@ -305,7 +315,7 @@ int nnfdm_cancel(int id) { int nnfdm_wait(int id) { - const int64_t max_seconds_to_wait{-1}; // Wait forever + const int64_t max_seconds_to_wait{1}; // wait 1 second kvtree* file_list = axl_kvtrees[id]; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); @@ -369,4 +379,4 @@ int nnfdm_wait(int id) kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_DEST); return AXL_SUCCESS; } -} \ No newline at end of file +} diff --git a/src/nnfdm.h b/src/nnfdm.h index f85b3cf..dbb1968 100644 --- a/src/nnfdm.h +++ b/src/nnfdm.h @@ -5,7 +5,7 @@ extern "C" { #endif -void nnfdm_init(const char* sname, const char* workflow, const char* name_space); +void nnfdm_init(); void nnfdm_finalize(); int nnfdm_start(int id); int nnfdm_test(int id); From 13bdf5f8d1b8b68223b35818eb5f6d5610509ed5 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 27 Feb 2023 09:12:38 -0800 Subject: [PATCH 09/37] Initial working version with single rank --- src/nnfdm.cpp | 85 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index aa261b7..8e8aa03 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -1,3 +1,6 @@ +#include +#include + #include #include @@ -15,13 +18,42 @@ namespace { nnfdm::DataMoverClient* nnfdm_client{nullptr}; nnfdm::Workflow* nnfdm_workflow{nullptr}; - int nnfdm_stat(const char* uid, int64_t max_seconds_to_wait) + std::string print_status(nnfdm::StatusResponse& status_response) + { + std::stringstream ss; + + ss << " stat_response->state()= " << status_response.state(); + switch (status_response.state()) { + case nnfdm::StatusResponse::State::STATE_PENDING: ss << " {STATE_PENDING}"; break; + case nnfdm::StatusResponse::State::STATE_STARTING: ss << " {STATE_STARTING}"; break; + case nnfdm::StatusResponse::State::STATE_RUNNING: ss << " {STATE_RUNNING}"; break; + case nnfdm::StatusResponse::State::STATE_COMPLETED: ss << " {STATE_COMPLETED}"; break; + case nnfdm::StatusResponse::State::STATE_CANCELLING: ss << " {STATE_CANCELLING}"; break; + case nnfdm::StatusResponse::State::STATE_UNKNOWN: ss << " {STATE_UNKNOWN}"; break; + default: ss << " {STATE_???}"; break; + } + + ss << std::endl << " stat_response->status() = " << status_response.status(); + switch (status_response.status()) { + case nnfdm::StatusResponse::Status::STATUS_INVALID: ss << " {STATUS_INVALID}"; break; + case nnfdm::StatusResponse::Status::STATUS_NOT_FOUND: ss << " {STATUS_NOT_FOUND}"; break; + case nnfdm::StatusResponse::Status::STATUS_SUCCESS: ss << " {STATUS_SUCCESS}"; break; + case nnfdm::StatusResponse::Status::STATUS_FAILED: ss << " {STATUS_FAILED}"; break; + case nnfdm::StatusResponse::Status::STATUS_CANCELLED: ss << " {STATUS_CANCELLED}"; break; + case nnfdm::StatusResponse::Status::STATUS_UNKNOWN: ss << " {STATUS_UNKNOWN}"; break; + default: ss << " {STATE_???}"; break; + } + + ss << std::endl << " stat_response->message() = " << status_response.message() << std::endl; + + return ss.str(); + } + + int nnfdm_stat(const char* fname, const char* uid, int64_t max_seconds_to_wait) { - AXL_DBG(0, "Status(uid=%s, max_seconds_to_wait=%" PRId64 ")", uid, max_seconds_to_wait); int rval = 0; - nnfdm::StatusRequest status_request{std::string{uid}, max_seconds_to_wait}; nnfdm::StatusResponse status_response; - nnfdm::RPCStatus rpc_status{ nnfdm_client->Status(*nnfdm_workflow, status_request, &status_response) }; + nnfdm::RPCStatus rpc_status{ nnfdm_client->Status(*nnfdm_workflow, nnfdm::StatusRequest{std::string{uid}, max_seconds_to_wait}, &status_response) }; if (!rpc_status.ok()) { axl_abort( -1 @@ -33,38 +65,34 @@ namespace { /*NOTREACHED*/ } + AXL_DBG(1, "Status of offload(filename=%s, uid=%s) is:\n%s", fname, uid, print_status(status_response).c_str()); + switch (status_response.state()) { - case nnfdm::StatusResponse::STATE_PENDING: - AXL_DBG(0, "nnfdm::StatusResponse::STATE_PENDING"); + case nnfdm::StatusResponse::State::STATE_PENDING: rval = AXL_STATUS_INPROG; break; - case nnfdm::StatusResponse::STATE_STARTING: - AXL_DBG(0, "nnfdm::StatusResponse::STATE_STARTING"); + case nnfdm::StatusResponse::State::STATE_STARTING: rval = AXL_STATUS_INPROG; break; - case nnfdm::StatusResponse::STATE_RUNNING: - AXL_DBG(0, "nnfdm::StatusResponse::STATE_RUNNING"); + case nnfdm::StatusResponse::State::STATE_RUNNING: rval = AXL_STATUS_INPROG; break; - case nnfdm::StatusResponse::STATE_COMPLETED: - AXL_DBG(0, "nnfdm::StatusResponse::STATE_COMPLETED"); - switch (status_response.status()) { - case nnfdm::StatusResponse::STATUS_SUCCESS: - rval = AXL_STATUS_DEST; - break; - default: - rval = AXL_STATUS_ERROR; - axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d %s" - , status_response.status() - , status_response.message().c_str()); - break; + case nnfdm::StatusResponse::State::STATE_COMPLETED: + if (status_response.status() == nnfdm::StatusResponse::Status::STATUS_SUCCESS) { + rval = AXL_STATUS_DEST; + } + else { + rval = AXL_STATUS_ERROR; + axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d %s" + , status_response.status() + , status_response.message().c_str()); } + break; default: axl_abort( -1 , "NNFDM Offload State STATE UNKNOWN: %d %s" , status_response.status() , status_response.message().c_str()); - /*NOTREACHED*/ break; } @@ -81,7 +109,7 @@ void nnfdm_init() const std::string workflow_name{getenv("DW_WORKFLOW_NAME")}; const std::string workflow_namespace{getenv("DW_WORKFLOW_NAMESPACE")}; - nnfdm_client = new nnfdm::DataMoverClient{std::string{socket_name}}; + nnfdm_client = new nnfdm::DataMoverClient{ socket_name }; if (nnfdm_client == nullptr) { axl_abort(-1 , "NNFDM init: Failed to create data movement client " @@ -160,7 +188,7 @@ int nnfdm_start(int id) kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, create_response.uid().c_str()); } else { - AXL_DBG(0, "Create(%s, %s) FAILED:\n error=%d (%s)" + AXL_DBG(1, "Create(%s, %s) FAILED:\n error=%d (%s)" , src_filename , dst_filename , create_response.status() @@ -179,6 +207,7 @@ int nnfdm_test(int id) { /* iterate/wait over in-progress files */ for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + char* src_filename = kvtree_elem_key(elem); int status; char* uid; @@ -190,7 +219,7 @@ int nnfdm_test(int id) { } kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); - status = nnfdm_stat(uid, 1); + status = nnfdm_stat(src_filename, uid, 1); if (status != AXL_STATUS_DEST) { return AXL_FAILURE; /* At least one file is not done */ @@ -321,6 +350,7 @@ int nnfdm_wait(int id) /* iterate/wait over in-progress files */ for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + char* src_filename = kvtree_elem_key(elem); int status; char* uid; @@ -328,7 +358,7 @@ int nnfdm_wait(int id) kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); do { - status = nnfdm_stat(uid, max_seconds_to_wait); + status = nnfdm_stat(src_filename, uid, max_seconds_to_wait); } while (status == AXL_STATUS_INPROG); nnfdm::DeleteRequest delete_request(std::string{uid}); @@ -345,7 +375,6 @@ int nnfdm_wait(int id) case nnfdm::DeleteResponse::STATUS_SUCCESS: break; default: - char* src_filename = kvtree_elem_key(elem); axl_abort(-1, "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s)", src_filename, deleteResponse.status(), deleteResponse.message().c_str()); From e64691e8ab86d19c3e90b7ac38bf9e0f16174380 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 27 Apr 2023 13:24:53 -0700 Subject: [PATCH 10/37] Removed unused code --- src/axl_mpi.c | 53 --------------------------------------------------- 1 file changed, 53 deletions(-) diff --git a/src/axl_mpi.c b/src/axl_mpi.c index 07d9719..13f0032 100644 --- a/src/axl_mpi.c +++ b/src/axl_mpi.c @@ -144,59 +144,6 @@ int AXL_Dispatch_comm ( int id, /**< [IN] - transfer hander ID returned from AXL_Create */ MPI_Comm comm) /**< [IN] - communicator used for coordination and flow control */ { -#if 0 - /* lookup transfer info for the given id */ - kvtree* file_list = NULL; - axl_xfer_t xtype = AXL_XFER_NULL; - axl_xfer_state_t xstate = AXL_XFER_STATE_NULL; - if (axl_get_info(id, &file_list, &xtype, &xstate) != AXL_SUCCESS) { - AXL_ERR("Could not find transfer info for UID %d", id); - return AXL_FAILURE; - } - - /* check that handle is in correct state to dispatch */ - if (xstate != AXL_XFER_STATE_CREATED) { - AXL_ERR("Invalid state to dispatch UID %d", id); - return AXL_FAILURE; - } - kvtree_util_set_int(file_list, AXL_KEY_STATE, (int)AXL_XFER_STATE_DISPATCHED); -#endif - -#if 0 - /* create destination directories for each file */ - if (axl_make_directories) { - /* count number of files we have */ - kvtree* file_list = kvtree_get_kv_int(axl_file_lists, AXL_KEY_HANDLE_UID, id); - kvtree* files_hash = kvtree_get(file_list, AXL_KEY_FILES); - int num_files = kvtree_size(files_hash); - - /* allocate pointer for each one */ - const char** files = (const char**) AXL_MALLOC(num_files * sizeof(char*)); - - /* set pointer to each file */ - int i; - char* dest; - kvtree_elem* elem; - while ((elem = axl_get_next_path(id, elem, NULL, &dest))) { - files[i] = dest; - i++; - } - - /* create directories */ - axl_create_dirs(num_files, files, comm); - - /* free list of files */ - axl_free2(&files); - } - - /* TODO: this is hacky */ - /* delegate remaining work to regular dispatch, - * but disable mkdir since we already did that */ - int make_dir = axl_make_directories; - axl_make_directories = 0; - int rc = AXL_Dispatch(id); - axl_make_directories = make_dir; -#endif /* delegate remaining work to regular dispatch */ int rc = AXL_Dispatch(id); From bc0f07fdd9df8192fd5ab8a2e8257b41f9b22597 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 27 Apr 2023 14:09:38 -0700 Subject: [PATCH 11/37] Minor debug statement updates --- src/nnfdm.cpp | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 8e8aa03..e81acd5 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -1,7 +1,7 @@ #include +#include #include -#include #include #include "nnfdm/client.h" @@ -22,29 +22,29 @@ namespace { { std::stringstream ss; - ss << " stat_response->state()= " << status_response.state(); + ss << "{" << status_response.state() << "/"; switch (status_response.state()) { - case nnfdm::StatusResponse::State::STATE_PENDING: ss << " {STATE_PENDING}"; break; - case nnfdm::StatusResponse::State::STATE_STARTING: ss << " {STATE_STARTING}"; break; - case nnfdm::StatusResponse::State::STATE_RUNNING: ss << " {STATE_RUNNING}"; break; - case nnfdm::StatusResponse::State::STATE_COMPLETED: ss << " {STATE_COMPLETED}"; break; - case nnfdm::StatusResponse::State::STATE_CANCELLING: ss << " {STATE_CANCELLING}"; break; - case nnfdm::StatusResponse::State::STATE_UNKNOWN: ss << " {STATE_UNKNOWN}"; break; - default: ss << " {STATE_???}"; break; + case nnfdm::StatusResponse::State::STATE_PENDING: ss << "STATE_PENDING}"; break; + case nnfdm::StatusResponse::State::STATE_STARTING: ss << "STATE_STARTING}"; break; + case nnfdm::StatusResponse::State::STATE_RUNNING: ss << "STATE_RUNNING}"; break; + case nnfdm::StatusResponse::State::STATE_COMPLETED: ss <<"STATE_COMPLETED}"; break; + case nnfdm::StatusResponse::State::STATE_CANCELLING: ss <<"STATE_CANCELLING}"; break; + case nnfdm::StatusResponse::State::STATE_UNKNOWN: ss << "STATE_UNKNOWN}"; break; + default: ss << "STATE_???}"; break; } - ss << std::endl << " stat_response->status() = " << status_response.status(); + ss << " {" << status_response.status() << "/"; switch (status_response.status()) { - case nnfdm::StatusResponse::Status::STATUS_INVALID: ss << " {STATUS_INVALID}"; break; - case nnfdm::StatusResponse::Status::STATUS_NOT_FOUND: ss << " {STATUS_NOT_FOUND}"; break; - case nnfdm::StatusResponse::Status::STATUS_SUCCESS: ss << " {STATUS_SUCCESS}"; break; - case nnfdm::StatusResponse::Status::STATUS_FAILED: ss << " {STATUS_FAILED}"; break; - case nnfdm::StatusResponse::Status::STATUS_CANCELLED: ss << " {STATUS_CANCELLED}"; break; - case nnfdm::StatusResponse::Status::STATUS_UNKNOWN: ss << " {STATUS_UNKNOWN}"; break; - default: ss << " {STATE_???}"; break; + case nnfdm::StatusResponse::Status::STATUS_INVALID: ss << "STATUS_INVALID}"; break; + case nnfdm::StatusResponse::Status::STATUS_NOT_FOUND: ss << "STATUS_NOT_FOUND}"; break; + case nnfdm::StatusResponse::Status::STATUS_SUCCESS: ss << "STATUS_SUCCESS}"; break; + case nnfdm::StatusResponse::Status::STATUS_FAILED: ss << "STATUS_FAILED}"; break; + case nnfdm::StatusResponse::Status::STATUS_CANCELLED: ss << "STATUS_CANCELLED}"; break; + case nnfdm::StatusResponse::Status::STATUS_UNKNOWN: ss << "STATUS_UNKNOWN}"; break; + default: ss << "STATE_???}"; break; } - ss << std::endl << " stat_response->message() = " << status_response.message() << std::endl; + ss << std::endl << " " << status_response.message(); return ss.str(); } @@ -65,7 +65,7 @@ namespace { /*NOTREACHED*/ } - AXL_DBG(1, "Status of offload(filename=%s, uid=%s) is:\n%s", fname, uid, print_status(status_response).c_str()); + AXL_DBG(1, "Status of offload(filename=%s, uid=%s): %s", fname, uid, print_status(status_response).c_str()); switch (status_response.state()) { case nnfdm::StatusResponse::State::STATE_PENDING: From 2f7479babb1409eee8fbe0daf307c29a185467a0 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 8 May 2023 16:39:49 -0700 Subject: [PATCH 12/37] Updates to build with latest version of nnfdm library --- src/nnfdm.cpp | 4 +++- test/axl_test_nnfdm.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index e81acd5..a94b87c 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -165,7 +165,9 @@ int nnfdm_start(int id) kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); nnfdm::CreateRequest create_request( std::string{src_filename} - , std::string{dst_filename}); + , std::string{dst_filename} + , false + , ""); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow , create_request diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp index 9f04385..cb8260f 100644 --- a/test/axl_test_nnfdm.cpp +++ b/test/axl_test_nnfdm.cpp @@ -48,7 +48,7 @@ int main(int argc, char** argv) { // Create an offload request - CreateRequest createRequest("/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo", "/p/lslide/martymcf/bar"); + CreateRequest createRequest("/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo", "/p/lslide/martymcf/bar", false, ""); CreateResponse createResponse; RPCStatus status = client.Create(workflow, createRequest, &createResponse); From 5c2d38d816a10ccc01dba1d9912eb905b38dfd9a Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 8 May 2023 17:05:19 -0700 Subject: [PATCH 13/37] Update signatures to latest nnfdm library --- src/nnfdm.cpp | 11 +++++++---- test/axl_test_nnfdm.cpp | 22 ++++++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index a94b87c..6fc1cc0 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -164,10 +164,13 @@ int nnfdm_start(int id) char* dst_filename; kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); - nnfdm::CreateRequest create_request( std::string{src_filename} - , std::string{dst_filename} - , false - , ""); + nnfdm::CreateRequest create_request( std::string{src_filename} // Source + , std::string{dst_filename} // Destination + , false // ?? + , "" // ?? + , false // ?? + , false // ?? + ); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow , create_request diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp index cb8260f..a28bbdd 100644 --- a/test/axl_test_nnfdm.cpp +++ b/test/axl_test_nnfdm.cpp @@ -48,22 +48,28 @@ int main(int argc, char** argv) { // Create an offload request - CreateRequest createRequest("/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo", "/p/lslide/martymcf/bar", false, ""); - CreateResponse createResponse; - - RPCStatus status = client.Create(workflow, createRequest, &createResponse); + CreateRequest create_request( "/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo" + , "/p/lslide/martymcf/bar" + , false // ?? + , "" // ?? + , false // ?? + , false // ?? + ); + CreateResponse create_response; + + RPCStatus status = client.Create(workflow, create_request, &create_response); if (!status.ok()) { std::cout << "Create RPC FAILED (" << status.error_code() << "): " << status.error_message() << std::endl; return 1; } - switch (createResponse.status()) { + switch (create_response.status()) { case CreateResponse::Status::STATUS_SUCCESS: - uid = createResponse.uid(); - std::cout << "Offload Created: UID: " << createResponse.uid() << std::endl; + uid = create_response.uid(); + std::cout << "Offload Created: UID: " << create_response.uid() << std::endl; break; default: - std::cout << "Offload Create FAILED: " << createResponse.status() << ": " << createResponse.message() << std::endl; + std::cout << "Offload Create FAILED: " << create_response.status() << ": " << create_response.message() << std::endl; return 1; } } From e14bd94124232126e66634cbbb1667195f41b6cb Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 9 May 2023 16:24:53 -0700 Subject: [PATCH 14/37] Added find dependency for NNFDM if/when NNFDM is included --- cmake/axlConfig.cmake.in | 4 ++++ src/CMakeLists.txt | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/axlConfig.cmake.in b/cmake/axlConfig.cmake.in index 7f6d351..b141ea7 100644 --- a/cmake/axlConfig.cmake.in +++ b/cmake/axlConfig.cmake.in @@ -19,6 +19,10 @@ if (@HAVE_BBAPI@) find_dependency(BBAPI REQUIRED) endif() +if (@HAVE_NNFDM@) + find_dependency(nnfdm REQUIRED) +endif() + if (@MPI@) find_dependency(MPI REQUIRED) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a381c6b..bec0ae0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,10 +38,6 @@ IF(HAVE_NNFDM) LIST(APPEND libaxl_srcs nnfdm.cpp) ENDIF(HAVE_NNFDM) -# Default AXL library is withOUT MPI -ADD_LIBRARY(axl_o OBJECT ${libaxl_srcs}) -TARGET_LINK_LIBRARIES(axl_o PRIVATE ${AXL_EXTERNAL_LIBS}) - IF(BUILD_SHARED_LIBS) # Default AXL library is withOUT MPI ADD_LIBRARY(axl_o OBJECT ${libaxl_srcs}) From 830cee776bbafb09d882eab6f80c9795ef3982bd Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 10 May 2023 13:56:15 -0700 Subject: [PATCH 15/37] Add nnfdm to list of external static libs for AXL --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bf845d..4445fa6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,11 +80,12 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) - FIND_PACKAGE(NNFDM) + FIND_PACKAGE(NNFDM REQUIRED) IF(NNFDM_FOUND) SET(HAVE_NNFDM TRUE) INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) + LIST(APPEND AXL_EXTERNAL_STATIC_LIBS ${NNFDM_LIBRARIES}) ENDIF(NNFDM_FOUND) ENDIF(ENABLE_HPE_NNFDM) From 2958a7451be3594bacf8d8fb489b46cf86630a86 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 27 Jun 2023 10:14:38 -0700 Subject: [PATCH 16/37] Added nnf user container example --- test/nnf/hetchy.container/Dockerfile | 20 +++++ .../hetchy.container/allocation-computes.yaml | 3 + .../hetchy.container/allocation-servers.yaml | 12 +++ .../nnf-container-example.yaml | 62 ++++++++++++++ .../hetchy.container/run_nnf_user_container | 65 +++++++++++++++ test/nnf/hetchy.container/src/Makefile | 10 +++ .../hetchy.container/src/mpi_hello_world.c | 81 +++++++++++++++++++ 7 files changed, 253 insertions(+) create mode 100644 test/nnf/hetchy.container/Dockerfile create mode 100644 test/nnf/hetchy.container/allocation-computes.yaml create mode 100644 test/nnf/hetchy.container/allocation-servers.yaml create mode 100644 test/nnf/hetchy.container/nnf-container-example.yaml create mode 100755 test/nnf/hetchy.container/run_nnf_user_container create mode 100644 test/nnf/hetchy.container/src/Makefile create mode 100644 test/nnf/hetchy.container/src/mpi_hello_world.c diff --git a/test/nnf/hetchy.container/Dockerfile b/test/nnf/hetchy.container/Dockerfile new file mode 100644 index 0000000..f36829a --- /dev/null +++ b/test/nnf/hetchy.container/Dockerfile @@ -0,0 +1,20 @@ +# Start from the NearNodeFlash MPI File Utils Image that is used for Data Movement. This is an easy +# way to get MPI File Utils/mpi-operator as a base. Use a stage for building the application. +# Alernatively, use mpi-operator's image directly: +# FROM mpioperator/openmpi:0.3.0 AS build +FROM ghcr.io/nearnodeflash/nnf-mfu:master AS build + +# Install build tools +RUN apt update +RUN apt install -y make libopenmpi-dev + +# Build application +WORKDIR /src +COPY src . +RUN make + +# Final stage - start fresh and don't carry over the build artifacts +FROM ghcr.io/nearnodeflash/nnf-mfu:master + +# Copy application from build stage into final stage +COPY --from=build /src/mpi_hello_world /usr/bin/mpi_hello_world diff --git a/test/nnf/hetchy.container/allocation-computes.yaml b/test/nnf/hetchy.container/allocation-computes.yaml new file mode 100644 index 0000000..7af123f --- /dev/null +++ b/test/nnf/hetchy.container/allocation-computes.yaml @@ -0,0 +1,3 @@ +data: + - name: "hetchy28" + - name: "hetchy29" diff --git a/test/nnf/hetchy.container/allocation-servers.yaml b/test/nnf/hetchy.container/allocation-servers.yaml new file mode 100644 index 0000000..7118d89 --- /dev/null +++ b/test/nnf/hetchy.container/allocation-servers.yaml @@ -0,0 +1,12 @@ +spec: + allocationSets: + - allocationSize: 50000000000 + label: ost + storage: + - allocationCount: 1 + name: hetchy44 + - allocationSize: 50000000000 + label: mdt + storage: + - allocationCount: 1 + name: hetchy44 diff --git a/test/nnf/hetchy.container/nnf-container-example.yaml b/test/nnf/hetchy.container/nnf-container-example.yaml new file mode 100644 index 0000000..113a2f5 --- /dev/null +++ b/test/nnf/hetchy.container/nnf-container-example.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: demo + namespace: nnf-system +data: + storages: + - name: DW_JOB_my_storage + optional: false + mpiSpec: + mpiReplicaSpecs: + Launcher: + template: + spec: + containers: + - name: nnf-container-example + image: ghcr.io/nearnodeflash/nnf-container-example:latest + command: + - /bin/bash + - -c + - echo "Hello, my DW JOB is $(DW_JOB_my_storage)" + # - "sleep 500" + # - mpirun + # - hostname + # - mpi_hello_world + # - "$(DW_JOB_my_storage)" + Worker: + template: + spec: + containers: + - name: nnf-container-example + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + - SYS_CHROOT + - AUDIT_WRITE + # command: + # - /usr/sbin/sshd + # - -De + # - -p + # - "2222" + image: ghcr.io/nearnodeflash/nnf-container-example:latest + +--- +apiVersion: dws.cray.hpe.com/v1alpha1 +kind: Workflow +metadata: + name: demo-container + namespace: default +spec: + desiredState: "Proposal" + dwDirectives: + - "#DW jobdw name=demo-lustre type=lustre capacity=50GB" + - "#DW container name=demo-container profile=demo \ + DW_JOB_my_storage=demo-lustre" + wlmID: "5f239bd8-30db-450b-8c2c-a1a7c8631a1a" + jobID: 900001 + userID: 1050 + groupID: 1051 diff --git a/test/nnf/hetchy.container/run_nnf_user_container b/test/nnf/hetchy.container/run_nnf_user_container new file mode 100755 index 0000000..459ef03 --- /dev/null +++ b/test/nnf/hetchy.container/run_nnf_user_container @@ -0,0 +1,65 @@ +#!/bin/bash + +if [ ! -f nnf-container-example.yaml ]; then + echo "Cannot find container example file" + exit 1 +fi + +run_cmd() { + if ! eval $1 > /dev/null 2>&1 ; then + echo "FAILED" + exit 1 + fi +} + +wait_for_good_state() { + seconds=0 + pattern=$1 + + until kubectl get workflows | grep demo-container | grep -q "$pattern" + do + sleep 1 + ((seconds++)) + done + echo ${seconds} +} + +get_logs() { + pod=$1 + seconds=0 + + printf "[Getting Logs-------------->]\r" + until kubectl logs --tail=100 ${pod} |& wc -l > 0 + do + sleep 1 + ((seconds++)) + done + + printf "[Logs obtained in %02d seconds]\n" $seconds + echo "----------------------" + kubectl logs --tail=100 ${pod} + echo "----------------------" +} + +run_and_wait() { + printf "[Running-->] - %s\r" "$1" + run_cmd "$1" + t=$( wait_for_good_state "$2" ) + printf "[%02d seconds] - %s\n" $t "$1" +} + +run_and_wait "kubectl apply -f nnf-container-example.yaml" "true Completed" + +# +# The following two patch commands finish synchronously and we do not need to wait for them to complete +# +run_cmd "kubectl patch --type merge --patch-file=allocation-computes.yaml computes demo-container" +run_cmd "kubectl patch --type merge --patch-file=allocation-servers.yaml servers demo-container-0" + +run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"Setup\"}}'" "Setup true Completed" +run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"DataIn\"}}'" "DataIn true Completed" +run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"PreRun\"}}'" "PreRun true Completed" +pod="$( kubectl get pods | grep demo-container-launcher | awk '{print $1}' )" +get_logs "$( kubectl get pods | grep demo-container-launcher | awk '{print $1}' )" +run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"Teardown\"}}'" "Teardown true Completed" +run_cmd "kubectl delete -f nnf-container-example.yaml" diff --git a/test/nnf/hetchy.container/src/Makefile b/test/nnf/hetchy.container/src/Makefile new file mode 100644 index 0000000..d2240ce --- /dev/null +++ b/test/nnf/hetchy.container/src/Makefile @@ -0,0 +1,10 @@ +EXECS=mpi_hello_world +MPICC?=mpicc + +all: ${EXECS} + +mpi_hello_world: mpi_hello_world.c + ${MPICC} -o mpi_hello_world mpi_hello_world.c + +clean: + rm -f ${EXECS} \ No newline at end of file diff --git a/test/nnf/hetchy.container/src/mpi_hello_world.c b/test/nnf/hetchy.container/src/mpi_hello_world.c new file mode 100644 index 0000000..6eb0398 --- /dev/null +++ b/test/nnf/hetchy.container/src/mpi_hello_world.c @@ -0,0 +1,81 @@ +// Author: Wes Kendall +// Copyright 2011 www.mpitutorial.com +// This code is provided freely with the tutorials on mpitutorial.com. Feel +// free to modify it for your own use. Any distribution of the code must +// either provide a link to www.mpitutorial.com or keep this header intact. +// +// An intro MPI hello world program that uses MPI_Init, MPI_Comm_size, +// MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name. +// +#include +#include +#include +#include + +#include + +#include + +int main(int argc, char **argv) +{ + char nnf_storage_path[PATH_MAX]; + + // Initialize the MPI environment. The two arguments to MPI Init are not + // currently used by MPI implementations, but are there in case future + // implementations might need the arguments. + MPI_Init(NULL, NULL); + + // Get the number of processes + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + + // Get the rank of the process + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + // Get the name of the processor + char processor_name[MPI_MAX_PROCESSOR_NAME]; + int name_len; + MPI_Get_processor_name(processor_name, &name_len); + + if (argc < 2) + { + printf("Storage parameter not supplied\n"); + return -1; + } + strncpy(nnf_storage_path, argv[1], PATH_MAX); + + // Print off a hello world message + printf("Hello world from processor %s, rank %d out of %d processors. NNF Storage path: %s\n", + processor_name, world_rank, world_size, nnf_storage_path); + +#if 0 + // We're using a GFS2 filesystem, which has index mounts for every compute node + // e.g. /mnt/nnf/5d335081-cd0f-4b8a-a1f4-94860a8ae702-0/0/ + // So use the rank to index into the NNF storage path + if (sprintf(nnf_storage_path, "%s/%d/testfile", nnf_storage_path, world_rank) == -1) + { + fprintf(stderr, "rank %d: %s\n", world_rank, strerror(errno)); + return errno; + } + + int fd = open(nnf_storage_path, O_WRONLY | O_CREAT); + if (fd == -1) + { + fprintf(stderr, "rank %d: %s\n", world_rank, strerror(errno)); + return errno; + } + + int res = posix_fallocate(fd, 0, 100); + if (res == -1) + { + fprintf(stderr, "rank %d: %s\n", world_rank, strerror(errno)); + return errno; + } + + printf("rank %d: wrote file to '%s'\n", world_rank, nnf_storage_path); +#endif // 0 + + // Finalize the MPI environment. No more MPI calls can be made after this + MPI_Finalize(); +} From 45a0b4a18c08f3753644fc4a0fbad7d4872a37f9 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 27 Jun 2023 16:05:23 -0700 Subject: [PATCH 17/37] Updates --- src/nnfdm.cpp | 73 ++++++++++--------- .../nnf-container-example.yaml | 12 +-- .../hetchy.container/run_nnf_user_container | 1 + 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 6fc1cc0..4155de9 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -22,29 +22,40 @@ namespace { { std::stringstream ss; - ss << "{" << status_response.state() << "/"; + ss << "Offload Command Status: " << "{" << status_response.state() << "/"; switch (status_response.state()) { - case nnfdm::StatusResponse::State::STATE_PENDING: ss << "STATE_PENDING}"; break; - case nnfdm::StatusResponse::State::STATE_STARTING: ss << "STATE_STARTING}"; break; - case nnfdm::StatusResponse::State::STATE_RUNNING: ss << "STATE_RUNNING}"; break; - case nnfdm::StatusResponse::State::STATE_COMPLETED: ss <<"STATE_COMPLETED}"; break; - case nnfdm::StatusResponse::State::STATE_CANCELLING: ss <<"STATE_CANCELLING}"; break; - case nnfdm::StatusResponse::State::STATE_UNKNOWN: ss << "STATE_UNKNOWN}"; break; - default: ss << "STATE_???}"; break; + case nnfdm::StatusResponse::State::STATE_PENDING: ss << "STATE_PENDING"; break; + case nnfdm::StatusResponse::State::STATE_STARTING: ss << "STATE_STARTING"; break; + case nnfdm::StatusResponse::State::STATE_RUNNING: ss << "STATE_RUNNING"; break; + case nnfdm::StatusResponse::State::STATE_COMPLETED: ss <<"STATE_COMPLETED"; break; + case nnfdm::StatusResponse::State::STATE_CANCELLING: ss <<"STATE_CANCELLING"; break; + case nnfdm::StatusResponse::State::STATE_UNKNOWN: ss << "STATE_UNKNOWN"; break; + default: ss << "STATE_???"; break; } + ss << "}, "; - ss << " {" << status_response.status() << "/"; + ss << "{" << status_response.status() << "/"; switch (status_response.status()) { - case nnfdm::StatusResponse::Status::STATUS_INVALID: ss << "STATUS_INVALID}"; break; - case nnfdm::StatusResponse::Status::STATUS_NOT_FOUND: ss << "STATUS_NOT_FOUND}"; break; - case nnfdm::StatusResponse::Status::STATUS_SUCCESS: ss << "STATUS_SUCCESS}"; break; - case nnfdm::StatusResponse::Status::STATUS_FAILED: ss << "STATUS_FAILED}"; break; - case nnfdm::StatusResponse::Status::STATUS_CANCELLED: ss << "STATUS_CANCELLED}"; break; - case nnfdm::StatusResponse::Status::STATUS_UNKNOWN: ss << "STATUS_UNKNOWN}"; break; - default: ss << "STATE_???}"; break; + case nnfdm::StatusResponse::Status::STATUS_INVALID: ss << "STATUS_INVALID"; break; + case nnfdm::StatusResponse::Status::STATUS_NOT_FOUND: ss << "STATUS_NOT_FOUND"; break; + case nnfdm::StatusResponse::Status::STATUS_SUCCESS: ss << "STATUS_SUCCESS"; break; + case nnfdm::StatusResponse::Status::STATUS_FAILED: ss << "STATUS_FAILED"; break; + case nnfdm::StatusResponse::Status::STATUS_CANCELLED: ss << "STATUS_CANCELLED"; break; + case nnfdm::StatusResponse::Status::STATUS_UNKNOWN: ss << "STATUS_UNKNOWN"; break; + default: ss << "STATE_???"; break; } - - ss << std::endl << " " << status_response.message(); + ss << "}" << std::endl; + + nnfdm::CommandStatus cmd = status_response.commandStatus(); + ss << " Offload Command Status:" << std::endl; + ss << " Command: " << cmd.command << std::endl; + ss << " Progress: " << cmd.progress << "%" << std::endl; + ss << " ElapsedTime: " << cmd.elapsedTime << std::endl; + ss << " LastMessage: " << cmd.lastMessage << std::endl; + ss << " LastMessageTime: " << cmd.lastMessageTime << std::endl; + ss << " Offload StartTime: " << status_response.startTime() << std::endl; + ss << " Offload EndTime: " << status_response.endTime() << std::endl; + ss << " Offload Message: " << status_response.message() << std::endl; return ss.str(); } @@ -65,34 +76,30 @@ namespace { /*NOTREACHED*/ } - AXL_DBG(1, "Status of offload(filename=%s, uid=%s): %s", fname, uid, print_status(status_response).c_str()); switch (status_response.state()) { case nnfdm::StatusResponse::State::STATE_PENDING: - rval = AXL_STATUS_INPROG; - break; case nnfdm::StatusResponse::State::STATE_STARTING: - rval = AXL_STATUS_INPROG; - break; case nnfdm::StatusResponse::State::STATE_RUNNING: rval = AXL_STATUS_INPROG; break; case nnfdm::StatusResponse::State::STATE_COMPLETED: if (status_response.status() == nnfdm::StatusResponse::Status::STATUS_SUCCESS) { + AXL_DBG(1, "Status of offload(filename=%s, uid=%s): %s", fname, uid, print_status(status_response).c_str()); rval = AXL_STATUS_DEST; } else { rval = AXL_STATUS_ERROR; - axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d %s" + axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d\n%s" , status_response.status() - , status_response.message().c_str()); + , print_status(status_response).c_str()); } break; default: axl_abort( -1 - , "NNFDM Offload State STATE UNKNOWN: %d %s" + , "NNFDM Offload State STATE UNKNOWN: %d\n%s" , status_response.status() - , status_response.message().c_str()); + , print_status(status_response).c_str()); break; } @@ -164,12 +171,12 @@ int nnfdm_start(int id) char* dst_filename; kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); - nnfdm::CreateRequest create_request( std::string{src_filename} // Source - , std::string{dst_filename} // Destination - , false // ?? - , "" // ?? - , false // ?? - , false // ?? + nnfdm::CreateRequest create_request( std::string{src_filename} // Source file or directory + , std::string{dst_filename} // Destination file or directory + , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement + , "" // Extra options to pass to `dcp` if present in the Data Movement command. + , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. + , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. ); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow diff --git a/test/nnf/hetchy.container/nnf-container-example.yaml b/test/nnf/hetchy.container/nnf-container-example.yaml index 113a2f5..b5e2bf7 100644 --- a/test/nnf/hetchy.container/nnf-container-example.yaml +++ b/test/nnf/hetchy.container/nnf-container-example.yaml @@ -19,12 +19,7 @@ data: command: - /bin/bash - -c - - echo "Hello, my DW JOB is $(DW_JOB_my_storage)" - # - "sleep 500" - # - mpirun - # - hostname - # - mpi_hello_world - # - "$(DW_JOB_my_storage)" + - env && df -h Worker: template: spec: @@ -37,11 +32,6 @@ data: - NET_BIND_SERVICE - SYS_CHROOT - AUDIT_WRITE - # command: - # - /usr/sbin/sshd - # - -De - # - -p - # - "2222" image: ghcr.io/nearnodeflash/nnf-container-example:latest --- diff --git a/test/nnf/hetchy.container/run_nnf_user_container b/test/nnf/hetchy.container/run_nnf_user_container index 459ef03..1a946c5 100755 --- a/test/nnf/hetchy.container/run_nnf_user_container +++ b/test/nnf/hetchy.container/run_nnf_user_container @@ -7,6 +7,7 @@ fi run_cmd() { if ! eval $1 > /dev/null 2>&1 ; then + eval $1 echo "FAILED" exit 1 fi From fdb967d95db6a74bdcb458d833db8e1b8c05a55f Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 09:53:35 -0700 Subject: [PATCH 18/37] Working set of hpe containers for rabbits --- .../Dockerfile | 0 .../allocation-computes.yaml | 0 .../allocation-servers.yaml | 0 .../nnf-container-example.yaml | 0 .../run_nnf_user_container | 23 +++++++++++-------- .../src/Makefile | 0 .../src/mpi_hello_world.c | 0 7 files changed, 13 insertions(+), 10 deletions(-) rename test/nnf/{hetchy.container => hpe_demo.container}/Dockerfile (100%) rename test/nnf/{hetchy.container => hpe_demo.container}/allocation-computes.yaml (100%) rename test/nnf/{hetchy.container => hpe_demo.container}/allocation-servers.yaml (100%) rename test/nnf/{hetchy.container => hpe_demo.container}/nnf-container-example.yaml (100%) rename test/nnf/{hetchy.container => hpe_demo.container}/run_nnf_user_container (52%) rename test/nnf/{hetchy.container => hpe_demo.container}/src/Makefile (100%) rename test/nnf/{hetchy.container => hpe_demo.container}/src/mpi_hello_world.c (100%) diff --git a/test/nnf/hetchy.container/Dockerfile b/test/nnf/hpe_demo.container/Dockerfile similarity index 100% rename from test/nnf/hetchy.container/Dockerfile rename to test/nnf/hpe_demo.container/Dockerfile diff --git a/test/nnf/hetchy.container/allocation-computes.yaml b/test/nnf/hpe_demo.container/allocation-computes.yaml similarity index 100% rename from test/nnf/hetchy.container/allocation-computes.yaml rename to test/nnf/hpe_demo.container/allocation-computes.yaml diff --git a/test/nnf/hetchy.container/allocation-servers.yaml b/test/nnf/hpe_demo.container/allocation-servers.yaml similarity index 100% rename from test/nnf/hetchy.container/allocation-servers.yaml rename to test/nnf/hpe_demo.container/allocation-servers.yaml diff --git a/test/nnf/hetchy.container/nnf-container-example.yaml b/test/nnf/hpe_demo.container/nnf-container-example.yaml similarity index 100% rename from test/nnf/hetchy.container/nnf-container-example.yaml rename to test/nnf/hpe_demo.container/nnf-container-example.yaml diff --git a/test/nnf/hetchy.container/run_nnf_user_container b/test/nnf/hpe_demo.container/run_nnf_user_container similarity index 52% rename from test/nnf/hetchy.container/run_nnf_user_container rename to test/nnf/hpe_demo.container/run_nnf_user_container index 1a946c5..37b76ad 100755 --- a/test/nnf/hetchy.container/run_nnf_user_container +++ b/test/nnf/hpe_demo.container/run_nnf_user_container @@ -1,5 +1,7 @@ #!/bin/bash +container_name="demo-container" + if [ ! -f nnf-container-example.yaml ]; then echo "Cannot find container example file" exit 1 @@ -17,7 +19,7 @@ wait_for_good_state() { seconds=0 pattern=$1 - until kubectl get workflows | grep demo-container | grep -q "$pattern" + until kubectl get workflows | grep ${container_name} | grep -q "$pattern" do sleep 1 ((seconds++)) @@ -30,7 +32,8 @@ get_logs() { seconds=0 printf "[Getting Logs-------------->]\r" - until kubectl logs --tail=100 ${pod} |& wc -l > 0 + # until [ $( get_output_lines ${pod} ) -gt 0 ] + until [ $( kubectl logs --tail=100 ${pod} |& wc -l ) -gt 0 ] do sleep 1 ((seconds++)) @@ -54,13 +57,13 @@ run_and_wait "kubectl apply -f nnf-container-example.yaml" "true Completed" # # The following two patch commands finish synchronously and we do not need to wait for them to complete # -run_cmd "kubectl patch --type merge --patch-file=allocation-computes.yaml computes demo-container" -run_cmd "kubectl patch --type merge --patch-file=allocation-servers.yaml servers demo-container-0" +run_cmd "kubectl patch --type merge --patch-file=allocation-computes.yaml computes ${container_name}" +run_cmd "kubectl patch --type merge --patch-file=allocation-servers.yaml servers ${container_name}-0" -run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"Setup\"}}'" "Setup true Completed" -run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"DataIn\"}}'" "DataIn true Completed" -run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"PreRun\"}}'" "PreRun true Completed" -pod="$( kubectl get pods | grep demo-container-launcher | awk '{print $1}' )" -get_logs "$( kubectl get pods | grep demo-container-launcher | awk '{print $1}' )" -run_and_wait "kubectl patch --type merge workflow demo-container --patch '{\"spec\": {\"desiredState\": \"Teardown\"}}'" "Teardown true Completed" +run_and_wait "kubectl patch --type merge workflow ${container_name} --patch '{\"spec\": {\"desiredState\": \"Setup\"}}'" "Setup true Completed" +run_and_wait "kubectl patch --type merge workflow ${container_name} --patch '{\"spec\": {\"desiredState\": \"DataIn\"}}'" "DataIn true Completed" +run_and_wait "kubectl patch --type merge workflow ${container_name} --patch '{\"spec\": {\"desiredState\": \"PreRun\"}}'" "PreRun true Completed" +pod="$( kubectl get pods | grep ${container_name}-launcher | awk '{print $1}' )" +get_logs "$( kubectl get pods | grep ${container_name}-launcher | awk '{print $1}' )" +run_and_wait "kubectl patch --type merge workflow ${container_name} --patch '{\"spec\": {\"desiredState\": \"Teardown\"}}'" "Teardown true Completed" run_cmd "kubectl delete -f nnf-container-example.yaml" diff --git a/test/nnf/hetchy.container/src/Makefile b/test/nnf/hpe_demo.container/src/Makefile similarity index 100% rename from test/nnf/hetchy.container/src/Makefile rename to test/nnf/hpe_demo.container/src/Makefile diff --git a/test/nnf/hetchy.container/src/mpi_hello_world.c b/test/nnf/hpe_demo.container/src/mpi_hello_world.c similarity index 100% rename from test/nnf/hetchy.container/src/mpi_hello_world.c rename to test/nnf/hpe_demo.container/src/mpi_hello_world.c From 6d2b38b5505d22456c8660650320a16daec1c72c Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 10:49:46 -0700 Subject: [PATCH 19/37] Added documentation for building Dockerfile --- test/nnf/axl.containers/Dockerfile | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 test/nnf/axl.containers/Dockerfile diff --git a/test/nnf/axl.containers/Dockerfile b/test/nnf/axl.containers/Dockerfile new file mode 100644 index 0000000..4674aff --- /dev/null +++ b/test/nnf/axl.containers/Dockerfile @@ -0,0 +1,38 @@ +# +# This image contains the NNF MPI File Utils Image that is also used for Data Movement. +# On top of that, we place a simple mpi_hello_world program that will take two arguments +# as input: +# 1) The name of the rabbit-local directory ($DW_JOB_storage) +# 2) The name of the global/shared Lustre directory +# +# This image is loaded +# +# IMPORTANT IMAGE BUILD NOTES: +# +# This image must be built on an x86 LC machine that is configure with SLURM. This image will fail +# to build on flux machines (toss4). +# +# Steps to successful image build: +# ssh oslic +# podman login -u docker.io//axl +# salloc --userns --exclusive +# podman build -t docker.io/ Date: Mon, 10 Jul 2023 10:51:08 -0700 Subject: [PATCH 20/37] Initial container created --- .../axl.containers/axl-container-marty.yaml | 43 ++++++++++++++ test/nnf/axl.containers/src/Makefile | 10 ++++ test/nnf/axl.containers/src/mpi_hello_world.c | 56 +++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 test/nnf/axl.containers/axl-container-marty.yaml create mode 100644 test/nnf/axl.containers/src/Makefile create mode 100644 test/nnf/axl.containers/src/mpi_hello_world.c diff --git a/test/nnf/axl.containers/axl-container-marty.yaml b/test/nnf/axl.containers/axl-container-marty.yaml new file mode 100644 index 0000000..5005ab0 --- /dev/null +++ b/test/nnf/axl.containers/axl-container-marty.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: axl-container-marty + namespace: nnf-system +data: + storages: + - name: DW_JOB_storage + optional: false + - name: DW_PERSISTENT_storage + optional: true + mpiSpec: + mpiReplicaSpecs: + Launcher: + template: + spec: + containers: + - name: axl-container-marty + image: docker.io/mcfadden8/axl:latest + command: ["/bin/bash"] + args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymfc"] + volumeMounts: + - mountPath: /p/lslide + name: lslide + volumes: + - name: lslide + persistentVolumeClaim: + claimName: kern-default-readwritemany-pvc + Worker: + template: + spec: + containers: + - name: axl-container-marty + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + - SYS_CHROOT + - AUDIT_WRITE + image: docker.io/mcfadden8/axl:latest + diff --git a/test/nnf/axl.containers/src/Makefile b/test/nnf/axl.containers/src/Makefile new file mode 100644 index 0000000..d2240ce --- /dev/null +++ b/test/nnf/axl.containers/src/Makefile @@ -0,0 +1,10 @@ +EXECS=mpi_hello_world +MPICC?=mpicc + +all: ${EXECS} + +mpi_hello_world: mpi_hello_world.c + ${MPICC} -o mpi_hello_world mpi_hello_world.c + +clean: + rm -f ${EXECS} \ No newline at end of file diff --git a/test/nnf/axl.containers/src/mpi_hello_world.c b/test/nnf/axl.containers/src/mpi_hello_world.c new file mode 100644 index 0000000..ba16a1f --- /dev/null +++ b/test/nnf/axl.containers/src/mpi_hello_world.c @@ -0,0 +1,56 @@ +// Author: Wes Kendall +// Copyright 2011 www.mpitutorial.com +// This code is provided freely with the tutorials on mpitutorial.com. Feel +// free to modify it for your own use. Any distribution of the code must +// either provide a link to www.mpitutorial.com or keep this header intact. +// +// An intro MPI hello world program that uses MPI_Init, MPI_Comm_size, +// MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name. +// +#include +#include +#include +#include + +#include + +#include + +int main(int argc, char **argv) +{ + char nnf_storage_path[PATH_MAX]; + char global_storage_path[PATH_MAX]; + + // Initialize the MPI environment. The two arguments to MPI Init are not + // currently used by MPI implementations, but are there in case future + // implementations might need the arguments. + MPI_Init(NULL, NULL); + + // Get the number of processes + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + + // Get the rank of the process + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + // Get the name of the processor + char processor_name[MPI_MAX_PROCESSOR_NAME]; + int name_len; + MPI_Get_processor_name(processor_name, &name_len); + + if (argc < 3) + { + printf("Usage: %s nnf_storage_dir global_storage_dir\n", argv[0]); + return -1; + } + strncpy(nnf_storage_path, argv[1], PATH_MAX); + strncpy(global_storage_path, argv[2], PATH_MAX); + + // Print off a hello world message + printf("Hello processor %s, rank %d out of %d processors. NNF Storage path: %s, Global Storage path is: %s\n", + processor_name, world_rank, world_size, nnf_storage_path, global_storage_path); + + // Finalize the MPI environment. No more MPI calls can be made after this + MPI_Finalize(); +} From 638f969a8973f8dc6417d66f5f3d728279100c97 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 11:12:43 -0700 Subject: [PATCH 21/37] Updates --- ...ntainer-marty.yaml => axl-container-martymcf.yaml} | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename test/nnf/axl.containers/{axl-container-marty.yaml => axl-container-martymcf.yaml} (87%) diff --git a/test/nnf/axl.containers/axl-container-marty.yaml b/test/nnf/axl.containers/axl-container-martymcf.yaml similarity index 87% rename from test/nnf/axl.containers/axl-container-marty.yaml rename to test/nnf/axl.containers/axl-container-martymcf.yaml index 5005ab0..cda29f9 100644 --- a/test/nnf/axl.containers/axl-container-marty.yaml +++ b/test/nnf/axl.containers/axl-container-martymcf.yaml @@ -1,8 +1,11 @@ +# +# +# --- apiVersion: nnf.cray.hpe.com/v1alpha1 kind: NnfContainerProfile metadata: - name: axl-container-marty + name: axl-container-martymcf namespace: nnf-system data: storages: @@ -16,10 +19,10 @@ data: template: spec: containers: - - name: axl-container-marty + - name: axl-container-martymcf image: docker.io/mcfadden8/axl:latest command: ["/bin/bash"] - args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymfc"] + args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf"] volumeMounts: - mountPath: /p/lslide name: lslide @@ -31,7 +34,7 @@ data: template: spec: containers: - - name: axl-container-marty + - name: axl-container-martymcf securityContext: allowPrivilegeEscalation: true capabilities: From d911cf6371e3cd5731275e601b16ad5ba2916477 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 11:19:06 -0700 Subject: [PATCH 22/37] Added container for Hari --- .../axl.containers/axl-container-haridev.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/nnf/axl.containers/axl-container-haridev.yaml diff --git a/test/nnf/axl.containers/axl-container-haridev.yaml b/test/nnf/axl.containers/axl-container-haridev.yaml new file mode 100644 index 0000000..5b9d9a3 --- /dev/null +++ b/test/nnf/axl.containers/axl-container-haridev.yaml @@ -0,0 +1,46 @@ +# +# +# +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: axl-container-haridev + namespace: nnf-system +data: + storages: + - name: DW_JOB_storage + optional: false + - name: DW_PERSISTENT_storage + optional: true + mpiSpec: + mpiReplicaSpecs: + Launcher: + template: + spec: + containers: + - name: axl-container-haridev + image: docker.io/mcfadden8/axl:latest + command: ["/bin/bash"] + args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/haridev"] + volumeMounts: + - mountPath: /p/lslide + name: lslide + volumes: + - name: lslide + persistentVolumeClaim: + claimName: kern-default-readwritemany-pvc + Worker: + template: + spec: + containers: + - name: axl-container-haridev + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + - SYS_CHROOT + - AUDIT_WRITE + image: docker.io/mcfadden8/axl:latest + From 5755781964672d08a565e4f543390d21228ecf7a Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 11:41:37 -0700 Subject: [PATCH 23/37] Added container for Cameron --- .../axl-container-stanavig.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/nnf/axl.containers/axl-container-stanavig.yaml diff --git a/test/nnf/axl.containers/axl-container-stanavig.yaml b/test/nnf/axl.containers/axl-container-stanavig.yaml new file mode 100644 index 0000000..a17d538 --- /dev/null +++ b/test/nnf/axl.containers/axl-container-stanavig.yaml @@ -0,0 +1,46 @@ +# +# +# +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: axl-container-stanavig + namespace: nnf-system +data: + storages: + - name: DW_JOB_storage + optional: false + - name: DW_PERSISTENT_storage + optional: true + mpiSpec: + mpiReplicaSpecs: + Launcher: + template: + spec: + containers: + - name: axl-container-stanavig + image: docker.io/mcfadden8/axl:latest + command: ["/bin/bash"] + args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/stanavig"] + volumeMounts: + - mountPath: /p/lslide + name: lslide + volumes: + - name: lslide + persistentVolumeClaim: + claimName: kern-default-readwritemany-pvc + Worker: + template: + spec: + containers: + - name: axl-container-stanavig + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + - SYS_CHROOT + - AUDIT_WRITE + image: docker.io/mcfadden8/axl:latest + From e1552c6d3a6a1d7ca44192ef663c098e44703465 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 10 Jul 2023 12:38:01 -0700 Subject: [PATCH 24/37] Updates --- .../axl.containers/axl-container-haridev.yaml | 17 +++++++++++++++++ .../axl.containers/axl-container-martymcf.yaml | 17 +++++++++++++++++ .../axl.containers/axl-container-stanavig.yaml | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/test/nnf/axl.containers/axl-container-haridev.yaml b/test/nnf/axl.containers/axl-container-haridev.yaml index 5b9d9a3..f12ac4e 100644 --- a/test/nnf/axl.containers/axl-container-haridev.yaml +++ b/test/nnf/axl.containers/axl-container-haridev.yaml @@ -1,6 +1,23 @@ +# NNF Container Profile Management # +# When you get ready to pull your (most interesting) container image from your +# own container registry, you will need to update, and re-deploy you NNF +# container profile # +# When you are ready to do this, change the two lines below that contain: +# `image: docker.io/mcfadden8/axl:latest` to point to your image. # +# NOTE: feel free to ask me if you need to have your nnf container profile updated +# and redeployed. I'm happy to help! There do be some dragons +# +# To see the nnf container profiles already deployed, run: +# kubectl get nnfcontainerprofiles -A +# +# To remove your deployed nnf container profile so you can deploy a new one, run: +# kubectl delete nnfcontainerprofiles -n nnf-system axl-container-$( whoami ) +# +# To deploy a new nnf container profile run: +# kubectl apply -f axl-container-$( whoami ).yaml --- apiVersion: nnf.cray.hpe.com/v1alpha1 kind: NnfContainerProfile diff --git a/test/nnf/axl.containers/axl-container-martymcf.yaml b/test/nnf/axl.containers/axl-container-martymcf.yaml index cda29f9..aa8e100 100644 --- a/test/nnf/axl.containers/axl-container-martymcf.yaml +++ b/test/nnf/axl.containers/axl-container-martymcf.yaml @@ -1,6 +1,23 @@ +# NNF Container Profile Management # +# When you get ready to pull your (most interesting) container image from your +# own container registry, you will need to update, and re-deploy you NNF +# container profile # +# When you are ready to do this, change the two lines below that contain: +# `image: docker.io/mcfadden8/axl:latest` to point to your image. # +# NOTE: feel free to ask me if you need to have your nnf container profile updated +# and redeployed. I'm happy to help! There do be some dragons +# +# To see the nnf container profiles already deployed, run: +# kubectl get nnfcontainerprofiles -A +# +# To remove your deployed nnf container profile so you can deploy a new one, run: +# kubectl delete nnfcontainerprofiles -n nnf-system axl-container-$( whoami ) +# +# To deploy a new nnf container profile run: +# kubectl apply -f axl-container-$( whoami ).yaml --- apiVersion: nnf.cray.hpe.com/v1alpha1 kind: NnfContainerProfile diff --git a/test/nnf/axl.containers/axl-container-stanavig.yaml b/test/nnf/axl.containers/axl-container-stanavig.yaml index a17d538..906d0f5 100644 --- a/test/nnf/axl.containers/axl-container-stanavig.yaml +++ b/test/nnf/axl.containers/axl-container-stanavig.yaml @@ -1,6 +1,23 @@ +# NNF Container Profile Management # +# When you get ready to pull your (most interesting) container image from your +# own container registry, you will need to update, and re-deploy you NNF +# container profile # +# When you are ready to do this, change the two lines below that contain: +# `image: docker.io/mcfadden8/axl:latest` to point to your image. # +# NOTE: feel free to ask me if you need to have your nnf container profile updated +# and redeployed. I'm happy to help! There do be some dragons +# +# To see the nnf container profiles already deployed, run: +# kubectl get nnfcontainerprofiles -A +# +# To remove your deployed nnf container profile so you can deploy a new one, run: +# kubectl delete nnfcontainerprofiles -n nnf-system axl-container-$( whoami ) +# +# To deploy a new nnf container profile run: +# kubectl apply -f axl-container-$( whoami ).yaml --- apiVersion: nnf.cray.hpe.com/v1alpha1 kind: NnfContainerProfile From a1064ce44e364f26561b9045a07317251ae50884 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 3 Aug 2023 16:51:26 -0700 Subject: [PATCH 25/37] Updates --- .../axl-container-martymcf.yaml | 20 +++++++++++-------- test/nnf/axl.containers/src/mpi_hello_world.c | 13 ++++++++---- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/test/nnf/axl.containers/axl-container-martymcf.yaml b/test/nnf/axl.containers/axl-container-martymcf.yaml index aa8e100..f5a484d 100644 --- a/test/nnf/axl.containers/axl-container-martymcf.yaml +++ b/test/nnf/axl.containers/axl-container-martymcf.yaml @@ -35,21 +35,25 @@ data: Launcher: template: spec: + imagePullSecrets: + - name: readonly-red-rock-slushy containers: - name: axl-container-martymcf - image: docker.io/mcfadden8/axl:latest + image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty command: ["/bin/bash"] args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf"] - volumeMounts: - - mountPath: /p/lslide - name: lslide - volumes: - - name: lslide - persistentVolumeClaim: - claimName: kern-default-readwritemany-pvc + # volumeMounts: + # - mountPath: /p/lslide + # name: lslide + # volumes: + # - name: lslide + # persistentVolumeClaim: + # claimName: kern-default-readwritemany-pvc Worker: template: spec: + imagePullSecrets: + - name: readonly-red-rock-slushy containers: - name: axl-container-martymcf securityContext: diff --git a/test/nnf/axl.containers/src/mpi_hello_world.c b/test/nnf/axl.containers/src/mpi_hello_world.c index ba16a1f..bf95499 100644 --- a/test/nnf/axl.containers/src/mpi_hello_world.c +++ b/test/nnf/axl.containers/src/mpi_hello_world.c @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -20,6 +21,7 @@ int main(int argc, char **argv) { char nnf_storage_path[PATH_MAX]; char global_storage_path[PATH_MAX]; + char host_name[PATH_MAX]; // Initialize the MPI environment. The two arguments to MPI Init are not // currently used by MPI implementations, but are there in case future @@ -39,17 +41,20 @@ int main(int argc, char **argv) int name_len; MPI_Get_processor_name(processor_name, &name_len); - if (argc < 3) - { + if (argc < 3) { printf("Usage: %s nnf_storage_dir global_storage_dir\n", argv[0]); return -1; } strncpy(nnf_storage_path, argv[1], PATH_MAX); strncpy(global_storage_path, argv[2], PATH_MAX); + if (gethostname(host_name, PATH_MAX) < 0) { + printf("%s: gethostname failed\n", argv[0]); + } + // Print off a hello world message - printf("Hello processor %s, rank %d out of %d processors. NNF Storage path: %s, Global Storage path is: %s\n", - processor_name, world_rank, world_size, nnf_storage_path, global_storage_path); + printf("%s: processor %s, rank %d out of %d processors. NNF Storage path: %s, Global Storage path is: %s\n", + host_name, processor_name, world_rank, world_size, nnf_storage_path, global_storage_path); // Finalize the MPI environment. No more MPI calls can be made after this MPI_Finalize(); From da4b28c1d79ba555826ac3bfef4f195b1b476472 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 4 Aug 2023 12:55:52 -0700 Subject: [PATCH 26/37] Another update --- test/nnf/axl.containers/axl-container-martymcf.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/nnf/axl.containers/axl-container-martymcf.yaml b/test/nnf/axl.containers/axl-container-martymcf.yaml index f5a484d..0196c29 100644 --- a/test/nnf/axl.containers/axl-container-martymcf.yaml +++ b/test/nnf/axl.containers/axl-container-martymcf.yaml @@ -63,5 +63,4 @@ data: - NET_BIND_SERVICE - SYS_CHROOT - AUDIT_WRITE - image: docker.io/mcfadden8/axl:latest - + image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty From 0f9b6604c7395334f0d0e71851b435c26dd1894c Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 4 Aug 2023 13:19:39 -0700 Subject: [PATCH 27/37] Separate nnf containers for hetchy and tioga --- ...artymcf.yaml => axl-container-hetchy.yaml} | 14 ++-- .../axl.containers/axl-container-tioga.yaml | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) rename test/nnf/axl.containers/{axl-container-martymcf.yaml => axl-container-hetchy.yaml} (88%) create mode 100644 test/nnf/axl.containers/axl-container-tioga.yaml diff --git a/test/nnf/axl.containers/axl-container-martymcf.yaml b/test/nnf/axl.containers/axl-container-hetchy.yaml similarity index 88% rename from test/nnf/axl.containers/axl-container-martymcf.yaml rename to test/nnf/axl.containers/axl-container-hetchy.yaml index 0196c29..3ce98c1 100644 --- a/test/nnf/axl.containers/axl-container-martymcf.yaml +++ b/test/nnf/axl.containers/axl-container-hetchy.yaml @@ -42,13 +42,13 @@ data: image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty command: ["/bin/bash"] args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf"] - # volumeMounts: - # - mountPath: /p/lslide - # name: lslide - # volumes: - # - name: lslide - # persistentVolumeClaim: - # claimName: kern-default-readwritemany-pvc + volumeMounts: + - mountPath: /p/lslide + name: lslide + volumes: + - name: lslide + persistentVolumeClaim: + claimName: kern-default-readwritemany-pvc Worker: template: spec: diff --git a/test/nnf/axl.containers/axl-container-tioga.yaml b/test/nnf/axl.containers/axl-container-tioga.yaml new file mode 100644 index 0000000..13519a1 --- /dev/null +++ b/test/nnf/axl.containers/axl-container-tioga.yaml @@ -0,0 +1,66 @@ +# NNF Container Profile Management +# +# When you get ready to pull your (most interesting) container image from your +# own container registry, you will need to update, and re-deploy you NNF +# container profile +# +# When you are ready to do this, change the two lines below that contain: +# `image: docker.io/mcfadden8/axl:latest` to point to your image. +# +# NOTE: feel free to ask me if you need to have your nnf container profile updated +# and redeployed. I'm happy to help! There do be some dragons +# +# To see the nnf container profiles already deployed, run: +# kubectl get nnfcontainerprofiles -A +# +# To remove your deployed nnf container profile so you can deploy a new one, run: +# kubectl delete nnfcontainerprofiles -n nnf-system axl-container-$( whoami ) +# +# To deploy a new nnf container profile run: +# kubectl apply -f axl-container-$( whoami ).yaml +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: axl-container-martymcf + namespace: nnf-system +data: + storages: + - name: DW_JOB_storage + optional: false + - name: DW_PERSISTENT_storage + optional: true + mpiSpec: + mpiReplicaSpecs: + Launcher: + template: + spec: + imagePullSecrets: + - name: readonly-red-rock-slushy + containers: + - name: axl-container-martymcf + image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty + command: ["/bin/bash"] + args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf"] + volumeMounts: + - mountPath: /p/lslide + name: lslide + volumes: + - name: lslide + persistentVolumeClaim: + claimName: lustre2-default-readwritemany-pvc + Worker: + template: + spec: + imagePullSecrets: + - name: readonly-red-rock-slushy + containers: + - name: axl-container-martymcf + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + - SYS_CHROOT + - AUDIT_WRITE + image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty From b040ff6752b2aedf24a7983d25b457397562a298 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 7 Aug 2023 11:52:44 -0700 Subject: [PATCH 28/37] Make NNFDM inclusion optional --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4445fa6..b3fcf6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,7 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) - FIND_PACKAGE(NNFDM REQUIRED) + FIND_PACKAGE(NNFDM) IF(NNFDM_FOUND) SET(HAVE_NNFDM TRUE) INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) From 33bb2808160d9377607efa2cc9625ee6652a55b3 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 2 Oct 2023 11:41:18 -0700 Subject: [PATCH 29/37] Updates: run some mpiGraph tests on rabbits --- src/axl_internal.h | 2 +- test/nnf/axl.containers/Dockerfile | 4 ++-- test/nnf/axl.containers/axl-container-tioga.yaml | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/axl_internal.h b/src/axl_internal.h index 776b12d..5c49844 100644 --- a/src/axl_internal.h +++ b/src/axl_internal.h @@ -66,7 +66,7 @@ extern int axl_rank; /* attaches function name, file name, and line number to error messages * https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html */ #define AXL_ERR(format, ...) axl_err(format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) -#define AXL_DBG(level, format, ...) axl_dbg(level, format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) +#define AXL_DBG(level, format, ...) axl_dbg(level, " @ %s %s:%d\n " format, __func__, __FILE__, __LINE__, ##__VA_ARGS__) /* ========================================= diff --git a/test/nnf/axl.containers/Dockerfile b/test/nnf/axl.containers/Dockerfile index 4674aff..71f6a02 100644 --- a/test/nnf/axl.containers/Dockerfile +++ b/test/nnf/axl.containers/Dockerfile @@ -28,11 +28,11 @@ RUN apt install -y make libopenmpi-dev # Build application WORKDIR /src -COPY src . +COPY mpiGraph . RUN make # Final stage - start fresh and don't carry over the build artifacts FROM ghcr.io/nearnodeflash/nnf-mfu:master # Copy application from build stage into final stage -COPY --from=build /src/mpi_hello_world /usr/bin/mpi_hello_world +COPY --from=build /src/mpiGraph /usr/bin/mpiGraph diff --git a/test/nnf/axl.containers/axl-container-tioga.yaml b/test/nnf/axl.containers/axl-container-tioga.yaml index 13519a1..a257373 100644 --- a/test/nnf/axl.containers/axl-container-tioga.yaml +++ b/test/nnf/axl.containers/axl-container-tioga.yaml @@ -39,9 +39,10 @@ data: - name: readonly-red-rock-slushy containers: - name: axl-container-martymcf - image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty + image: czregistry.llnl.gov:5050/scr/rabbit-containers:mpigraph command: ["/bin/bash"] - args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf"] + # args: ["-c", "mpirun mpi_hello_world $(DW_JOB_storage) /p/lslide/martymcf >& /p/lslide/martymcf/mpirun.log"] + args: ["-c", "mpirun --map-by node -np 2 mpiGraph 1048576 100 1000 > /p/lslide/martymcf/mpiGraph.out"] volumeMounts: - mountPath: /p/lslide name: lslide @@ -63,4 +64,4 @@ data: - NET_BIND_SERVICE - SYS_CHROOT - AUDIT_WRITE - image: czregistry.llnl.gov:5050/scr/rabbit-containers:marty + image: czregistry.llnl.gov:5050/scr/rabbit-containers:mpigraph From 87068a9ef96dbe903050811c7525786f900a1dc7 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 2 Oct 2023 14:48:50 -0700 Subject: [PATCH 30/37] Updates for v0.0.6 version of library --- src/nnfdm.cpp | 2 ++ test/axl_test_nnfdm.cpp | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 4155de9..e92c1f2 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -177,6 +177,8 @@ int nnfdm_start(int id) , "" // Extra options to pass to `dcp` if present in the Data Movement command. , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. + , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. + , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. ); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp index a28bbdd..14364bb 100644 --- a/test/axl_test_nnfdm.cpp +++ b/test/axl_test_nnfdm.cpp @@ -48,13 +48,15 @@ int main(int argc, char** argv) { // Create an offload request - CreateRequest create_request( "/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo" - , "/p/lslide/martymcf/bar" - , false // ?? - , "" // ?? - , false // ?? - , false // ?? - ); + CreateRequest create_request( "/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo" + , "/p/lslide/martymcf/bar" + , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement + , "" // Extra options to pass to `dcp` if present in the Data Movement command. + , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. + , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. + , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. + , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. + ); CreateResponse create_response; RPCStatus status = client.Create(workflow, create_request, &create_response); From da3301422f0f4cbbd13c90a386beef952edce2bc Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 7 Jun 2024 15:40:38 -0700 Subject: [PATCH 31/37] Added new profile parameter for nnfdm::CreateResponse API --- src/nnfdm.cpp | 1 + test/axl_test_nnfdm.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index e92c1f2..1b9f07f 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -179,6 +179,7 @@ int nnfdm_start(int id) , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. + , "" // Data movement profile. Empty will default to the default profile. ); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp index 14364bb..5635d42 100644 --- a/test/axl_test_nnfdm.cpp +++ b/test/axl_test_nnfdm.cpp @@ -56,6 +56,7 @@ int main(int argc, char** argv) , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. + , "" // Data movement profile. Empty will default to the default profile. ); CreateResponse create_response; From 28922bd30364cdaac559c66121ef0a8a6068f6b5 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 18 Jun 2024 16:09:44 -0700 Subject: [PATCH 32/37] Improved error handling around nnfdm create --- src/nnfdm.cpp | 48 ++++++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 1b9f07f..bb9ee8f 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -76,7 +76,6 @@ namespace { /*NOTREACHED*/ } - switch (status_response.state()) { case nnfdm::StatusResponse::State::STATE_PENDING: case nnfdm::StatusResponse::State::STATE_STARTING: @@ -171,30 +170,26 @@ int nnfdm_start(int id) char* dst_filename; kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); - nnfdm::CreateRequest create_request( std::string{src_filename} // Source file or directory - , std::string{dst_filename} // Destination file or directory - , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement - , "" // Extra options to pass to `dcp` if present in the Data Movement command. - , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. - , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. - , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. - , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. - , "" // Data movement profile. Empty will default to the default profile. - ); + nnfdm::CreateRequest create_request( + std::string{src_filename} // Source file or directory + , std::string{dst_filename} // Destination file or directory + , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement + , "" // Extra options to pass to `dcp` if present in the Data Movement command. + , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. + , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. + , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. + , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. + , "" // Data movement profile. Empty will default to the default profile. + ); + nnfdm::CreateResponse create_response; - nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow - , create_request - , &create_response); + nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow, create_request, &create_response); if (!rpc_status.ok()) { - axl_abort( -1 - , "NNFDM Create(%s, %s) failed with error %d (%s) @ %s:%d" - , src_filename - , dst_filename - , rpc_status.error_code() - , rpc_status.error_message().c_str() - , __FILE__ - , __LINE__); - /*NOTREACHED*/ + axl_err("%s: NNFDM Create(%s, %s) RPC failed with error %d (%s)" + , __PRETTY_FUNCTION__, src_filename, dst_filename + , rpc_status.error_code(), rpc_status.error_message().c_str()); + kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_ERROR); + rc = AXL_FAILURE; } if (create_response.status() == nnfdm::CreateResponse::Status::STATUS_SUCCESS) { @@ -203,11 +198,8 @@ int nnfdm_start(int id) kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, create_response.uid().c_str()); } else { - AXL_DBG(1, "Create(%s, %s) FAILED:\n error=%d (%s)" - , src_filename - , dst_filename - , create_response.status() - , create_response.message().c_str()); + axl_err("%s: NNFDM Create(%s, %s) Response Status is not SUCCESS: response status is =%d\n%s" + , __PRETTY_FUNCTION__, src_filename, dst_filename, create_response.status(), create_response.message().c_str()); kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_ERROR); rc = AXL_FAILURE; } From ac585780724d154d46a345134649111cffe2942d Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 5 Aug 2024 12:39:04 -0700 Subject: [PATCH 33/37] Added more debugging capabilities --- src/axl_err.c | 66 ++++++++++++++++++++++++++---------------- src/axl_internal.h | 5 ++-- src/nnfdm.cpp | 72 ++++++++++++++++++++++++---------------------- 3 files changed, 82 insertions(+), 61 deletions(-) diff --git a/src/axl_err.c b/src/axl_err.c index 493a67b..f1dfa58 100644 --- a/src/axl_err.c +++ b/src/axl_err.c @@ -10,6 +10,11 @@ /* gethostname */ #include +#include +#include +#include +#include + /* axl version */ #include "axl.h" @@ -17,57 +22,68 @@ * set in AXL_Init used in axl_dbg */ int axl_debug; -/* print message to stdout if axl_debug is set and it is >= level */ -void axl_dbg(int level, const char* fmt, ...) +static void axl_print_date_and_hostinfo(const char* title) { - /* get my hostname */ char hostname[256]; if (gethostname(hostname, sizeof(hostname)) != 0) { /* TODO: error! */ } - - va_list argp; + + struct timeval curTime; + gettimeofday(&curTime, NULL); + int milli = curTime.tv_usec / 1000; + + time_t rawtime; + struct tm * timeinfo; + char buffer[80]; + + time(&rawtime); + timeinfo = localtime(&rawtime); + + strftime(buffer, 80, "%Y-%m-%d %H:%M:%S", timeinfo); + + char currentTime[84] = ""; + sprintf(currentTime, "%s:%d", buffer, milli); + + fprintf(stderr, "%s %s:%d %s:", currentTime, title, getpid(), hostname); +} + +/* print message to stderr if axl_debug is set and it is >= level */ +void axl_dbg(int level, const char* fmt, ...) +{ if (level == 0 || (axl_debug > 0 && axl_debug >= level)) { - fprintf(stdout, "AXL %s: %s: ", AXL_VERSION, hostname); + axl_print_date_and_hostinfo("AXL"); + + va_list argp; va_start(argp, fmt); - vfprintf(stdout, fmt, argp); + vfprintf(stderr, fmt, argp); va_end(argp); - fprintf(stdout, "\n"); + fprintf(stderr, "\n"); } } -/* print error message to stdout */ +/* print error message to stderr */ void axl_err(const char* fmt, ...) { - /* get my hostname */ - char hostname[256]; - if (gethostname(hostname, sizeof(hostname)) != 0) { - /* TODO: error! */ - } + axl_print_date_and_hostinfo("AXL ERROR"); va_list argp; - fprintf(stdout, "AXL %s ERROR: %s: ", AXL_VERSION, hostname); va_start(argp, fmt); - vfprintf(stdout, fmt, argp); + vfprintf(stderr, fmt, argp); va_end(argp); - fprintf(stdout, "\n"); + fprintf(stderr, "\n"); } /* print abort message and kill run */ void axl_abort(int rc, const char* fmt, ...) { - /* get my hostname */ - char hostname[256]; - if (gethostname(hostname, sizeof(hostname)) != 0) { - /* TODO: error! */ - } - + axl_print_date_and_hostinfo("AXL ABORT"); + va_list argp; - fprintf(stderr, "AXL %s ABORT: %s: ", AXL_VERSION, hostname); va_start(argp, fmt); vfprintf(stderr, fmt, argp); va_end(argp); fprintf(stderr, "\n"); - + exit(rc); } diff --git a/src/axl_internal.h b/src/axl_internal.h index 5c49844..acf2080 100644 --- a/src/axl_internal.h +++ b/src/axl_internal.h @@ -65,8 +65,9 @@ extern int axl_rank; /* attaches function name, file name, and line number to error messages * https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html */ -#define AXL_ERR(format, ...) axl_err(format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) -#define AXL_DBG(level, format, ...) axl_dbg(level, " @ %s %s:%d\n " format, __func__, __FILE__, __LINE__, ##__VA_ARGS__) +#define AXL_ERR(format, ...) axl_err( " @ %s:%d " format, __func__, __LINE__, ##__VA_ARGS__) +#define AXL_DBG(level, format, ...) axl_dbg(level, " @ %s:%d " format, __func__, __LINE__, ##__VA_ARGS__) +#define AXL_ABORT(rc, format, ...) axl_abort(rc, " @ %s:%d " format, __func__, __LINE__, ##__VA_ARGS__) /* ========================================= diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index bb9ee8f..8d4ca70 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -62,12 +62,14 @@ namespace { int nnfdm_stat(const char* fname, const char* uid, int64_t max_seconds_to_wait) { + AXL_DBG(1, "%s", fname); int rval = 0; + int retry_count = 0; nnfdm::StatusResponse status_response; nnfdm::RPCStatus rpc_status{ nnfdm_client->Status(*nnfdm_workflow, nnfdm::StatusRequest{std::string{uid}, max_seconds_to_wait}, &status_response) }; if (!rpc_status.ok()) { - axl_abort( -1 + AXL_ABORT( -1 , "NNFDM Status RPC FAILED %d: %s @ %s:%d" , rpc_status.error_code() , rpc_status.error_message().c_str() @@ -84,18 +86,18 @@ namespace { break; case nnfdm::StatusResponse::State::STATE_COMPLETED: if (status_response.status() == nnfdm::StatusResponse::Status::STATUS_SUCCESS) { - AXL_DBG(1, "Status of offload(filename=%s, uid=%s): %s", fname, uid, print_status(status_response).c_str()); + AXL_DBG(1, "Offload Complete(%s)", fname ); rval = AXL_STATUS_DEST; } else { rval = AXL_STATUS_ERROR; - axl_err( "NNFDM Offload Status UNSUCCESSFUL: %d\n%s" + AXL_ERR( "NNFDM Offload Status UNSUCCESSFUL: %d\n%s" , status_response.status() , print_status(status_response).c_str()); } break; default: - axl_abort( -1 + AXL_ABORT( -1 , "NNFDM Offload State STATE UNKNOWN: %d\n%s" , status_response.status() , print_status(status_response).c_str()); @@ -110,6 +112,7 @@ extern "C" { void nnfdm_init() { + AXL_DBG(1, "%p", nnfdm_client); if (nnfdm_client == nullptr) { const std::string socket_name{"unix:///var/run/nnf-dm.sock"}; const std::string workflow_name{getenv("DW_WORKFLOW_NAME")}; @@ -117,7 +120,7 @@ void nnfdm_init() nnfdm_client = new nnfdm::DataMoverClient{ socket_name }; if (nnfdm_client == nullptr) { - axl_abort(-1 + AXL_ABORT(-1 , "NNFDM init: Failed to create data movement client " "instance for %s@ %s:%d" , socket_name.c_str() @@ -127,7 +130,7 @@ void nnfdm_init() nnfdm_workflow = new nnfdm::Workflow{workflow_name, workflow_namespace}; if (nnfdm_workflow == nullptr) { - axl_abort( -1 + AXL_ABORT( -1 , "nnfdm_init: Failed to create data movement workflow " "instance for %s/%s@ %s:%d" , workflow_name.c_str() @@ -140,6 +143,7 @@ void nnfdm_init() void nnfdm_finalize() { + AXL_DBG(1, "%p", nnfdm_client); if (nnfdm_client) { delete nnfdm_workflow; delete nnfdm_client; @@ -151,6 +155,7 @@ void nnfdm_finalize() int nnfdm_start(int id) { + AXL_DBG(1, "%d", id); int rc = AXL_SUCCESS; /* Record that we started transfer of this file list */ @@ -170,6 +175,7 @@ int nnfdm_start(int id) char* dst_filename; kvtree_util_get_str(elem_hash, AXL_KEY_FILE_DEST, &dst_filename); + AXL_DBG(1, "nnfdm::CreateRequest(src=%s, dst=%s)" , src_filename, dst_filename); nnfdm::CreateRequest create_request( std::string{src_filename} // Source file or directory , std::string{dst_filename} // Destination file or directory @@ -185,7 +191,7 @@ int nnfdm_start(int id) nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow, create_request, &create_response); if (!rpc_status.ok()) { - axl_err("%s: NNFDM Create(%s, %s) RPC failed with error %d (%s)" + AXL_ERR("%s: NNFDM Create(%s, %s) RPC failed with error %d (%s)" , __PRETTY_FUNCTION__, src_filename, dst_filename , rpc_status.error_code(), rpc_status.error_message().c_str()); kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_ERROR); @@ -198,8 +204,8 @@ int nnfdm_start(int id) kvtree_util_set_str(elem_hash, AXL_KEY_FILE_SESSION_UID, create_response.uid().c_str()); } else { - axl_err("%s: NNFDM Create(%s, %s) Response Status is not SUCCESS: response status is =%d\n%s" - , __PRETTY_FUNCTION__, src_filename, dst_filename, create_response.status(), create_response.message().c_str()); + AXL_ERR("%s: NNFDM Create(%s, %s) Response Status is not SUCCESS: response status: %d - %s" + , __PRETTY_FUNCTION__, src_filename, dst_filename, create_response.status(), create_response.message().c_str() ); kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_ERROR); rc = AXL_FAILURE; } @@ -208,7 +214,9 @@ int nnfdm_start(int id) return rc; } -int nnfdm_test(int id) { +int nnfdm_test(int id) +{ + AXL_DBG(1, "%d", id); kvtree* file_list = axl_kvtrees[id]; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); @@ -238,7 +246,9 @@ int nnfdm_test(int id) { return AXL_SUCCESS; } -int nnfdm_cancel(int id) { +int nnfdm_cancel(int id) +{ + AXL_DBG(1, "%d", id); kvtree* file_list = axl_kvtrees[id]; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); @@ -265,7 +275,7 @@ int nnfdm_cancel(int id) { , cancel_request , &cancel_response); if (!rpc_status.ok()) { - axl_abort(-1, + AXL_ABORT(-1, "NNFDM Cancel(uid=%s, file=%s) failed with error %d (%s) @ %s:%d" , uid , src_filename @@ -282,24 +292,13 @@ int nnfdm_cancel(int id) { // // Assume that unfound uids simply completed previously // - axl_dbg( 0 - , "NNFDM Cancel(uid=%s, file=%s) NOTFOUND - " - "IGNORING @ %s:%d" - , uid - , src_filename - , __FILE__ - , __LINE__); + AXL_DBG(1, "NNFDM Cancel(uid=%s, file=%s) NOTFOUND - IGNORING", uid, src_filename); continue; case nnfdm::CancelResponse::STATUS_SUCCESS: - axl_dbg( 0 - , "NNFDM Cancel(uid=%s, file=%s) Canceled @ %s:%d" - , uid - , src_filename - , __FILE__ - , __LINE__); + AXL_DBG(1, "NNFDM Cancel(uid=%s, file=%s) Canceled @ %s:%d", uid, src_filename); break; default: - axl_abort( -1 + AXL_ABORT( -1 , "NNFDM Cancel(uid=%s, file=%s) " "Failed with error %d - IGNORING @ %s:%d" , uid @@ -319,7 +318,7 @@ int nnfdm_cancel(int id) { , &deleteResponse); if (!rpc_status.ok()) { - axl_abort( -1 + AXL_ABORT( -1 , "NNFDM Delete(uid=%s, file=%s) RPC FAILED: " "%d (%s) @ %s:%d" , uid @@ -336,7 +335,7 @@ int nnfdm_cancel(int id) { case nnfdm::DeleteResponse::STATUS_SUCCESS: break; default: - axl_abort(-1, + AXL_ABORT(-1, "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s) @ %s:%d", src_filename, deleteResponse.status(), deleteResponse.message().c_str(), __FILE__, __LINE__); @@ -351,6 +350,7 @@ int nnfdm_cancel(int id) { int nnfdm_wait(int id) { + AXL_DBG(1, "id=%d", id); const int64_t max_seconds_to_wait{1}; // wait 1 second kvtree* file_list = axl_kvtrees[id]; kvtree* files = kvtree_get(file_list, AXL_KEY_FILES); @@ -358,12 +358,12 @@ int nnfdm_wait(int id) /* iterate/wait over in-progress files */ for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { char* src_filename = kvtree_elem_key(elem); - int status; - char* uid; - kvtree* elem_hash = kvtree_elem_hash(elem); + + char* uid; kvtree_util_get_str(elem_hash, AXL_KEY_FILE_SESSION_UID, &uid); + int status; do { status = nnfdm_stat(src_filename, uid, max_seconds_to_wait); } while (status == AXL_STATUS_INPROG); @@ -373,7 +373,7 @@ int nnfdm_wait(int id) nnfdm::RPCStatus rpc_status = nnfdm_client->Delete(*nnfdm_workflow, delete_request, &deleteResponse); if (!rpc_status.ok()) { - axl_abort(-1, "NNFDM Delete RPC FAILED: %d (%s)", rpc_status.error_code(), rpc_status.error_message().c_str()); + AXL_ABORT(-1, "NNFDM Delete RPC FAILED: %d (%s)", rpc_status.error_code(), rpc_status.error_message().c_str()); /*NOTEACHED*/ } @@ -382,17 +382,19 @@ int nnfdm_wait(int id) case nnfdm::DeleteResponse::STATUS_SUCCESS: break; default: - axl_abort(-1, + AXL_ABORT(-1, "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s)", src_filename, deleteResponse.status(), deleteResponse.message().c_str()); return 1; } + AXL_DBG(1, "%s Marked Done", src_filename); kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, status); } /* iterate over files */ for (kvtree_elem* elem = kvtree_elem_first(files); elem != NULL; elem = kvtree_elem_next(elem)) { + char* src_filename = kvtree_elem_key(elem); kvtree* elem_hash = kvtree_elem_hash(elem); int status; @@ -400,11 +402,12 @@ int nnfdm_wait(int id) switch (status) { case AXL_STATUS_DEST: + AXL_DBG(1, "%s Is marked as done", src_filename); break; case AXL_STATUS_SOURCE: case AXL_STATUS_ERROR: default: - axl_abort(-1, + AXL_ABORT(-1, "Wait operation called on file with invalid status @ %s:%d", __FILE__, __LINE__ ); @@ -412,6 +415,7 @@ int nnfdm_wait(int id) } /* record transfer complete */ + AXL_DBG(1, "File List is done"); kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_DEST); return AXL_SUCCESS; } From 2608e223cc8062f286ad0e70be06acdc60bc59b9 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 28 Aug 2024 08:58:48 -0700 Subject: [PATCH 34/37] Use SCR data movement profile --- src/nnfdm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 8d4ca70..4758297 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -185,7 +185,7 @@ int nnfdm_start(int id) , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. - , "" // Data movement profile. Empty will default to the default profile. + , "scr" // Data movement profile. Empty will default to the default profile. ); nnfdm::CreateResponse create_response; From e59fc12a5f4ca2c2f4cf473a8554d37329961c5d Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 29 Aug 2024 14:02:53 -0700 Subject: [PATCH 35/37] Work with nnfdm SCR profile --- CMakeLists.txt | 4 +++- src/nnfdm.cpp | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3fcf6c..036040f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,10 +80,12 @@ ENDIF(ENABLE_CRAY_DW) ## DataMover IF(ENABLE_HPE_NNFDM) - FIND_PACKAGE(NNFDM) + FIND_PACKAGE(NNFDM REQUIRED) IF(NNFDM_FOUND) SET(HAVE_NNFDM TRUE) INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIR}) + MESSAGE(STATUS "NNFDM_INCLUDE_DIR: ${NNFDM_INCLUDE_DIR}") + MESSAGE(STATUS "NNFDM_LIBRARIES: ${NNFDM_LIBRARIES}") LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES}) LIST(APPEND AXL_EXTERNAL_STATIC_LIBS ${NNFDM_LIBRARIES}) ENDIF(NNFDM_FOUND) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 4758297..33480ce 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -170,6 +170,7 @@ int nnfdm_start(int id) { char* src_filename = kvtree_elem_key(elem); kvtree* elem_hash = kvtree_elem_hash(elem); + const std::string dm_profile{"scr"}; /* get the destination for this file */ char* dst_filename; @@ -185,9 +186,11 @@ int nnfdm_start(int id) , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. , -1 // The number of max_slots specified in the MPI hostfile. A value of 0 disables the use of max_slots in the hostfile. -1 will defer to the server side configuration. - , "scr" // Data movement profile. Empty will default to the default profile. + , dm_profile // Data movement profile. Empty will default to the default profile. ); + AXL_DBG(1, "create_request(%s, %s, ..., %s)", src_filename, dst_filename, dm_profile.c_str() ); + nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow, create_request, &create_response); if (!rpc_status.ok()) { From 6598792c92bfdef7b18da15acd9be4ddcba0b166 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 10 Sep 2024 13:52:33 -0700 Subject: [PATCH 36/37] axl: Update to latest v0.1.7 nnf-dm version --- src/nnfdm.cpp | 1 + test/axl_test_nnfdm.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 33480ce..502b869 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -181,6 +181,7 @@ int nnfdm_start(int id) std::string{src_filename} // Source file or directory , std::string{dst_filename} // Destination file or directory , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement + , "" // mpirun command line options , "" // Extra options to pass to `dcp` if present in the Data Movement command. , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. diff --git a/test/axl_test_nnfdm.cpp b/test/axl_test_nnfdm.cpp index 5635d42..36fd231 100644 --- a/test/axl_test_nnfdm.cpp +++ b/test/axl_test_nnfdm.cpp @@ -51,6 +51,7 @@ int main(int argc, char** argv) CreateRequest create_request( "/mnt/nnf/854e2467-968a-4f40-aa8c-4ac53cb86866-0/foo" , "/p/lslide/martymcf/bar" , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement + , "" // mpirun command line options , "" // Extra options to pass to `dcp` if present in the Data Movement command. , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. From 5d4cbccda6ce4675439952410c4266fe262d33d2 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 28 Feb 2025 07:30:55 -0800 Subject: [PATCH 37/37] Changed error message of cancel from an abort to a warning --- src/nnfdm.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/nnfdm.cpp b/src/nnfdm.cpp index 502b869..60180e5 100644 --- a/src/nnfdm.cpp +++ b/src/nnfdm.cpp @@ -17,6 +17,7 @@ namespace nnfdm = near_node_flash::data_movement; namespace { nnfdm::DataMoverClient* nnfdm_client{nullptr}; nnfdm::Workflow* nnfdm_workflow{nullptr}; + std::string axl_nnfdm_dcp_arguments; std::string print_status(nnfdm::StatusResponse& status_response) { @@ -138,6 +139,11 @@ void nnfdm_init() , __FILE__ , __LINE__); } + + char* evar = getenv("AXL_NNFDM_DCP_ARGUMENTS"); + if ( evar != NULL ) { + axl_nnfdm_dcp_arguments = evar; + } } } @@ -182,7 +188,7 @@ int nnfdm_start(int id) , std::string{dst_filename} // Destination file or directory , false // If True, the data movement command runs `/bin/true` rather than perform actual data movement , "" // mpirun command line options - , "" // Extra options to pass to `dcp` if present in the Data Movement command. + , axl_nnfdm_dcp_arguments // Extra options to pass to `dcp` if present in the Data Movement command. , false // If true, enable server-side logging of stdout when the command is successful. Failures output is always logged. , true // If true, store stdout in DataMovementStatusResponse.Message when the command is successful. Failure output is always contained in the message. , -1 // The number of slots specified in the MPI hostfile. A value of 0 disables the use of slots in the hostfile. -1 will defer to the server side configuration. @@ -190,7 +196,7 @@ int nnfdm_start(int id) , dm_profile // Data movement profile. Empty will default to the default profile. ); - AXL_DBG(1, "create_request(%s, %s, ..., %s)", src_filename, dst_filename, dm_profile.c_str() ); + AXL_DBG(1, "create_request(%s, %s, %s, ..., %s)", src_filename, dst_filename, axl_nnfdm_dcp_arguments.c_str(), dm_profile.c_str()); nnfdm::CreateResponse create_response; nnfdm::RPCStatus rpc_status = nnfdm_client->Create( *nnfdm_workflow, create_request, &create_response); @@ -339,11 +345,13 @@ int nnfdm_cancel(int id) case nnfdm::DeleteResponse::STATUS_SUCCESS: break; default: - AXL_ABORT(-1, - "NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s) @ %s:%d", - src_filename, deleteResponse.status(), deleteResponse.message().c_str(), - __FILE__, __LINE__); - return 1; + AXL_ERR( "Warning: NNFDM Offload Delete(%s) UNSUCCESSFUL: %d (%s) @ %s:%d" + , src_filename + , deleteResponse.status() + , deleteResponse.message().c_str() + , __FILE__ + , __LINE__); + break; } kvtree_util_set_int(elem_hash, AXL_KEY_FILE_STATUS, AXL_STATUS_DEST);