diff --git a/include/rogue/hardware/axi/AxiStreamDma.h b/include/rogue/hardware/axi/AxiStreamDma.h index 6d3ce71a71..48d7f63687 100644 --- a/include/rogue/hardware/axi/AxiStreamDma.h +++ b/include/rogue/hardware/axi/AxiStreamDma.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,9 @@ typedef std::shared_ptr AxiStreamDmaSh * - A background RX thread is started in the constructor and runs until `stop()` * or destruction. * - TX operations execute synchronously in caller context of `acceptFrame()`. + * - Callers must quiesce public driver operations before calling `stop()`. + * Deferred zero-copy buffer returns may overlap `stop()` and are serialized + * against descriptor closure internally. * * Zero-copy model: * - Enabled by default per device path. @@ -104,6 +108,9 @@ class AxiStreamDma : public rogue::interfaces::stream::Master, public rogue::int // Process-local descriptor for TX/RX operations and dest mask programming. int32_t fd_; + // Serializes fd_ close against deferred zero-copy buffer returns. + std::mutex fdMtx_; + // Destination selector used when transmitting frames. uint32_t dest_; @@ -204,7 +211,19 @@ class AxiStreamDma : public rogue::interfaces::stream::Master, public rogue::int /** @brief Destroys the interface and stops background activity. */ ~AxiStreamDma(); - /** @brief Stops RX thread and closes DMA file descriptors. */ + /** + * @brief Stops RX thread and closes the per-instance DMA file descriptor. + * + * @details + * The shared zero-copy DMA mapping remains valid until destruction so + * downstream Rogue buffers retained after `stop()` do not reference + * unmapped memory. + * + * Before calling `stop()`, callers must ensure that stream operations and + * driver controls/accessors have completed and that no new ones can begin. + * Deferred returns from previously issued zero-copy buffers may overlap + * `stop()`; descriptor closure is synchronized with those callbacks. + */ void stop(); /** diff --git a/src/rogue/hardware/axi/AxiStreamDma.cpp b/src/rogue/hardware/axi/AxiStreamDma.cpp index 437c1f0f25..c2d0dc6295 100644 --- a/src/rogue/hardware/axi/AxiStreamDma.cpp +++ b/src/rogue/hardware/axi/AxiStreamDma.cpp @@ -294,6 +294,11 @@ rha::AxiStreamDma::AxiStreamDma(std::string path, uint32_t dest, bool ssiEnable) //! Close the device rha::AxiStreamDma::~AxiStreamDma() { this->stop(); + + if (desc_) { + closeShared(desc_); + desc_.reset(); + } } void rha::AxiStreamDma::stop() { @@ -314,19 +319,15 @@ void rha::AxiStreamDma::stop() { } thread_.reset(); - // Release the shared descriptor exactly once even if stop() is called - // again (or runThread() self-exited and the user called stop() before - // ~AxiStreamDma()): clearing desc_ after closeShared() prevents a - // double-decrement of openCount. - if (desc_) { - closeShared(desc_); - desc_.reset(); - } - - // Close the per-instance fd exactly once. - if (fd_ >= 0) { - ::close(fd_); - fd_ = -1; + // Close the per-instance fd exactly once. The shared DMA mapping remains + // open until ~AxiStreamDma() because zero-copy Rogue buffers can outlive a + // user-initiated stop() and still point into desc_->rawBuff. + { + std::lock_guard lock(fdMtx_); + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } } } @@ -358,10 +359,8 @@ ris::FramePtr rha::AxiStreamDma::acceptReq(uint32_t size, bool zeroCopyEn) { ris::FramePtr frame; uint32_t buffSize; - // Reject use after stop()/teardown. stop() resets the shared-descriptor - // shared_ptr to nullptr and closes the per-instance fd, so a post-stop - // call would otherwise segfault on the buffer-size read below instead - // of throwing a clean GeneralError. + // Reject use after stop()/teardown. stop() closes the per-instance fd, + // and destruction releases the shared descriptor. if (!desc_ || fd_ < 0) throw rogue::GeneralError("AxiStreamDma::acceptReq", "instance has been stopped or did not finish construction"); @@ -389,7 +388,7 @@ ris::FramePtr rha::AxiStreamDma::acceptReq(uint32_t size, bool zeroCopyEn) { // Keep trying since poll call can fire // but getIndex fails because we did not win the buffer lock do { - // Re-check fd validity: a concurrent stop() can toggle fd_ to -1. + // Re-check fd validity before using it in poll() and the driver call. // poll() imposes no FD_SETSIZE ceiling, so large fd values are fine. if (fd_ < 0) throw rogue::GeneralError::create( @@ -442,10 +441,8 @@ void rha::AxiStreamDma::acceptFrame(ris::FramePtr frame) { uint32_t cont; bool emptyFrame; - // Reject use after stop()/teardown. stop() releases the shared descriptor - // and closes fd_, so the inner-loop fd_ guard would still catch this, but - // that fires only after frame->lock() and buffer iteration have already - // run. Fail-fast at entry for a cleaner error path, mirroring acceptReq(). + // Reject use after stop()/teardown before locking or iterating the frame, + // mirroring acceptReq(). if (!desc_ || fd_ < 0) throw rogue::GeneralError("AxiStreamDma::acceptFrame", "instance has been stopped or did not finish construction"); @@ -512,7 +509,7 @@ void rha::AxiStreamDma::acceptFrame(ris::FramePtr frame) { // Keep trying since poll call can fire // but write fails because we did not win the (*it)er lock do { - // Re-check fd validity: a concurrent stop() can toggle fd_ to -1. + // Re-check fd validity before using it in poll() and the driver call. // poll() imposes no FD_SETSIZE ceiling, so large fd values are fine. if (fd_ < 0) throw rogue::GeneralError::create( @@ -557,16 +554,14 @@ void rha::AxiStreamDma::acceptFrame(ris::FramePtr frame) { //! Return a buffer void rha::AxiStreamDma::retBuffer(uint8_t* data, uint32_t meta, uint32_t size) { rogue::GilRelease noGil; - uint32_t ret[100]; - uint32_t count; - uint32_t x; // Buffer is zero copy as indicated by bit 31 if ((meta & 0x80000000) != 0) { // Device is open and buffer is not stale // Bit 30 indicates buffer has already been returned to hardware - if ((fd_ >= 0) && ((meta & 0x40000000) == 0)) { - if (dmaRetIndex(fd_, meta & 0x3FFFFFFF) < 0) + if ((meta & 0x40000000) == 0) { + std::lock_guard lock(fdMtx_); + if (fd_ >= 0 && dmaRetIndex(fd_, meta & 0x3FFFFFFF) < 0) throw(rogue::GeneralError("AxiStreamDma::retBuffer", "AXIS Return Buffer Call Failed!!!!")); } decCounter(size); diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 8aaca17179..2c7c08bd32 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -53,6 +53,7 @@ function(rogue_add_cpp_test target) endfunction() add_subdirectory(core) +add_subdirectory(hardware) add_subdirectory(memory) add_subdirectory(stream) add_subdirectory(protocols) diff --git a/tests/cpp/hardware/CMakeLists.txt b/tests/cpp/hardware/CMakeLists.txt new file mode 100644 index 0000000000..0788699a2f --- /dev/null +++ b/tests/cpp/hardware/CMakeLists.txt @@ -0,0 +1,43 @@ +# Hardware-class tests that run AxiStreamDma against an LD_PRELOAD shim for +# the aes-stream-driver device. ELF symbol interposition is Linux-only; other +# platforms skip this subtree while still building rogue-core. +if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + return() +endif() + +add_library(rogue-cpp-fake-dma-preload MODULE support/fake_dma_preload.c) +target_link_libraries(rogue-cpp-fake-dma-preload PRIVATE ${CMAKE_DL_LIBS} pthread) + +set(_rogue_fake_dma_env + "MPLCONFIGDIR=${CMAKE_BINARY_DIR}/tests/cpp/mplconfig;LD_PRELOAD=$" +) + +rogue_add_cpp_test(rogue-cpp-hardware-axi-stream-dma-zerocopy-lifetime + SOURCES + test_axi_stream_dma_zerocopy_lifetime.cpp + LABELS + cpp-hardware + no-python + LIBRARIES + ${CMAKE_DL_LIBS} +) +add_dependencies(rogue-cpp-hardware-axi-stream-dma-zerocopy-lifetime rogue-cpp-fake-dma-preload) +set_tests_properties(rogue-cpp-hardware-axi-stream-dma-zerocopy-lifetime PROPERTIES + ENVIRONMENT "${_rogue_fake_dma_env}" + TIMEOUT 10 +) + +rogue_add_cpp_test(rogue-cpp-hardware-axi-stream-dma-lifecycle + SOURCES + test_axi_stream_dma_lifecycle.cpp + LABELS + cpp-hardware + no-python + LIBRARIES + ${CMAKE_DL_LIBS} +) +add_dependencies(rogue-cpp-hardware-axi-stream-dma-lifecycle rogue-cpp-fake-dma-preload) +set_tests_properties(rogue-cpp-hardware-axi-stream-dma-lifecycle PROPERTIES + ENVIRONMENT "${_rogue_fake_dma_env}" + TIMEOUT 10 +) diff --git a/tests/cpp/hardware/support/fake_dma_hooks.h b/tests/cpp/hardware/support/fake_dma_hooks.h new file mode 100644 index 0000000000..368afce5ea --- /dev/null +++ b/tests/cpp/hardware/support/fake_dma_hooks.h @@ -0,0 +1,100 @@ +/** + * ---------------------------------------------------------------------------- + * Company : SLAC National Accelerator Laboratory + * ---------------------------------------------------------------------------- + * Description: + * Shared C++ accessors for the fake_dma_preload LD_PRELOAD test shim. + * ---------------------------------------------------------------------------- + * This file is part of the rogue software platform. It is subject to + * the license terms in the LICENSE.txt file found in the top-level directory + * of this distribution and at: + * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. + * No part of the rogue software platform, including this file, may be + * copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE.txt file. + * ---------------------------------------------------------------------------- + **/ +#ifndef ROGUE_TEST_CPP_HARDWARE_SUPPORT_FAKE_DMA_HOOKS_H +#define ROGUE_TEST_CPP_HARDWARE_SUPPORT_FAKE_DMA_HOOKS_H + +#include +#include + +#include + +namespace rogue_test { + +class FakeDma { + public: + using PathFn = const char* (*)(); + using ResetFn = void (*)(); + using BlockNextFn = void (*)(); + using WaitBlockedFn = int (*)(uint32_t); + using ReleaseFn = void (*)(); + using CountFn = int (*)(); + + FakeDma() { + path_ = load("fakedma_path"); + reset_ = load("fakedma_reset"); + blockNext_ = load("fakedma_block_next_ret_index"); + waitBlocked_ = load("fakedma_wait_blocked"); + release_ = load("fakedma_release_blocked"); + mappedCount_ = load("fakedma_mapped_count"); + fdCount_ = load("fakedma_fd_count"); + regionCount_ = load("fakedma_region_count"); + activeCalls_ = load("fakedma_active_call_count"); + overflowCount_ = load("fakedma_tracking_overflow_count"); + isClean_ = load("fakedma_is_clean"); + retIndexCount_ = load("fakedma_ret_index_count"); + closeDuringCall_ = load("fakedma_close_during_driver_call"); + } + + const char* path() const { return path_(); } + void reset() const { reset_(); } + void blockNextRetIndex() const { blockNext_(); } + bool waitBlocked(uint32_t timeoutMs) const { return waitBlocked_(timeoutMs) != 0; } + void releaseBlocked() const { release_(); } + int mappedCount() const { return mappedCount_(); } + int fdCount() const { return fdCount_(); } + int regionCount() const { return regionCount_(); } + int activeCallCount() const { return activeCalls_(); } + int overflowCount() const { return overflowCount_(); } + bool isClean() const { return isClean_() != 0; } + int retIndexCount() const { return retIndexCount_(); } + bool closeDuringDriverCall() const { return closeDuringCall_() != 0; } + + private: + template + Func load(const char* name) { + dlerror(); + void* sym = dlsym(RTLD_DEFAULT, name); + if (sym == nullptr) { + const char* err = dlerror(); + throw std::runtime_error(err != nullptr ? err : name); + } + return reinterpret_cast(sym); + } + + PathFn path_; + ResetFn reset_; + BlockNextFn blockNext_; + WaitBlockedFn waitBlocked_; + ReleaseFn release_; + CountFn mappedCount_; + CountFn fdCount_; + CountFn regionCount_; + CountFn activeCalls_; + CountFn overflowCount_; + CountFn isClean_; + CountFn retIndexCount_; + CountFn closeDuringCall_; +}; + +inline FakeDma& fakeDma() { + static FakeDma fake; + return fake; +} + +} // namespace rogue_test + +#endif diff --git a/tests/cpp/hardware/support/fake_dma_preload.c b/tests/cpp/hardware/support/fake_dma_preload.c new file mode 100644 index 0000000000..e83ee6e885 --- /dev/null +++ b/tests/cpp/hardware/support/fake_dma_preload.c @@ -0,0 +1,507 @@ +/** + * ---------------------------------------------------------------------------- + * Company : SLAC National Accelerator Laboratory + * ---------------------------------------------------------------------------- + * Description: + * Test-only LD_PRELOAD shim that emulates just enough of the aes-stream-driver + * character device for AxiStreamDma to construct, hand out a zero-copy buffer, + * and tear down on a machine with no DMA hardware. It intercepts + * open/close/ioctl/mmap/munmap/poll for a single sentinel device path and + * backs the "DMA" buffers with ordinary anonymous mappings, tracking how many + * are currently mapped. Tests query that count via fakedma_mapped_count() and + * use the blocking hooks below to exercise deferred buffer returns overlapping + * stop() deterministically. + * + * This object is built only under -DROGUE_BUILD_TESTS=ON and is never linked + * into rogue-core or shipped. + * ---------------------------------------------------------------------------- + * This file is part of the rogue software platform. It is subject to + * the license terms in the LICENSE.txt file found in the top-level directory + * of this distribution and at: + * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. + * No part of the rogue software platform, including this file, may be + * copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE.txt file. + * ---------------------------------------------------------------------------- + **/ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * DMA ioctl request codes and version. These mirror the stable kernel ABI in + * include/rogue/hardware/drivers/DmaDriver.h (which is C++-only and cannot be + * included here). Keep in sync with that header. + */ +#define FAKE_DMA_Get_Buff_Count 0x1001 +#define FAKE_DMA_Get_Buff_Size 0x1002 +#define FAKE_DMA_Ret_Index 0x1005 +#define FAKE_DMA_Get_Index 0x1006 +#define FAKE_DMA_Set_MaskBytes 0x1008 +#define FAKE_DMA_Get_Version 0x1009 +#define FAKE_DMA_Get_RxBuff_Count 0x100C +#define FAKE_DMA_Get_TxBuff_Count 0x100D +#define FAKE_DMA_VERSION 0x06 + +/* Sentinel device path intercepted by this shim. */ +#define FAKE_PATH "/tmp/rogue-fake-datadev" +#define FAKE_BUFF_SIZE 4096u +#define FAKE_BUFF_COUNT 8u + +#define MAX_FDS 16 +#define MAX_REGIONS 64 + +static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER; + +static int g_fds[MAX_FDS]; +static int g_fd_n = 0; + +static void* g_region_addr[MAX_REGIONS]; +static size_t g_region_len[MAX_REGIONS]; +static int g_region_n = 0; + +static int g_mapped_count = 0; +static uint32_t g_next_index = 0; +static int g_block_ret_index = 0; +static int g_blocked = 0; +static int g_release = 0; +static int g_active_calls = 0; +static int g_close_during_driver_call = 0; +static int g_ret_index_count = 0; +static int g_tracking_overflow_count = 0; + +typedef int (*open_fn)(const char*, int, ...); +typedef void* (*mmap_fn)(void*, size_t, int, int, int, off_t); +typedef int (*munmap_fn)(void*, size_t); +typedef int (*close_fn)(int); +typedef int (*ioctl_fn)(int, unsigned long, ...); +typedef int (*poll_fn)(struct pollfd*, nfds_t, int); + +static open_fn real_open; +static open_fn real_open64; +static mmap_fn real_mmap; +static mmap_fn real_mmap64; +static munmap_fn real_munmap; +static close_fn real_close; +static ioctl_fn real_ioctl; +static poll_fn real_poll; + +/* ------- bookkeeping helpers (caller holds g_lock) ------- */ + +static int is_our_fd_locked(int fd) { + int i; + for (i = 0; i < g_fd_n; i++) + if (g_fds[i] == fd) return 1; + return 0; +} + +static void add_fd_locked(int fd) { + if (g_fd_n < MAX_FDS) { + g_fds[g_fd_n++] = fd; + } else { + g_tracking_overflow_count++; + } +} + +static void del_fd_locked(int fd) { + int i; + for (i = 0; i < g_fd_n; i++) { + if (g_fds[i] == fd) { + g_fds[i] = g_fds[--g_fd_n]; + return; + } + } +} + +static int is_our_fd(int fd) { + int ours; + pthread_mutex_lock(&g_lock); + ours = is_our_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + return ours; +} + +static void add_region_locked(void* addr, size_t len) { + if (g_region_n < MAX_REGIONS) { + g_region_addr[g_region_n] = addr; + g_region_len[g_region_n] = len; + g_region_n++; + g_mapped_count++; + } else { + g_tracking_overflow_count++; + } +} + +static int del_region_locked(void* addr) { + int i; + for (i = 0; i < g_region_n; i++) { + if (g_region_addr[i] == addr) { + g_region_addr[i] = g_region_addr[g_region_n - 1]; + g_region_len[i] = g_region_len[g_region_n - 1]; + g_region_n--; + g_mapped_count--; + return 1; + } + } + return 0; +} + +/* Return an anonymous mapping standing in for a DMA buffer, and record it. */ +static void* make_fake_buffer(size_t len, int prot) { + void* p = real_mmap(NULL, len, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + pthread_mutex_lock(&g_lock); + add_region_locked(p, len); + pthread_mutex_unlock(&g_lock); + } + return p; +} + +static void driver_call_begin(void) { + pthread_mutex_lock(&g_lock); + g_active_calls++; + pthread_mutex_unlock(&g_lock); +} + +static void driver_call_end(void) { + pthread_mutex_lock(&g_lock); + g_active_calls--; + pthread_cond_broadcast(&g_cond); + pthread_mutex_unlock(&g_lock); +} + +static void maybe_block_ret_index(void) { + pthread_mutex_lock(&g_lock); + if (g_block_ret_index) { + g_block_ret_index = 0; + g_blocked = 1; + pthread_cond_broadcast(&g_cond); + while (!g_release) pthread_cond_wait(&g_cond, &g_lock); + g_blocked = 0; + g_release = 0; + pthread_cond_broadcast(&g_cond); + } + pthread_mutex_unlock(&g_lock); +} + +/* ------- intercepted entry points ------- */ + +int open(const char* path, int flags, ...) { + mode_t mode = 0; + if (!real_open) real_open = (open_fn)dlsym(RTLD_NEXT, "open"); + + if (flags & O_CREAT) { + va_list ap; + va_start(ap, flags); + mode = (mode_t)va_arg(ap, int); + va_end(ap); + } + + if (path != NULL && strcmp(path, FAKE_PATH) == 0) { + int fd = real_open("/dev/null", O_RDWR); + if (fd >= 0) { + pthread_mutex_lock(&g_lock); + add_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + } + return fd; + } + return real_open(path, flags, mode); +} + +int open64(const char* path, int flags, ...) { + mode_t mode = 0; + if (!real_open64) real_open64 = (open_fn)dlsym(RTLD_NEXT, "open64"); + + if (flags & O_CREAT) { + va_list ap; + va_start(ap, flags); + mode = (mode_t)va_arg(ap, int); + va_end(ap); + } + + if (path != NULL && strcmp(path, FAKE_PATH) == 0) { + int fd = real_open64("/dev/null", O_RDWR); + if (fd >= 0) { + pthread_mutex_lock(&g_lock); + add_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + } + return fd; + } + return real_open64(path, flags, mode); +} + +int close(int fd) { + if (!real_close) real_close = (close_fn)dlsym(RTLD_NEXT, "close"); + pthread_mutex_lock(&g_lock); + if (is_our_fd_locked(fd) && g_active_calls != 0) g_close_during_driver_call = 1; + del_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + return real_close(fd); +} + +void* mmap(void* addr, size_t len, int prot, int flags, int fd, off_t off) { + int ours; + if (!real_mmap) { + static __thread int resolving = 0; + if (resolving) return (void*)syscall(SYS_mmap, addr, len, prot, flags, fd, off); + resolving = 1; + real_mmap = (mmap_fn)dlsym(RTLD_NEXT, "mmap"); + resolving = 0; + } + pthread_mutex_lock(&g_lock); + ours = is_our_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + + if (ours) return make_fake_buffer(len, prot); + return real_mmap(addr, len, prot, flags, fd, off); +} + +void* mmap64(void* addr, size_t len, int prot, int flags, int fd, off_t off) { + int ours; + if (!real_mmap64) { + static __thread int resolving = 0; + if (resolving) return (void*)syscall(SYS_mmap, addr, len, prot, flags, fd, off); + resolving = 1; + real_mmap64 = (mmap_fn)dlsym(RTLD_NEXT, "mmap64"); + resolving = 0; + } + if (!real_mmap) real_mmap = (mmap_fn)dlsym(RTLD_NEXT, "mmap"); + + pthread_mutex_lock(&g_lock); + ours = is_our_fd_locked(fd); + pthread_mutex_unlock(&g_lock); + + if (ours) return make_fake_buffer(len, prot); + return real_mmap64(addr, len, prot, flags, fd, off); +} + +int munmap(void* addr, size_t len) { + if (!real_munmap) { + static __thread int resolving = 0; + if (resolving) return (int)syscall(SYS_munmap, addr, len); + resolving = 1; + real_munmap = (munmap_fn)dlsym(RTLD_NEXT, "munmap"); + resolving = 0; + } + pthread_mutex_lock(&g_lock); + del_region_locked(addr); + pthread_mutex_unlock(&g_lock); + return real_munmap(addr, len); +} + +int ioctl(int fd, unsigned long request, ...) { + void* arg; + int ours; + if (!real_ioctl) real_ioctl = (ioctl_fn)dlsym(RTLD_NEXT, "ioctl"); + + ours = is_our_fd(fd); + + if (!ours) { + va_list ap; + va_start(ap, request); + arg = va_arg(ap, void*); + va_end(ap); + return real_ioctl(fd, request, arg); + } + + driver_call_begin(); + switch (request & 0xFFFF) { + case FAKE_DMA_Get_Version: + driver_call_end(); + return FAKE_DMA_VERSION; + case FAKE_DMA_Get_Buff_Size: + driver_call_end(); + return (int)FAKE_BUFF_SIZE; + case FAKE_DMA_Get_Buff_Count: + driver_call_end(); + return (int)FAKE_BUFF_COUNT; + case FAKE_DMA_Get_RxBuff_Count: + driver_call_end(); + return (int)(FAKE_BUFF_COUNT / 2); + case FAKE_DMA_Get_TxBuff_Count: + driver_call_end(); + return (int)(FAKE_BUFF_COUNT / 2); + case FAKE_DMA_Set_MaskBytes: + driver_call_end(); + return 0; + case FAKE_DMA_Get_Index: { + int idx; + pthread_mutex_lock(&g_lock); + idx = (int)(g_next_index++ % FAKE_BUFF_COUNT); + pthread_mutex_unlock(&g_lock); + driver_call_end(); + return idx; + } + case FAKE_DMA_Ret_Index: + maybe_block_ret_index(); + pthread_mutex_lock(&g_lock); + g_ret_index_count++; + pthread_mutex_unlock(&g_lock); + driver_call_end(); + return 0; + default: + driver_call_end(); + return 0; + } +} + +int poll(struct pollfd* fds, nfds_t nfds, int timeout) { + if (!real_poll) real_poll = (poll_fn)dlsym(RTLD_NEXT, "poll"); + + if (nfds == 1 && fds != NULL) { + int ours; + pthread_mutex_lock(&g_lock); + ours = is_our_fd_locked(fds[0].fd); + pthread_mutex_unlock(&g_lock); + + if (ours) { + /* TX/alloc path waits for POLLOUT: report ready so acceptReq/ + * acceptFrame proceed to dmaGetIndex/dmaWrite. */ + if (fds[0].events & POLLOUT) { + fds[0].revents = POLLOUT; + return 1; + } + /* RX path waits for POLLIN: stay idle for the requested timeout and + * report nothing so the worker thread never issues a read. */ + if (timeout > 0) real_poll(NULL, 0, timeout); + fds[0].revents = 0; + return 0; + } + } + return real_poll(fds, nfds, timeout); +} + +/* Query hook for the test: number of fake DMA buffers currently mapped. */ +int fakedma_mapped_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_mapped_count; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_fd_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_fd_n; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_region_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_region_n; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_active_call_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_active_calls; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_tracking_overflow_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_tracking_overflow_count; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_is_clean(void) { + int clean; + pthread_mutex_lock(&g_lock); + clean = (g_fd_n == 0) && (g_region_n == 0) && (g_active_calls == 0) && + (g_tracking_overflow_count == 0); + pthread_mutex_unlock(&g_lock); + return clean; +} + +const char* fakedma_path(void) { + return FAKE_PATH; +} + +void fakedma_reset(void) { + pthread_mutex_lock(&g_lock); + g_block_ret_index = 0; + g_blocked = 0; + g_release = 0; + g_active_calls = 0; + g_close_during_driver_call = 0; + g_ret_index_count = 0; + g_next_index = 0; + pthread_cond_broadcast(&g_cond); + pthread_mutex_unlock(&g_lock); +} + +void fakedma_block_next_ret_index(void) { + pthread_mutex_lock(&g_lock); + g_block_ret_index = 1; + g_blocked = 0; + g_release = 0; + pthread_mutex_unlock(&g_lock); +} + +int fakedma_wait_blocked(uint32_t timeout_ms) { + struct timespec deadline; + int ret = 0; + + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += timeout_ms / 1000u; + deadline.tv_nsec += (long)(timeout_ms % 1000u) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { + deadline.tv_sec++; + deadline.tv_nsec -= 1000000000L; + } + + pthread_mutex_lock(&g_lock); + while (!g_blocked) { + if (pthread_cond_timedwait(&g_cond, &g_lock, &deadline) == ETIMEDOUT) break; + } + ret = g_blocked; + pthread_mutex_unlock(&g_lock); + return ret; +} + +void fakedma_release_blocked(void) { + pthread_mutex_lock(&g_lock); + g_release = 1; + pthread_cond_broadcast(&g_cond); + pthread_mutex_unlock(&g_lock); +} + +int fakedma_ret_index_count(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_ret_index_count; + pthread_mutex_unlock(&g_lock); + return n; +} + +int fakedma_close_during_driver_call(void) { + int n; + pthread_mutex_lock(&g_lock); + n = g_close_during_driver_call; + pthread_mutex_unlock(&g_lock); + return n; +} diff --git a/tests/cpp/hardware/test_axi_stream_dma_lifecycle.cpp b/tests/cpp/hardware/test_axi_stream_dma_lifecycle.cpp new file mode 100644 index 0000000000..58a3b897e1 --- /dev/null +++ b/tests/cpp/hardware/test_axi_stream_dma_lifecycle.cpp @@ -0,0 +1,112 @@ +/** + * ---------------------------------------------------------------------------- + * Company : SLAC National Accelerator Laboratory + * ---------------------------------------------------------------------------- + * Description: + * Native C++ lifecycle test for deferred AxiStreamDma zero-copy buffer + * returns overlapping stop(), using the fake_dma_preload LD_PRELOAD backend. + * ---------------------------------------------------------------------------- + * This file is part of the rogue software platform. It is subject to + * the license terms in the LICENSE.txt file found in the top-level directory + * of this distribution and at: + * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. + * No part of the rogue software platform, including this file, may be + * copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE.txt file. + * ---------------------------------------------------------------------------- + **/ +#include + +#include +#include +#include +#include + +#include "doctest/doctest.h" +#include "rogue/hardware/axi/AxiStreamDma.h" +#include "rogue/interfaces/stream/Frame.h" +#include "support/fake_dma_hooks.h" + +namespace rha = rogue::hardware::axi; + +namespace { + +void requireFakeDmaClean() { + auto& fake = rogue_test::fakeDma(); + INFO("fd_count=" << fake.fdCount()); + INFO("region_count=" << fake.regionCount()); + INFO("active_calls=" << fake.activeCallCount()); + INFO("overflow_count=" << fake.overflowCount()); + REQUIRE(fake.isClean()); + REQUIRE_EQ(fake.mappedCount(), 0); +} + +void checkFakeDmaClean() { + auto& fake = rogue_test::fakeDma(); + INFO("fd_count=" << fake.fdCount()); + INFO("region_count=" << fake.regionCount()); + INFO("active_calls=" << fake.activeCallCount()); + INFO("overflow_count=" << fake.overflowCount()); + CHECK(fake.isClean()); + CHECK_EQ(fake.mappedCount(), 0); +} + +} // namespace + +TEST_CASE("AxiStreamDma stop waits for an in-flight zero-copy buffer return") { + requireFakeDmaClean(); + auto& fake = rogue_test::fakeDma(); + fake.reset(); + + auto dma = rha::AxiStreamDma::create(fake.path(), 0, false); + auto frame = dma->acceptReq(64, true); + REQUIRE(static_cast(frame)); + REQUIRE_EQ(frame->bufferCount(), 1U); + + fake.blockNextRetIndex(); + + std::atomic returnDone{false}; + std::exception_ptr returnError; + std::thread returner([&] { + try { + frame->clear(); + returnDone.store(true); + } catch (...) { + returnError = std::current_exception(); + } + }); + + const bool blocked = fake.waitBlocked(1000); + CHECK(blocked); + if (!blocked) { + fake.releaseBlocked(); + returner.join(); + if (returnError) std::rethrow_exception(returnError); + return; + } + CHECK_FALSE(returnDone.load()); + + std::atomic stopDone{false}; + std::thread stopper([&] { + dma->stop(); + stopDone.store(true); + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + CHECK_FALSE(stopDone.load()); + CHECK_FALSE(fake.closeDuringDriverCall()); + + fake.releaseBlocked(); + returner.join(); + stopper.join(); + if (returnError) std::rethrow_exception(returnError); + + CHECK(returnDone.load()); + CHECK(stopDone.load()); + CHECK_EQ(fake.retIndexCount(), 1); + CHECK_FALSE(fake.closeDuringDriverCall()); + + frame.reset(); + dma.reset(); + checkFakeDmaClean(); +} diff --git a/tests/cpp/hardware/test_axi_stream_dma_zerocopy_lifetime.cpp b/tests/cpp/hardware/test_axi_stream_dma_zerocopy_lifetime.cpp new file mode 100644 index 0000000000..270a5db7eb --- /dev/null +++ b/tests/cpp/hardware/test_axi_stream_dma_zerocopy_lifetime.cpp @@ -0,0 +1,95 @@ +/** + * ---------------------------------------------------------------------------- + * Company : SLAC National Accelerator Laboratory + * ---------------------------------------------------------------------------- + * Description: + * Regression test for the AxiStreamDma zero-copy teardown lifetime fix. + * + * A zero-copy Rogue Buffer points directly into the shared DMA mapping + * (desc_->rawBuff, established by dmaMapDma()). Releasing that mapping inside + * stop() left any frame retained past stop() pointing at unmapped memory. The + * fix moves the shared-descriptor teardown out of stop() and into + * ~AxiStreamDma(), so the mapping outlives stop() and is freed only at + * destruction. + * + * The aes-stream-driver character device does not exist on CI runners, so this + * test runs against the fake_dma_preload LD_PRELOAD shim (wired in by the + * accompanying CMakeLists.txt). The shim reports how many DMA buffers are + * currently mapped via fakedma_mapped_count(); the test asserts the mapping is + * still present after stop() and released exactly at destruction. Reverting the + * fix (moving closeShared() back into stop()) drops the count to zero at stop() + * and fails the post-stop() check. + * ---------------------------------------------------------------------------- + * This file is part of the rogue software platform. It is subject to + * the license terms in the LICENSE.txt file found in the top-level directory + * of this distribution and at: + * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. + * No part of the rogue software platform, including this file, may be + * copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE.txt file. + * ---------------------------------------------------------------------------- + **/ +#include "doctest/doctest.h" +#include "rogue/hardware/axi/AxiStreamDma.h" +#include "rogue/interfaces/stream/Frame.h" +#include "support/fake_dma_hooks.h" + +namespace rha = rogue::hardware::axi; + +namespace { + +void requireFakeDmaClean() { + auto& fake = rogue_test::fakeDma(); + INFO("fd_count=" << fake.fdCount()); + INFO("region_count=" << fake.regionCount()); + INFO("active_calls=" << fake.activeCallCount()); + INFO("overflow_count=" << fake.overflowCount()); + REQUIRE(fake.isClean()); + REQUIRE_EQ(fake.mappedCount(), 0); +} + +void checkFakeDmaClean() { + auto& fake = rogue_test::fakeDma(); + INFO("fd_count=" << fake.fdCount()); + INFO("region_count=" << fake.regionCount()); + INFO("active_calls=" << fake.activeCallCount()); + INFO("overflow_count=" << fake.overflowCount()); + CHECK(fake.isClean()); + CHECK_EQ(fake.mappedCount(), 0); +} + +} // namespace + +TEST_CASE("AxiStreamDma keeps the shared DMA mapping alive across stop()") { + requireFakeDmaClean(); + auto& fake = rogue_test::fakeDma(); + fake.reset(); + + auto dma = rha::AxiStreamDma::create(fake.path(), 0, false); + + // dmaMapDma() maps all shared zero-copy buffers at construction. + const int mappedAtCtor = fake.mappedCount(); + REQUIRE(mappedAtCtor > 0); + + // Obtain a zero-copy frame and retain it past stop(); its buffer points + // into the shared mapping. + auto frame = dma->acceptReq(4096, true); + REQUIRE(static_cast(frame)); + REQUIRE(frame->bufferCount() > 0); + + dma->stop(); + + // The fix keeps the shared mapping valid after stop() so the retained frame + // does not dangle. Reverted code releases it inside stop(), dropping the + // count to zero here. + CHECK_EQ(fake.mappedCount(), mappedAtCtor); + + // Returning the buffer post-stop() is a no-op on the driver (fd_ == -1); it + // must not touch the still-mapped memory or throw. + frame.reset(); + CHECK_EQ(fake.mappedCount(), mappedAtCtor); + + // Destruction is the single point that releases the shared mapping. + dma.reset(); + checkFakeDmaClean(); +}