Fabrics 2/2: Implementation of the fabrics API

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project.
+# SPDX-FileCopyrightText: 2026 Contributors to the Media eXchange Layer project.
 # SPDX-License-Identifier: Apache-2.0
 
 name: Build and Test the MXL Project
@@ -145,7 +145,8 @@ jobs:
               cmake -S /workspace/mxl -B /workspace/mxl/build/${{ matrix.compiler }} \
                 --preset ${{ matrix.compiler }} \
                 -DMXL_BUILD_NUMBER=${{ github.run_number }} \
-                -DCMAKE_INSTALL_PREFIX=/workspace/mxl/install
+                -DCMAKE_INSTALL_PREFIX=/workspace/mxl/install \
+                -DMXL_ENABLE_FABRICS_OFI=ON  
             "
 
       - name: Save vcpkg cache
@@ -165,6 +166,7 @@ jobs:
       - name: Run Tests
         run: |
           docker run --mount src=${{ github.workspace }},target=/workspace/mxl,type=bind \
+            --shm-size=1g \
             -i mxl_build_container_with_source \
             bash -c "
               cd /workspace/mxl/build/${{ matrix.compiler }} && \

lib/fabrics/include/mxl/fabrics.h

@@ -1,4 +1,5 @@
-// SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project.
+// SPDX-FileCopyrightText: 2026 Contributors to the Media eXchange Layer project.
+//
 // SPDX-License-Identifier: Apache-2.0
 
 #pragma once
@@ -32,7 +33,7 @@ extern "C"
     /** The TargetInfo object holds the local fabric address, keys and memory region addresses for a target. It is returned after setting
      *  up a new target and must be passed to the initiator to connect it.
      */
-    typedef struct mxlTargetInfo_t* mxlTargetInfo;
+    typedef struct mxlFabricsTargetInfo_t* mxlFabricsTargetInfo;
 
     /** The initiator is the logical sender of grains over the network. It is the initiator of transfer requests
      *  to registered memory regions of a target.
@@ -43,15 +44,15 @@ extern "C"
      * Can be obtained by using a flow reader or writer, and converting it to a regions collection
      * with mxlFabricsRegionsForFlowReader() or mxlFabricsRegionsForFlowWriter().
      */
-    typedef struct mxlRegions_t* mxlRegions;
+    typedef struct mxlFabricsRegions_t* mxlFabricsRegions;
 
-    typedef enum mxlFabricsProvider
+    typedef enum mxlFabricsProvider_t
     {
-        MXL_SHARING_PROVIDER_AUTO = 0,  /**< Auto select the best provider. ** This might not be supported by all implementations. */
-        MXL_SHARING_PROVIDER_TCP = 1,   /**< Provider that uses linux tcp sockets. */
-        MXL_SHARING_PROVIDER_VERBS = 2, /**< Provider for userspace verbs (libibverbs) and librdmcm for connection management. */
-        MXL_SHARING_PROVIDER_EFA = 3,   /**< Provider for AWS Elastic Fabric Adapter. */
-        MXL_SHARING_PROVIDER_SHM = 4,   /**< Provider used for moving data between 2 memory regions inside the same system. Supported */
+        MXL_FABRICS_PROVIDER_AUTO = 0,  /**< Auto select the best provider. ** This might not be supported by all implementations. */
+        MXL_FABRICS_PROVIDER_TCP = 1,   /**< Provider that uses linux tcp sockets. */
+        MXL_FABRICS_PROVIDER_VERBS = 2, /**< Provider for userspace verbs (libibverbs) and librdmcm for connection management. */
+        MXL_FABRICS_PROVIDER_EFA = 3,   /**< Provider for AWS Elastic Fabric Adapter. */
+        MXL_FABRICS_PROVIDER_SHM = 4,   /**< Provider used for moving data between 2 memory regions inside the same system. Supported */
     } mxlFabricsProvider;
 
     /** Address of a logical network endpoint. This is analogous to a hostname and port number in classic ipv4 networking.
@@ -60,48 +61,31 @@ extern "C"
      * `node` and `service` pointers are expected to live at least until the target or initiator `setup` function is executed and are
      * internally cloned.
      */
-    typedef struct mxlEndpointAddress_t
+    typedef struct mxlFabricsEndpointAddress_t
     {
         char const* node;
         char const* service;
-    } mxlEndpointAddress;
+    } mxlFabricsEndpointAddress;
 
     /** Configuration object required to set up a new target.
      */
-    typedef struct mxlTargetConfig_t
+    typedef struct mxlFabricsTargetConfig_t
     {
-        mxlEndpointAddress endpointAddress; /**< Bind address for the local endpoint. */
-        mxlFabricsProvider provider;        /**< The provider that should be used */
-        mxlRegions regions;                 /**< Local memory regions of the flow that grains should be written to. */
-        bool deviceSupport;                 /**< Require support of transfers involving device memory. */
-    } mxlTargetConfig;
+        mxlFabricsEndpointAddress endpointAddress; /**< Bind address for the local endpoint. */
+        mxlFabricsProvider provider;               /**< The provider that should be used */
+        mxlFabricsRegions regions;                 /**< Local memory regions of the flow that grains should be written to. */
+        bool deviceSupport;                        /**< Require support of transfers involving device memory. */
+    } mxlFabricsTargetConfig;
 
     /** Configuration object required to set up an initiator.
      */
-    typedef struct mxlInitiatorConfig_t
+    typedef struct mxlFabricsInitiatorConfig_t
     {
-        mxlEndpointAddress endpointAddress; /**< Bind address for the local endpoint. */
-        mxlFabricsProvider provider;        /**< The provider that should be used. */
-        mxlRegions regions;                 /**< Local memory regions of the flow that grains should source of remote write requests. */
-        bool deviceSupport;                 /**< Require support of transfers involving device memory. */
-    } mxlInitiatorConfig;
-
-    /** Configuration for a memory region location.
-     */
-    typedef struct mxlFabricsMemoryRegionLocation_t
-    {
-        mxlPayloadLocation type; /**< Memory type of the payload. */
-        uint64_t deviceId;       /**< Device Index when device memory is used, otherwise it is ignored. */
-    } mxlFabricsMemoryRegionLocation;
-
-    /** Configuration for a user supplied memory region.
-     */
-    typedef struct mxlFabricsMemoryRegion_t
-    {
-        uintptr_t addr;                     /**< Start address of the contiguous memory region. */
-        size_t size;                        /**< Size of that memory region */
-        mxlFabricsMemoryRegionLocation loc; /**< Location information for that memory region. */
-    } mxlFabricsMemoryRegion;
+        mxlFabricsEndpointAddress endpointAddress; /**< Bind address for the local endpoint. */
+        mxlFabricsProvider provider;               /**< The provider that should be used. */
+        mxlFabricsRegions regions;                 /**< Local memory regions of the flow that grains should source of remote write requests. */
+        bool deviceSupport;                        /**< Require support of transfers involving device memory. */
+    } mxlFabricsInitiatorConfig;
 
     /**
      * Get the backing memory regions of a flow associated with a flow reader.
@@ -111,7 +95,7 @@ extern "C"
      * \param out_regions A pointer to a memory location where the address of the returned collection of memory regions will be written.
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsRegionsForFlowReader(mxlFlowReader in_reader, mxlRegions* out_regions);
+    mxlStatus mxlFabricsRegionsForFlowReader(mxlFlowReader in_reader, mxlFabricsRegions* out_regions);
 
     /**
      * Get the backing memory regions of a flow associated with a flow writer.
@@ -121,18 +105,7 @@ extern "C"
      * \param out_regions A pointer to a memory location where the address of the returned collection of memory regions will be written.
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsRegionsForFlowWriter(mxlFlowWriter in_writer, mxlRegions* out_regions);
-
-    /**
-     * Create a regions object from a list of memory region groups.
-     * \param in_regions A pointer to an array of memory region groups.
-     * \param in_count The number of memory region groups in the array.
-     * \param out_regions Returns a pointer to the created regions object. The user is responsible for freeing this object by calling
-     * `mxlFabricsRegionsFree()`.
-     * \return MXL_STATUS_OK if the regions object was successfully created.
-     */
-    MXL_EXPORT
-    mxlStatus mxlFabricsRegionsFromUserBuffers(mxlFabricsMemoryRegion const* in_regions, size_t in_count, mxlRegions* out_regions);
+    mxlStatus mxlFabricsRegionsForFlowWriter(mxlFlowWriter in_writer, mxlFabricsRegions* out_regions);
 
     /**
      * Free a regions object previously allocated by mxlFabricsRegionsForFlowReader(), mxlFabricsRegionsForFlowWriter() or
@@ -141,7 +114,7 @@ extern "C"
      * \return MXL_STATUS_OK if the regions object was freed
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsRegionsFree(mxlRegions in_regions);
+    mxlStatus mxlFabricsRegionsFree(mxlFabricsRegions in_regions);
 
     /**
      * Create a new mxl-fabrics from an mxl instance. Targets and initiators created from this mxl-fabrics instance
@@ -184,31 +157,31 @@ extern "C"
      * \param in_target A valid fabrics target
      * \param in_config The target configuration. This will be used to create an endpoint and register a memory region. The memory region
      * corresponds to the one that will be written to by the initiator.
-     * \param out_info An mxlTargetInfo_t object which should be shared to a remote initiator which this target should receive data from. The
+     * \param out_info An mxlFabricsTargetInfo_t object which should be shared to a remote initiator which this target should receive data from. The
      * object must be freed with mxlFabricsFreeTargetInfo().
      * \return The result code. \see mxlStatus
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsTargetSetup(mxlFabricsTarget in_target, mxlTargetConfig* in_config, mxlTargetInfo* out_info);
+    mxlStatus mxlFabricsTargetSetup(mxlFabricsTarget in_target, mxlFabricsTargetConfig const* in_config, mxlFabricsTargetInfo* out_info);
 
     /**
      * Non-blocking accessor for a flow grain at a specific index.
      * \param in_target A valid fabrics target
-     * \param out_index The index of the grain that is ready, if any.
+     * \param out_grainIndex The index of the grain that was written, if any.
      * \return The result code. MXL_ERR_NOT_READY if no grain was available at the time of the call, and the call should be retried. \see mxlStatus
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsTargetTryNewGrain(mxlFabricsTarget in_target, uint64_t* out_index);
+    mxlStatus mxlFabricsTargetReadGrainNonBlocking(mxlFabricsTarget in_target, uint64_t* out_grainIndex);
 
     /**
      * Blocking accessor for a flow grain at a specific index.
      * \param in_target A valid fabrics target
-     * \param out_index The index of the grain that is ready, if any.
+     * \param out_grainIndex The index of the grain that was written, if any.
      * \param in_timeoutMs How long should we wait for the grain (in milliseconds)
      * \return The result code. MXL_ERR_NOT_READY if no grain was available before the timeout. \see mxlStatus
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsTargetWaitForNewGrain(mxlFabricsTarget in_target, uint64_t* out_index, uint16_t in_timeoutMs);
+    mxlStatus mxlFabricsTargetReadGrain(mxlFabricsTarget in_target, uint16_t in_timeoutMs, uint64_t* out_entryIndex);
 
     /**
      * Create a fabrics initiator instance.
@@ -234,7 +207,7 @@ extern "C"
      * \return The result code. \see mxlStatus
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsInitiatorSetup(mxlFabricsInitiator in_initiator, mxlInitiatorConfig const* in_config);
+    mxlStatus mxlFabricsInitiatorSetup(mxlFabricsInitiator in_initiator, mxlFabricsInitiatorConfig const* in_config);
 
     /**
      * Add a target to the initiator. This will allow the initiator to send data to the target in subsequent calls to
@@ -244,7 +217,7 @@ extern "C"
      * \param in_targetInfo The target information. This should be the same as the one returned from "mxlFabricsTargetSetup".
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsInitiatorAddTarget(mxlFabricsInitiator in_initiator, mxlTargetInfo const in_targetInfo);
+    mxlStatus mxlFabricsInitiatorAddTarget(mxlFabricsInitiator in_initiator, mxlFabricsTargetInfo const in_targetInfo);
 
     /**
      * Remove a target from the initiator. This function is always non-blocking. If any additional communication for a graceful shutdown is
@@ -255,17 +228,21 @@ extern "C"
      * \param in_targetInfo The target information. This should be the same as the one returned from "mxlFabricsTargetSetup".
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsInitiatorRemoveTarget(mxlFabricsInitiator in_initiator, mxlTargetInfo const in_targetInfo);
+    mxlStatus mxlFabricsInitiatorRemoveTarget(mxlFabricsInitiator in_initiator, mxlFabricsTargetInfo const in_targetInfo);
 
     /**
      * Enqueue a transfer operation to all added targets. This function is always non-blocking. The transfer operation might be started right
      * away, but is only guaranteed to have completed after mxlFabricsInitiatorMakeProgress*() no longer returns MXL_ERR_NOT_READY.
      * \param in_initiator A valid fabrics initiator
-     * \param in_grainIndex The index of the grain to transfer.
+     * \param in_grainIndex The  grain index to transfer. The ordering was given when mxlFabricsRegions object were created. This is true for both
+     * local and remote memory regions.
+     * \param in_startSlice The start slice in the slice range to transfer. This is inclusive.
+     * \param in_endSlice The end slice in the slice range to transfer. This is exclusive.
      * \return The result code. \see mxlStatus
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsInitiatorTransferGrain(mxlFabricsInitiator in_initiator, uint64_t in_grainIndex);
+    mxlStatus mxlFabricsInitiatorTransferGrain(mxlFabricsInitiator in_initiator, uint64_t in_grainIndex, uint16_t in_startSlice,
+        uint16_t in_endSlice);
 
     /**
      * This function must be called regularly for the initiator to make progress on queued transfer operations, connection establishment
@@ -314,23 +291,23 @@ extern "C"
      * \param in_stringSize The size of the output string.
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsTargetInfoToString(mxlTargetInfo const in_targetInfo, char* out_string, size_t* in_stringSize);
+    mxlStatus mxlFabricsTargetInfoToString(mxlFabricsTargetInfo const in_targetInfo, char* out_string, size_t* in_stringSize);
 
     /**
      * Parse a targetInfo object from its string representation.
      * \param in_string A valid string to deserialize
      * \param out_targetInfo A valid target info to deserialize to
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsTargetInfoFromString(char const* in_string, mxlTargetInfo* out_targetInfo);
+    mxlStatus mxlFabricsTargetInfoFromString(char const* in_string, mxlFabricsTargetInfo* out_targetInfo);
 
     /**
-     * Free a mxlTargetInfo object obtained from mxlFabricsTargetSetup() or mxlFabricsTargetInfoFromString().
-     * \param in_info A mxlTargetInfo object
-     * \return MXL_STATUS_OK if the mxlTargetInfo object was freed.
+     * Free a mxlFabricsTargetInfo object obtained from mxlFabricsTargetSetup() or mxlFabricsTargetInfoFromString().
+     * \param in_info A mxlFabricsTargetInfo object
+     * \return MXL_STATUS_OK if the mxlFabricsTargetInfo object was freed.
      */
     MXL_EXPORT
-    mxlStatus mxlFabricsFreeTargetInfo(mxlTargetInfo in_info);
+    mxlStatus mxlFabricsFreeTargetInfo(mxlFabricsTargetInfo in_info);
 
 #ifdef __cplusplus
 }

lib/fabrics/ofi/CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project.
+# SPDX-FileCopyrightText: 2026 Contributors to the Media eXchange Layer project.
 # SPDX-License-Identifier: Apache-2.0
 
 message ("-- Building fabrics library with libfabric/ofi")
@@ -23,24 +23,40 @@ target_compile_features(mxl-fabrics-objects
 target_sources(mxl-fabrics-objects
         PRIVATE
             src/fabrics.cpp
-            src/internal/Exception.cpp
-            src/internal/Provider.cpp
-            src/internal/FabricVersion.cpp
-            src/internal/FabricInfo.cpp
-            src/internal/Fabric.cpp
-            src/internal/Domain.cpp
-            src/internal/Region.cpp
-            src/internal/MemoryRegion.cpp
-            src/internal/RegisteredRegion.cpp
-            src/internal/LocalRegion.cpp
-            src/internal/RemoteRegion.cpp
             src/internal/Address.cpp
             src/internal/AddressVector.cpp
             src/internal/Completion.cpp
             src/internal/CompletionQueue.cpp
+            src/internal/DataLayout.cpp
+            src/internal/Domain.cpp
+            src/internal/Endpoint.cpp
             src/internal/Event.cpp
             src/internal/EventQueue.cpp
-            src/internal/Endpoint.cpp
+            src/internal/Exception.cpp
+            src/internal/FILogging.cpp
+            src/internal/Fabric.cpp
+            src/internal/FabricInfo.cpp
+            src/internal/FabricInstance.cpp
+            src/internal/FabricVersion.cpp
+            src/internal/ImmData.cpp
+            src/internal/Initiator.cpp
+            src/internal/LocalRegion.cpp
+            src/internal/MemoryRegion.cpp
+            src/internal/PassiveEndpoint.cpp
+            src/internal/Protocol.cpp
+            src/internal/ProtocolEgressRMA.cpp
+            src/internal/ProtocolIngressRMA.cpp
+            src/internal/Provider.cpp
+            src/internal/RCInitiator.cpp
+            src/internal/RCTarget.cpp
+            src/internal/RDMInitiator.cpp
+            src/internal/RDMTarget.cpp
+            src/internal/Region.cpp
+            src/internal/RegisteredRegion.cpp
+            src/internal/RemoteRegion.cpp
+            src/internal/GrainSlices.cpp
+            src/internal/Target.cpp
+            src/internal/TargetInfo.cpp
     )
 target_link_libraries(mxl-fabrics-objects
         PRIVATE
@@ -54,6 +70,10 @@ set_target_properties(mxl-fabrics-objects
             C_EXTENSIONS              OFF
             CXX_EXTENSIONS            OFF
     )
+target_include_directories(mxl-fabrics-objects
+        PUBLIC
+            "${CMAKE_CURRENT_LIST_DIR}/src/internal"
+    )
 
 # Enable LTO/IPO on target if enabled and supported
 mxl_enable_target_ipo(mxl-fabrics-objects)