Refactor the Zerocopy Direct API 22/4222/10
authorNitin Bhat <nbhat4@illinois.edu>
Fri, 25 May 2018 14:45:36 +0000 (09:45 -0500)
committerNitin Bhat <nbhat4@illinois.edu>
Mon, 2 Jul 2018 22:25:11 +0000 (17:25 -0500)
Use a single dynamically allocated object of the type NcpyOperationInfo,
as a single argument, instead of passing many parameters. This design leads
to fewer smaller allocations in the machine layer code and allows to simplify
code significantly. It also makes the code more maintainable and flexible
towards changing behavior or passing new information down to the Lrts RDMA methods.

This commit also adds the feature (Feature #1919) for an optional arbitrary pointer
being passed by the user before the zerocopy operation (get/put). This reference
pointer is returned back to the user in the callback using the CkNcpyAck object.

Change-Id: I0ec2320b664aec8dee6d0a4e327c0ee689326e4f

20 files changed:
doc/charm++/zerocopyapi.tex
src/arch/gni/machine-onesided.c
src/arch/gni/machine.C
src/arch/mpi/machine-onesided.c
src/arch/mpi/machine.C
src/arch/ofi/machine-onesided.c
src/arch/ofi/machine-onesided.h
src/arch/util/machine-rdma.h
src/arch/verbs/machine-ibverbs.c
src/arch/verbs/machine-onesided.c
src/ck-core/ckrdma.C
src/ck-core/ckrdma.h
src/ck-core/init.C
src/conv-core/conv-header.h [new file with mode: 0644]
src/conv-core/conv-rdma.c
src/conv-core/conv-rdma.h
src/conv-core/converse.h
src/scripts/Makefile
src/util/cmirdmautils.c [new file with mode: 0644]
src/util/cmirdmautils.h [new file with mode: 0644]

index 25d34120d1c526504000f432ed30391840ceed4a..e2bd3accee16cc9e05422e92967c9c2a206ce77e 100644 (file)
@@ -147,6 +147,51 @@ void destinationDone() \{
 \}
 \end{alltt}
 
+The callback methods can also take a pointer to a \texttt{CkDataMsg} message. This message can be
+used to access the original pointer passed into the buffer and another optional reference pointer that was
+initially set for the \kw{CkNcpyBuffer} object using the method \texttt{setRef}. The following code snippet
+illustrates the accessing of the original buffer pointer in the callback method by casting the \texttt{data}
+field of the \texttt{CkDataMsg} object into a \texttt{CkNcpyAck} object.
+
+\begin{alltt}
+// Invoked by the runtime on source (Index 0)
+void sourceDone(CkDataMsg *msg) \{
+    // Cast msg->data to a CkNcpyAck
+    CkNcpyAck *ack = (CkNcpyAck *)(msg->data);
+
+    // access buffer pointer and free it
+    free(ack->ptr);
+\}
+\end{alltt}
+
+The following code snippet illustrates the usage of the \texttt{setRef} method.
+
+\begin{alltt}
+const void *refPtr = &index;
+CkNcpyBuffer source(arr1, arr1Size * sizeof(int), srcCb, CK_BUFFER_UNREG);
+source.setRef(refPtr);
+\end{alltt}
+
+Similar to the buffer pointer, the user set arbitrary reference pointer can be also accessed in the
+callback method. This is shown in the next code snippet.
+
+\begin{alltt}
+// Invoked by the runtime on source (Index 0)
+void sourceDone(CkDataMsg *msg) \{
+    // update the buffer to the next pointer
+    updateBuffer();
+
+    // Cast msg->data to a CkNcpyAck
+    CkNcpyAck *ack = (CkNcpyAck *)(msg->data);
+
+    // access buffer pointer and free it
+    free(ack->ptr);
+
+    // get reference pointer
+    const void *refPtr = ack->ref;
+\}
+\end{alltt}
+
 Both the source and destination buffers are of the same type i.e. \kw{CkNcpyBuffer}.
 What distinguishes a source buffer from a destination buffer is the way the \kw{get} or
 \kw{put} call is made. A valid \kw{get} call using two \kw{CkNcpyBuffer} objects \texttt{obj1} and
index d00aa2168c239482765fdb54172ee5781e8fd811..faf2ae4ce2b6ee20b19ee859a0cbf6e879c22069 100644 (file)
@@ -352,119 +352,86 @@ void LrtsSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short
 
 // Perform an RDMA Get call into the local destination address from the remote source address
 void LrtsIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
-
-  // Remote buffer is registered, perform GET
-  CmiAssert(srcAckSize == destAckSize);
+  unsigned short int *destMode) {
 
   // Register local buffer if it is not registered
   if(*destMode == CMK_BUFFER_UNREG) {
-    ((CmiGNIRzvRdmaPtr_t *)destInfo)->mem_hndl = registerDirectMem(destAddr, size, GNI_MEM_READWRITE);
+    ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl =
+                                      registerDirectMem(ncpyOpInfo->destPtr,
+                                                        ncpyOpInfo->size,
+                                                        GNI_MEM_READWRITE);
     *destMode = CMK_BUFFER_REG;
+
+    // set mem_hndl in the origDestLayerInfoPtr
+    ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->origDestLayerInfoPtr))->mem_hndl =
+                            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl;
   }
 
   if(*srcMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform PUT
 
-    int mdMsgSize = sizeof(CmiGNIRzvRdmaReverseOp_t) + destAckSize + srcAckSize;
-    CmiGNIRzvRdmaReverseOp_t *regAndPutMsg = (CmiGNIRzvRdmaReverseOp_t *)malloc(mdMsgSize);
-
-    regAndPutMsg->destAddr      = destAddr;
-    regAndPutMsg->destPe        = destPe;
-    regAndPutMsg->rem_mem_hndl  = ((CmiGNIRzvRdmaPtr_t *)destInfo)->mem_hndl;
-    regAndPutMsg->destMode      = *destMode;
-
-    regAndPutMsg->srcAddr       = srcAddr;
-    regAndPutMsg->srcPe         = srcPe;
-    regAndPutMsg->srcMode       = *srcMode;
-
-    regAndPutMsg->ackSize       = destAckSize;
-    regAndPutMsg->size          = size;
-
-    memcpy((char*)regAndPutMsg + sizeof(CmiGNIRzvRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndPutMsg + sizeof(CmiGNIRzvRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
-    // send all the data to the source to register and perform a put
 #if CMK_SMP
     // send the small message to the other node through the comm thread
-    buffer_small_msgs(&smsg_queue, regAndPutMsg, mdMsgSize, CmiNodeOf(srcPe), RDMA_REG_AND_PUT_MD_DIRECT_TAG);
+    buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize,
+                          CmiNodeOf(ncpyOpInfo->srcPe),
+                          RDMA_REG_AND_PUT_MD_DIRECT_TAG);
 #else // non-smp mode
     // send the small message directly
-    gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(srcPe), regAndPutMsg, mdMsgSize, RDMA_REG_AND_PUT_MD_DIRECT_TAG, 0, NULL, NONCHARM_SMSG, 1);
+    gni_return_t status = send_smsg_message(&smsg_queue,
+                            CmiNodeOf(ncpyOpInfo->srcPe),
+                            ncpyOpInfo,
+                            ncpyOpInfo->ncpyOpInfoSize,
+                            RDMA_REG_AND_PUT_MD_DIRECT_TAG,
+                            0, NULL, CHARM_SMSG, 1);
     GNI_RC_CHECK("Sending REG & PUT metadata msg failed!", status);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
     if(status == GNI_RC_SUCCESS) {
-      free(regAndPutMsg);
+      CmiFree(ncpyOpInfo);
     }
 #endif // end of !CMK_SMSGS_FREE_AFTER_EVENT
 #endif // end of CMK_SMP
 
   } else {
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
+    // Remote buffer is registered, perform GET
 
-    uint64_t src_addr = (uint64_t)srcAddr;
-    uint64_t dest_addr = (uint64_t)destAddr;
-    uint64_t ref_addr = (uint64_t)ref;
-
-    CmiGNIRzvRdmaPtr_t *src_info = (CmiGNIRzvRdmaPtr_t *)srcInfo;
-    CmiGNIRzvRdmaPtr_t *dest_info = (CmiGNIRzvRdmaPtr_t *)destInfo;
+    uint64_t src_addr = (uint64_t)(ncpyOpInfo->srcPtr);
+    uint64_t dest_addr = (uint64_t)(ncpyOpInfo->destPtr);
+    uint64_t length    = (uint64_t)(ncpyOpInfo->size);
 
     //check alignment as Rget in GNI requires 4 byte alignment for src_addr, dest_adder and size
-    if(((src_addr % 4)==0) && ((dest_addr % 4)==0) && ((size % 4)==0)) {
+    if(((src_addr % 4)==0) && ((dest_addr % 4)==0) && ((length % 4)==0)) {
       // 4-byte aligned, perform GET
 #if CMK_SMP
       // send a message to the comm thread, making it do the GET
-      CmiGNIRzvRdmaDirectInfo_t *getOpInfo = (CmiGNIRzvRdmaDirectInfo_t *)malloc(sizeof(CmiGNIRzvRdmaDirectInfo_t));
-      getOpInfo->dest_mem_hndl = dest_info->mem_hndl;
-      getOpInfo->dest_addr = dest_addr;
-      getOpInfo->src_mem_hndl = src_info->mem_hndl;
-      getOpInfo->src_addr = src_addr;
-      getOpInfo->destPe = CmiMyPe();
-      getOpInfo->size = size;
-      getOpInfo->ref = ref_addr;
-
-      buffer_small_msgs(&smsg_queue, getOpInfo, sizeof(CmiGNIRzvRdmaDirectInfo_t), CmiNodeOf(srcPe), RDMA_COMM_PERFORM_GET_TAG);
+      buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, CmiNodeOf(ncpyOpInfo->srcPe), RDMA_COMM_PERFORM_GET_TAG);
 #else // non-smp mode
       // perform GET directly
-      gni_return_t status = post_rdma(src_addr, src_info->mem_hndl, dest_addr, dest_info->mem_hndl,
-          size, ref_addr, CmiNodeOf(srcPe), GNI_POST_RDMA_GET, DIRECT_SEND_RECV);
+      gni_return_t status = post_rdma(
+                            src_addr,
+                            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl,
+                            dest_addr,
+                            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl,
+                            ncpyOpInfo->size,
+                            (uint64_t)ncpyOpInfo,
+                            CmiNodeOf(ncpyOpInfo->srcPe),
+                            GNI_POST_RDMA_GET,
+                            DIRECT_SEND_RECV);
 #endif // end of !CMK_SMP
 
     } else {
-      // not 4-byte aligned, send md for performing PUT from other node
-      // Allocate machine specific metadata
-      CmiGNIRzvRdmaDirectInfo_t *putOpInfo = (CmiGNIRzvRdmaDirectInfo_t *)malloc(sizeof(CmiGNIRzvRdmaDirectInfo_t));
-      putOpInfo->dest_mem_hndl = dest_info->mem_hndl;
-      putOpInfo->dest_addr = dest_addr;
-      putOpInfo->src_mem_hndl = src_info->mem_hndl;
-      putOpInfo->src_addr = src_addr;
-      putOpInfo->destPe = CmiMyPe();
-      putOpInfo->size = size;
-      putOpInfo->ref = ref_addr;
-
       // send all the data to the source to perform a put
 #if CMK_SMP
       // send the small message to the other node through the comm thread
-      buffer_small_msgs(&smsg_queue, putOpInfo, sizeof(CmiGNIRzvRdmaDirectInfo_t), CmiNodeOf(srcPe), RDMA_PUT_MD_DIRECT_TAG);
+      buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, CmiNodeOf(ncpyOpInfo->srcPe), RDMA_PUT_MD_DIRECT_TAG);
 #else // nonsmp mode
       // send the small message directly
-      gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(srcPe), putOpInfo, sizeof(CmiGNIRzvRdmaDirectInfo_t), RDMA_PUT_MD_DIRECT_TAG, 0, NULL, NONCHARM_SMSG, 1);
+      gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(ncpyOpInfo->srcPe), ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, RDMA_PUT_MD_DIRECT_TAG, 0, NULL, CHARM_SMSG, 1);
       GNI_RC_CHECK("Sending PUT metadata msg failed!", status);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
       if(status == GNI_RC_SUCCESS) {
-        free(putOpInfo);
+        CmiFree(ncpyOpInfo);
       }
 #endif // end of !CMK_SMSGS_FREE_AFTER_EVENT
 #endif // end of CMK_SMP
@@ -474,91 +441,68 @@ void LrtsIssueRget(
 
 // Perform an RDMA Put call into the remote destination address from the local source address
 void LrtsIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size) {
+  unsigned short int *destMode) {
 
   // Register local buffer if it is not registered
   if(*srcMode == CMK_BUFFER_UNREG) {
-    ((CmiGNIRzvRdmaPtr_t *)srcInfo)->mem_hndl = registerDirectMem(srcAddr, size, GNI_MEM_READ_ONLY);
+    ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl =
+                                      registerDirectMem(ncpyOpInfo->srcPtr,
+                                                        ncpyOpInfo->size,
+                                                        GNI_MEM_READ_ONLY);
     *srcMode = CMK_BUFFER_REG;
+
+    // set mem_hndl in the origSrcLayerInfoPtr
+    ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->origSrcLayerInfoPtr))->mem_hndl =
+                            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl;
   }
 
   if(*destMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform GET
 
-    int mdMsgSize = sizeof(CmiGNIRzvRdmaReverseOp_t) + destAckSize + srcAckSize;
-    CmiGNIRzvRdmaReverseOp_t *regAndGetMsg = (CmiGNIRzvRdmaReverseOp_t *)malloc(mdMsgSize);
-
-    regAndGetMsg->srcAddr       = srcAddr;
-    regAndGetMsg->srcPe         = srcPe;
-    regAndGetMsg->rem_mem_hndl  = ((CmiGNIRzvRdmaPtr_t *)srcInfo)->mem_hndl;
-    regAndGetMsg->srcMode       = *srcMode;
-
-    regAndGetMsg->destAddr      = destAddr;
-    regAndGetMsg->destPe        = destPe;
-    regAndGetMsg->destMode      = *destMode;
-
-    regAndGetMsg->ackSize       = srcAckSize;
-    regAndGetMsg->size          = size;
-
-    memcpy((char*)regAndGetMsg + sizeof(CmiGNIRzvRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndGetMsg + sizeof(CmiGNIRzvRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
     // send all the data to the source to register and perform a get
 #if CMK_SMP
     // send the small message to the other node through the comm thread
-    buffer_small_msgs(&smsg_queue, regAndGetMsg, mdMsgSize, CmiNodeOf(destPe), RDMA_REG_AND_GET_MD_DIRECT_TAG);
+    buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize,
+                      CmiNodeOf(ncpyOpInfo->destPe),
+                      RDMA_REG_AND_GET_MD_DIRECT_TAG);
 #else // non-smp mode
     // send the small message directly
-    gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(destPe), regAndGetMsg, mdMsgSize, RDMA_REG_AND_GET_MD_DIRECT_TAG, 0, NULL, NONCHARM_SMSG, 1);
+    gni_return_t status = send_smsg_message(&smsg_queue,
+                              CmiNodeOf(ncpyOpInfo->destPe),
+                              ncpyOpInfo,
+                              ncpyOpInfo->ncpyOpInfoSize,
+                              RDMA_REG_AND_GET_MD_DIRECT_TAG,
+                              0, NULL, CHARM_SMSG, 1);
     GNI_RC_CHECK("Sending REG & GET metadata msg failed!", status);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
     if(status == GNI_RC_SUCCESS) {
-      free(regAndGetMsg);
+      CmiFree(ncpyOpInfo);
     }
 #endif // end of !CMK_SMSGS_FREE_AFTER_EVENT
 #endif // end of CMK_SMP
 
   } else {
-
     // Remote buffer is registered, perform PUT
-    CmiAssert(srcAckSize == destAckSize);
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
-
-    uint64_t dest_addr = (uint64_t)destAddr;
-    uint64_t src_addr = (uint64_t)srcAddr;
-    uint64_t ref_addr = (uint64_t)ref;
 
-    CmiGNIRzvRdmaPtr_t *dest_info = (CmiGNIRzvRdmaPtr_t *)destInfo;
-    CmiGNIRzvRdmaPtr_t *src_info = (CmiGNIRzvRdmaPtr_t *)srcInfo;
+    uint64_t dest_addr = (uint64_t)(ncpyOpInfo->destPtr);
+    uint64_t src_addr = (uint64_t)(ncpyOpInfo->srcPtr);
 
 #if CMK_SMP
     // send a message to the comm thread, making it do the PUT
-    CmiGNIRzvRdmaDirectInfo_t *putOpInfo = (CmiGNIRzvRdmaDirectInfo_t *)malloc(sizeof(CmiGNIRzvRdmaDirectInfo_t));
-    putOpInfo->dest_mem_hndl = dest_info->mem_hndl;
-    putOpInfo->dest_addr = dest_addr;
-    putOpInfo->src_mem_hndl = src_info->mem_hndl;
-    putOpInfo->src_addr = src_addr;
-    putOpInfo->destPe = CmiMyPe();
-    putOpInfo->size = size;
-    putOpInfo->ref = ref_addr;
-
-    buffer_small_msgs(&smsg_queue, putOpInfo, sizeof(CmiGNIRzvRdmaDirectInfo_t), CmiNodeOf(destPe), RDMA_COMM_PERFORM_PUT_TAG);
+    buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, CmiNodeOf(ncpyOpInfo->destPe), RDMA_COMM_PERFORM_PUT_TAG);
 #else // nonsmp mode
     // perform PUT directly
-    gni_return_t status = post_rdma(dest_addr, dest_info->mem_hndl, src_addr, src_info->mem_hndl,
-        size, ref_addr, CmiNodeOf(destPe), GNI_POST_RDMA_PUT, DIRECT_SEND_RECV);
+    gni_return_t status = post_rdma(dest_addr,
+                          ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl,
+                          src_addr,
+                          ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl,
+                          ncpyOpInfo->size,
+                          (uint64_t)ncpyOpInfo,
+                          CmiNodeOf(ncpyOpInfo->destPe),
+                          GNI_POST_RDMA_PUT,
+                          DIRECT_SEND_RECV);
 #endif // end of !CMK_SMP
   }
 }
@@ -591,31 +535,29 @@ void LrtsDeregisterMem(const void *ptr, void *info, int pe, unsigned short int m
 #if CMK_SMP
 // Method used by the comm thread to perform GET - called from SendBufferMsg
 void _performOneRgetForWorkerThread(MSG_LIST *ptr) {
-  CmiGNIRzvRdmaDirectInfo_t *getOpInfo = (CmiGNIRzvRdmaDirectInfo_t *)(ptr->msg);
-  post_rdma(getOpInfo->src_addr,
-            getOpInfo->src_mem_hndl,
-            getOpInfo->dest_addr,
-            getOpInfo->dest_mem_hndl,
-            getOpInfo->size,
-            getOpInfo->ref,
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
+  post_rdma((uint64_t)ncpyOpInfo->srcPtr,
+            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl,
+            (uint64_t)ncpyOpInfo->destPtr,
+            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl,
+            ncpyOpInfo->size,
+            (uint64_t)ncpyOpInfo,
             ptr->destNode,
             GNI_POST_RDMA_GET,
             DIRECT_SEND_RECV);
-  free(getOpInfo);
 }
 
 // Method used by the comm thread to perform PUT - called from SendBufferMsg
 void _performOneRputForWorkerThread(MSG_LIST *ptr) {
-  CmiGNIRzvRdmaDirectInfo_t *putOpInfo = (CmiGNIRzvRdmaDirectInfo_t *)(ptr->msg);
-  post_rdma(putOpInfo->dest_addr,
-            putOpInfo->dest_mem_hndl,
-            putOpInfo->src_addr,
-            putOpInfo->src_mem_hndl,
-            putOpInfo->size,
-            putOpInfo->ref,
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
+  post_rdma((uint64_t)ncpyOpInfo->destPtr,
+            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->mem_hndl,
+            (uint64_t)ncpyOpInfo->srcPtr,
+            ((CmiGNIRzvRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->mem_hndl,
+            ncpyOpInfo->size,
+            (uint64_t)ncpyOpInfo,
             ptr->destNode,
             GNI_POST_RDMA_PUT,
             DIRECT_SEND_RECV);
-  free(putOpInfo);
 }
 #endif
index d803b0031dd5e45e4beeea71761c36aa253ce18c..c2b418077f0584fe17af7c6e3d32d6776e0abceb 100644 (file)
@@ -2356,26 +2356,26 @@ static void PumpNetworkSmsg()
             }
             case RDMA_PUT_MD_DIRECT_TAG:
             {
-                // Direct API when PUT is used instead of a GET
-                // This tag implies the receival of a PUT metadata used for the Direct API
-                CmiGNIRzvRdmaDirectInfo_t *putOp = (CmiGNIRzvRdmaDirectInfo_t *)header;
-                CmiGNIRzvRdmaDirectInfo_t *newPutOp = (CmiGNIRzvRdmaDirectInfo_t *)malloc(sizeof(CmiGNIRzvRdmaDirectInfo_t));
-                memcpy(newPutOp, putOp, sizeof(CmiGNIRzvRdmaDirectInfo_t));
+                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)header;
+
+                // copy into a new object
+                NcpyOperationInfo *newNcpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyOpInfo->ncpyOpInfoSize);
+                memcpy(newNcpyOpInfo, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize);
+
                 GNI_SmsgRelease(ep_hndl_array[inst_id]);
                 CMI_GNI_UNLOCK(smsg_mailbox_lock)
 
-                // Issue PUT
-                status = post_rdma(
-                         newPutOp->dest_addr,
-                         newPutOp->dest_mem_hndl,
-                         newPutOp->src_addr,
-                         newPutOp->src_mem_hndl,
-                         newPutOp->size,
-                         (uint64_t)newPutOp,
-                         CmiNodeOf(newPutOp->destPe),
-                         GNI_POST_RDMA_PUT,
-                         DIRECT_SEND_RECV_UNALIGNED);
-                 break;
+                post_rdma((uint64_t)newNcpyOpInfo->destPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->destLayerInfo))->mem_hndl,
+                          (uint64_t)newNcpyOpInfo->srcPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->srcLayerInfo))->mem_hndl,
+                          newNcpyOpInfo->size,
+                          (uint64_t)newNcpyOpInfo,
+                          CmiNodeOf(newNcpyOpInfo->destPe),
+                          GNI_POST_RDMA_PUT,
+                          DIRECT_SEND_RECV);
+
+                break;
             }
             case RDMA_PUT_DONE_TAG:
             {
@@ -2432,59 +2432,64 @@ static void PumpNetworkSmsg()
             }
             case RDMA_REG_AND_PUT_MD_DIRECT_TAG:
             {
-                CmiGNIRzvRdmaReverseOp_t *revOp = (CmiGNIRzvRdmaReverseOp_t *)header;
+                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)header;
+
+                // copy into a new object
+                NcpyOperationInfo *newNcpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyOpInfo->ncpyOpInfoSize);
+                memcpy(newNcpyOpInfo, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize);
+
+                GNI_SmsgRelease(ep_hndl_array[inst_id]);
+                CMI_GNI_UNLOCK(smsg_mailbox_lock)
+
+                resetNcpyOpInfoPointers(newNcpyOpInfo);
+
                 // Register source buffer
-                gni_mem_handle_t src_mem_hndl = registerDirectMem(revOp->srcAddr, revOp->size, GNI_MEM_READ_ONLY);
-
-                // Perform PUT
-                void *ref = CmiGetNcpyAck(revOp->srcAddr,
-                                          (char *)revOp + sizeof(CmiGNIRzvRdmaReverseOp_t),
-                                          revOp->srcPe,
-                                          revOp->destAddr,
-                                          (char *)revOp + sizeof(CmiGNIRzvRdmaReverseOp_t) + revOp->ackSize,
-                                          revOp->destPe,
-                                          revOp->ackSize);
-                post_rdma((uint64_t)revOp->destAddr,
-                          revOp->rem_mem_hndl,
-                          (uint64_t)revOp->srcAddr,
-                          src_mem_hndl,
-                          revOp->size,
-                          (uint64_t)ref,
-                          CmiNodeOf(revOp->destPe),
+                ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->srcLayerInfo))->mem_hndl =
+                                              registerDirectMem(newNcpyOpInfo->srcPtr,
+                                                                newNcpyOpInfo->size,
+                                                                GNI_MEM_READ_ONLY);
+
+                post_rdma((uint64_t)newNcpyOpInfo->destPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->destLayerInfo))->mem_hndl,
+                          (uint64_t)newNcpyOpInfo->srcPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->srcLayerInfo))->mem_hndl,
+                          newNcpyOpInfo->size,
+                          (uint64_t)newNcpyOpInfo,
+                          CmiNodeOf(newNcpyOpInfo->destPe),
                           GNI_POST_RDMA_PUT,
                           DIRECT_SEND_RECV);
 
-                GNI_SmsgRelease(ep_hndl_array[inst_id]);
-                CMI_GNI_UNLOCK(smsg_mailbox_lock)
                 break;
             }
             case RDMA_REG_AND_GET_MD_DIRECT_TAG:
             {
-                CmiGNIRzvRdmaReverseOp_t *revOp = (CmiGNIRzvRdmaReverseOp_t *)header;
-                // Register destination buffer
-                gni_mem_handle_t dest_mem_hndl = registerDirectMem(revOp->destAddr, revOp->size, GNI_MEM_READWRITE);
-
-                // Perform PUT
-                void *ref = CmiGetNcpyAck(revOp->srcAddr,
-                                          (char *)revOp + sizeof(CmiGNIRzvRdmaReverseOp_t),
-                                          revOp->srcPe,
-                                          revOp->destAddr,
-                                          (char *)revOp + sizeof(CmiGNIRzvRdmaReverseOp_t) + revOp->ackSize,
-                                          revOp->destPe,
-                                          revOp->ackSize);
-                post_rdma((uint64_t)revOp->srcAddr,
-                          revOp->rem_mem_hndl,
-                          (uint64_t)revOp->destAddr,
-                          dest_mem_hndl,
-                          revOp->size,
-                          (uint64_t)ref,
-                          CmiNodeOf(revOp->srcPe),
-                          GNI_POST_RDMA_GET,
-                          DIRECT_SEND_RECV);
+                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)header;
+
+                // copy into a new object
+                NcpyOperationInfo *newNcpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyOpInfo->ncpyOpInfoSize);
+                memcpy(newNcpyOpInfo, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize);
 
                 GNI_SmsgRelease(ep_hndl_array[inst_id]);
                 CMI_GNI_UNLOCK(smsg_mailbox_lock)
-                break;
+
+                resetNcpyOpInfoPointers(newNcpyOpInfo);
+
+                ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->destLayerInfo))->mem_hndl =
+                                              registerDirectMem(newNcpyOpInfo->destPtr,
+                                                                newNcpyOpInfo->size,
+                                                                GNI_MEM_READWRITE);
+
+                post_rdma((uint64_t)newNcpyOpInfo->srcPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->srcLayerInfo))->mem_hndl,
+                          (uint64_t)newNcpyOpInfo->destPtr,
+                          ((CmiGNIRzvRdmaPtr_t *)(newNcpyOpInfo->destLayerInfo))->mem_hndl,
+                          newNcpyOpInfo->size,
+                          (uint64_t)newNcpyOpInfo,
+                          CmiNodeOf(newNcpyOpInfo->srcPe),
+                          GNI_POST_RDMA_GET,
+                          DIRECT_SEND_RECV);
+
+                 break;
             }
 #endif
             case BIG_MSG_TAG:  //big msg, de-register, transfer next seg
@@ -3367,6 +3372,7 @@ INLINE_KEYWORD gni_return_t _sendOneBufferedSmsg(SMSG_QUEUE *queue, MSG_LIST *pt
     CONTROL_MSG         *control_msg_tmp;
     gni_return_t        status = GNI_RC_ERROR_RESOURCE;
     int                 numRdmaOps, recvInfoSize, msgSize;
+    NcpyOperationInfo *ncpyOpInfo;
 
     MACHSTATE5(8, "noempty-smsg  %d (%d,%d,%d) tag=%d \n", ptr->destNode, buffered_send_msg, buffered_recv_msg, register_memory_size, ptr->tag); 
     if (useDynamicSMSG && smsg_connected_flag[ptr->destNode] != 2) {   
@@ -3461,21 +3467,23 @@ INLINE_KEYWORD gni_return_t _sendOneBufferedSmsg(SMSG_QUEUE *queue, MSG_LIST *pt
         break;
 
      case RDMA_PUT_MD_DIRECT_TAG:
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, sizeof(CmiGNIRzvRdmaDirectInfo_t), ptr->tag, 1, ptr, NONCHARM_SMSG, 1);
+        ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
+        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, CHARM_SMSG, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
         if(status == GNI_RC_SUCCESS) {
-          free(ptr->msg);
+          CmiFree(ptr->msg);
         }
 #endif
         break;
 
      case RDMA_REG_AND_GET_MD_DIRECT_TAG:
      case RDMA_REG_AND_PUT_MD_DIRECT_TAG:
-        msgSize = sizeof(CmiGNIRzvRdmaReverseOp_t) + 2*(((CmiGNIRzvRdmaReverseOp_t *)(ptr->msg))->ackSize);
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, msgSize, ptr->tag, 1, ptr, NONCHARM_SMSG, 1);
+        //msgSize = sizeof(CmiGNIRzvRdmaReverseOp_t) + 2*(((CmiGNIRzvRdmaReverseOp_t *)(ptr->msg))->ackSize);
+        ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
+        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, CHARM_SMSG, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
         if(status == GNI_RC_SUCCESS) {
-          free(ptr->msg);
+          CmiFree(ptr->msg);
         }
 #endif
         break;
index bd65a44027fd46fbdbcdd24c1681f8de54662411..ca2ba800f575926a57358448a6b759eabdf144dd 100644 (file)
@@ -92,141 +92,77 @@ void MPIPostOneBuffer(const void *buffer, void *ref, int size, int pe, int tag,
 
 // Perform an RDMA Get call into the local destination address from the remote source address
 void LrtsIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfoMsg,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
+  unsigned short int *destMode) {
 
   // Generate a new tag
   int tag = getNewMPITag();
   SMSG_LIST *msg_tmp;
   MPI_Request reqBufferRecv;
 
-  // Send a message to the source with a tag to have the source post an MPI_ISend
-  int postInfoMsgSize = CmiMsgHeaderSizeBytes + sizeof(CmiMPIRzvRdmaPostInfo_t) + srcAckSize;
-  char *postInfoMsg = (char *)CmiAlloc(postInfoMsgSize);
-
-  CmiMPIRzvRdmaPostInfo_t *postInfo = (CmiMPIRzvRdmaPostInfo_t *)(postInfoMsg + CmiMsgHeaderSizeBytes);
-  postInfo->buffer = (void *)srcAddr;
-  postInfo->size = size;
-  postInfo->tag = tag;
-  postInfo->ackSize = srcAckSize;
-  postInfo->destPe = destPe;
-  postInfo->srcPe = srcPe;
-
-  // Copy the source ack so that the remote source can invoke it after completion
-  memcpy((char *)(postInfo) + sizeof(CmiMPIRzvRdmaPostInfo_t),
-         srcAck,
-         srcAckSize);
-
   // Mark the message type as POST_DIRECT_SEND
   // On receiving a POST_DIRECT_SEND, the MPI rank should post an MPI_Isend
-  CMI_MSGTYPE(postInfoMsg) = POST_DIRECT_SEND;
+  CMI_MSGTYPE(ncpyOpInfoMsg) = POST_DIRECT_SEND;
+
+  // Send the tag for the receiver MPI rank to post an MPI_Isend
+  ncpyOpInfoMsg->tag = tag;
 
   // Determine the remote rank
-  int destLocalNode = CmiNodeOf(srcPe);
+  int destLocalNode = CmiNodeOf(ncpyOpInfoMsg->srcPe);
   int destRank = CmiGetNodeGlobal(destLocalNode, CmiMyPartition());
 
 #if CMK_SMP
   if (Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV) {
-    EnqueueMsg(postInfoMsg, postInfoMsgSize, destRank, 0, POST_DIRECT_SEND, NULL);
+    EnqueueMsg(ncpyOpInfoMsg, ncpyOpInfoMsg->ncpyOpInfoSize, destRank, 0, POST_DIRECT_SEND, NULL);
   }
   else
 #endif
   {
-    msg_tmp = allocateSmsgList(postInfoMsg, destRank, postInfoMsgSize, 0, POST_DIRECT_SEND, NULL);
+    msg_tmp = allocateSmsgList((char *)ncpyOpInfoMsg, destRank, ncpyOpInfoMsg->ncpyOpInfoSize, 0, POST_DIRECT_SEND, NULL);
     MPISendOneMsg(msg_tmp);
   }
 
-  // Create a local object to invoke the acknowledgement on completion of the MPI_Irecv
-  CmiMPIRzvRdmaAckInfo_t *destAckNew = (CmiMPIRzvRdmaAckInfo_t *)malloc(sizeof(CmiMPIRzvRdmaAckInfo_t) + destAckSize);
-  destAckNew->pe = destPe;
-  destAckNew->tag = tag;
-  memcpy((char *)destAckNew + sizeof(CmiMPIRzvRdmaAckInfo_t),
-         destAck,
-         destAckSize);
-
   // Post an MPI_Irecv for the destination buffer with the tag
   // ONESIDED_BUFFER_DIRECT_RECV indicates that the method should post an irecv
-  MPIPostOneBuffer(destAddr, destAckNew, size, srcPe, tag, ONESIDED_BUFFER_DIRECT_RECV);
+  MPIPostOneBuffer(ncpyOpInfoMsg->destPtr, ncpyOpInfoMsg, ncpyOpInfoMsg->size, ncpyOpInfoMsg->srcPe, tag, ONESIDED_BUFFER_DIRECT_RECV);
 }
 
 // Perform an RDMA Put call into the remote destination address from the local source address
 void LrtsIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfoMsg,
   unsigned short int *srcMode,
-  int size) {
+  unsigned short int *destMode) {
 
   // Generate a new tag
   int tag = getNewMPITag();
   SMSG_LIST *msg_tmp;
 
-  // Send a message to the destination with a tag to have the destination post a MPI_Irecv
-  int postInfoMsgSize = CmiMsgHeaderSizeBytes + sizeof(CmiMPIRzvRdmaPostInfo_t) + destAckSize;
-  char *postInfoMsg = (char *)CmiAlloc(postInfoMsgSize);
-
-  CmiMPIRzvRdmaPostInfo_t *postInfo = (CmiMPIRzvRdmaPostInfo_t *)(postInfoMsg + CmiMsgHeaderSizeBytes);
-  postInfo->buffer = (void *)destAddr;
-  postInfo->size = size;
-  postInfo->tag = tag;
-  postInfo->ackSize = destAckSize;
-  postInfo->srcPe = srcPe;
-  postInfo->destPe = destPe;
-
-  // Copy the destination ack so that the remote destination can invoke it after completion
-  memcpy((char *)(postInfo) + sizeof(CmiMPIRzvRdmaPostInfo_t),
-         destAck,
-         destAckSize);
-
   // Mark the message type as POST_DIRECT_RECV
   // On receiving a POST_DIRECT_RECV, the MPI rank should post an MPI_Irecv
-  CMI_MSGTYPE(postInfoMsg) = POST_DIRECT_RECV;
+  CMI_MSGTYPE(ncpyOpInfoMsg) = POST_DIRECT_RECV;
+
+  // Send the tag for the receiver MPI rank to post an MPI_Irecv
+  ncpyOpInfoMsg->tag = tag;
 
   // Determine the remote rank
-  int destLocalNode = CmiNodeOf(destPe);
+  int destLocalNode = CmiNodeOf(ncpyOpInfoMsg->destPe);
   int destRank = CmiGetNodeGlobal(destLocalNode, CmiMyPartition());
 
 #if CMK_SMP
   if (Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV) {
-    EnqueueMsg(postInfoMsg, postInfoMsgSize, destRank, 0, POST_DIRECT_RECV, NULL);
+    EnqueueMsg(ncpyOpInfoMsg, ncpyOpInfoMsg->ncpyOpInfoSize, destRank, 0, POST_DIRECT_RECV, NULL);
   }
   else
 #endif
   {
-    msg_tmp = allocateSmsgList(postInfoMsg, destRank, postInfoMsgSize, 0, POST_DIRECT_RECV, NULL);
+    msg_tmp = allocateSmsgList((char *)ncpyOpInfoMsg, destRank, ncpyOpInfoMsg->ncpyOpInfoSize, 0, POST_DIRECT_RECV, NULL);
     MPISendOneMsg(msg_tmp);
   }
 
-  // Create a local object to invoke the acknowledgement on completion of the MPI_Isend
-  CmiMPIRzvRdmaAckInfo_t *srcAckNew = (CmiMPIRzvRdmaAckInfo_t *)malloc(sizeof(CmiMPIRzvRdmaAckInfo_t) + srcAckSize);
-  srcAckNew->pe = srcPe;
-  srcAckNew->tag = tag;
-  memcpy((char *)srcAckNew + sizeof(CmiMPIRzvRdmaAckInfo_t),
-         srcAck,
-         srcAckSize);
-
   // Post an MPI_ISend for the source buffer with the tag
   // ONESIDED_BUFFER_DIRECT_SEND indicates that the method should post an isend
-  MPIPostOneBuffer(srcAddr, (void *)srcAckNew, size, destPe, tag, ONESIDED_BUFFER_DIRECT_SEND);
+  MPIPostOneBuffer(ncpyOpInfoMsg->srcPtr, ncpyOpInfoMsg, ncpyOpInfoMsg->size, ncpyOpInfoMsg->destPe, tag, ONESIDED_BUFFER_DIRECT_SEND);
 }
 
 // Method invoked to deregister source memory (Empty method to maintain API consistency)
index 7402c320f00e254ea0a2f302b30bb55f96a2d96c..85d7016270eb6f8d52241d9b99042d6ca916d86a 100644 (file)
@@ -639,18 +639,25 @@ static void ReleasePostedMessages(void) {
 
                 //free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
                 free(ack);
-            } else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV || msg_tmp->type == ONESIDED_BUFFER_DIRECT_SEND) {
-                // Get the ack information
-                CmiMPIRzvRdmaAckInfo_t *ack = (CmiMPIRzvRdmaAckInfo_t *)(msg_tmp->ref);
-
-                // Call Ack function
-                ncpyAckHandlerFn(
-                                       (char *)msg_tmp->ref + sizeof(CmiMPIRzvRdmaAckInfo_t),
-                                       ack->pe,
-                                       msg_tmp->msg);
+            } else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV) {
+                // MPI_Irecv posted as a part of the Direct API was completed
+                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(msg_tmp->ref);
+                // Invoke the destination ack
+                ncpyOpInfo->ackMode = 2;
+                CmiInvokeNcpyAck(ncpyOpInfo);
+
+            } else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_SEND) {
+                // MPI_Isend posted as a part of the Direct API was completed
+                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(msg_tmp->ref);
+                // Invoke the source ack
+                ncpyOpInfo->ackMode = 1;
+                CmiInvokeNcpyAck(ncpyOpInfo);
 
-                // free the ack information
-                free(ack);
+            }
+            else if(msg_tmp->type == POST_DIRECT_SEND || msg_tmp->type == POST_DIRECT_RECV) {
+                // do nothing as the received message is a NcpyOperationInfo object
+                // which is freed in the above code (either ONESIDED_BUFFER_DIRECT_RECV or
+                // ONESIDED_BUFFER_DIRECT_SEND)
             }
             else
 #endif
@@ -879,35 +886,32 @@ static int PumpMsgs(void) {
             if(CMI_MSGTYPE(msg) == REGULAR) {
               handleOneRecvedMsg(nbytes, msg);
             } else if(CMI_MSGTYPE(msg) == POST_DIRECT_RECV || CMI_MSGTYPE(msg) == POST_DIRECT_SEND) {
-              CmiMPIRzvRdmaPostInfo_t *postInfo = (CmiMPIRzvRdmaPostInfo_t *)(msg + CmiMsgHeaderSizeBytes);
 
-              // Allocate a local ack
-              CmiMPIRzvRdmaAckInfo_t *ackNew = (CmiMPIRzvRdmaAckInfo_t *)malloc(sizeof(CmiMPIRzvRdmaAckInfo_t) + postInfo->ackSize);
-              ackNew->tag = postInfo->tag;
-              memcpy((char *)ackNew + sizeof(CmiMPIRzvRdmaAckInfo_t),
-                     (char *)postInfo + sizeof(CmiMPIRzvRdmaPostInfo_t),
-                     postInfo->ackSize);
+              NcpyOperationInfo *ncpyOpInfoMsg = (NcpyOperationInfo *)msg;
+              resetNcpyOpInfoPointers(ncpyOpInfoMsg);
 
               int postMsgType, myPe, otherPe;
+              const void *myBuffer;
               if(CMI_MSGTYPE(msg) == POST_DIRECT_RECV) {
                 // Direct Buffer destination, post MPI_Irecv
                 postMsgType = ONESIDED_BUFFER_DIRECT_RECV;
-                myPe = postInfo->destPe;
-                otherPe = postInfo->srcPe;
+                myPe = ncpyOpInfoMsg->destPe;
+                otherPe = ncpyOpInfoMsg->srcPe;
+                myBuffer = ncpyOpInfoMsg->destPtr;
               }
               else {
                 // Direct Buffer Source, post MPI_Isend
                 postMsgType = ONESIDED_BUFFER_DIRECT_SEND;
-                myPe = postInfo->srcPe;
-                otherPe = postInfo->destPe;
+                myPe = ncpyOpInfoMsg->srcPe;
+                otherPe = ncpyOpInfoMsg->destPe;
+                myBuffer = ncpyOpInfoMsg->srcPtr;
               }
 
-              ackNew->pe = myPe;
-              MPIPostOneBuffer(postInfo->buffer,
-                               (void *)ackNew,
-                               postInfo->size,
+              MPIPostOneBuffer(myBuffer,
+                               ncpyOpInfoMsg,
+                               ncpyOpInfoMsg->size,
                                otherPe,
-                               postInfo->tag,
+                               ncpyOpInfoMsg->tag,
                                postMsgType);
 
             } else {
index 9709d406d1cf74294d594b7dc04fd6c72b8ed41c..033d6d31fb64d2203bce6d3fc36ab68e3ea24ba8 100644 (file)
@@ -221,29 +221,32 @@ static inline void ofi_onesided_direct_operation_callback(struct fi_cq_tagged_en
 }
 
 void process_onesided_reg_and_put(struct fi_cq_tagged_entry *e, OFIRequest *req) {
-  CmiOfiRdmaReverseOp_t *regAndPutMsg = (CmiOfiRdmaReverseOp_t *)(req->data.rma_ncpy_ack);
-  struct fid_mr *mr = registerDirectMemory(regAndPutMsg->srcAddr, regAndPutMsg->size);
-  void *ref = CmiGetNcpyAck(regAndPutMsg->srcAddr,
-                           (char *)regAndPutMsg + sizeof(CmiOfiRdmaReverseOp_t)+ regAndPutMsg->ackSize, //srcAck
-                           regAndPutMsg->srcPe,
-                           regAndPutMsg->destAddr,
-                           (char *)regAndPutMsg + sizeof(CmiOfiRdmaReverseOp_t), //destAck
-                           regAndPutMsg->destPe,
-                           regAndPutMsg->ackSize);
-
-  const char *rbuf  = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)regAndPutMsg->destAddr;
+
+  char *data = (char *)req->data.rma_ncpy_ack;
+
+  // Allocate a new receiver buffer to receive other messages
+  req->data.recv_buffer = CmiAlloc(context.eager_maxsize);
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(data);
+  resetNcpyOpInfoPointers(ncpyOpInfo);
+
+  // Do not free as this message
+  ncpyOpInfo->freeMe = 0;
+
+  struct fid_mr *mr = registerDirectMemory(ncpyOpInfo->srcPtr, ncpyOpInfo->size);
+  const char *rbuf  = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)(ncpyOpInfo->destPtr);
 
   // Allocate a completion object for tracking write completion and ack handling
   CmiOfiRdmaComp_t* rdmaComp = (CmiOfiRdmaComp_t *)malloc(sizeof(CmiOfiRdmaComp_t));
-  rdmaComp->ack_info         = ref;
+  rdmaComp->ack_info         = ncpyOpInfo;
   rdmaComp->completion_count = 0;
 
   ofi_post_nocopy_operation(
-      (char*)regAndPutMsg->srcAddr,
+      (char*)(ncpyOpInfo->srcPtr),
       rbuf,
-      CmiNodeOf(regAndPutMsg->destPe),
-      regAndPutMsg->rem_key,
-      regAndPutMsg->size,
+      CmiNodeOf(ncpyOpInfo->destPe),
+      ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->destLayerInfo))->key,
+      ncpyOpInfo->size,
       mr,
       ofi_onesided_direct_operation_callback,
       (void *)rdmaComp,
@@ -252,29 +255,29 @@ void process_onesided_reg_and_put(struct fi_cq_tagged_entry *e, OFIRequest *req)
 }
 
 void process_onesided_reg_and_get(struct fi_cq_tagged_entry *e, OFIRequest *req) {
-  CmiOfiRdmaReverseOp_t *regAndGetMsg = (CmiOfiRdmaReverseOp_t *)(req->data.rma_ncpy_ack);
-  struct fid_mr *mr = registerDirectMemory(regAndGetMsg->destAddr, regAndGetMsg->size);
-  void *ref = CmiGetNcpyAck(regAndGetMsg->srcAddr,
-                           (char *)regAndGetMsg + sizeof(CmiOfiRdmaReverseOp_t)+ regAndGetMsg->ackSize, //srcAck
-                           regAndGetMsg->srcPe,
-                           regAndGetMsg->destAddr,
-                           (char *)regAndGetMsg + sizeof(CmiOfiRdmaReverseOp_t), //destAck
-                           regAndGetMsg->destPe,
-                           regAndGetMsg->ackSize);
-
-  const char *rbuf  = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)regAndGetMsg->srcAddr;
+
+  char *data = (char *)req->data.rma_ncpy_ack;
+
+  // Allocate a new receiver buffer to receive other messages
+  req->data.recv_buffer = CmiAlloc(context.eager_maxsize);
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(data);
+  resetNcpyOpInfoPointers(ncpyOpInfo);
+
+  struct fid_mr *mr = registerDirectMemory(ncpyOpInfo->destPtr, ncpyOpInfo->size);
+  const char *rbuf  = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)(ncpyOpInfo->srcPtr);
 
   // Allocate a completion object for tracking write completion and ack handling
   CmiOfiRdmaComp_t* rdmaComp = (CmiOfiRdmaComp_t *)malloc(sizeof(CmiOfiRdmaComp_t));
-  rdmaComp->ack_info         = ref;
+  rdmaComp->ack_info         = ncpyOpInfo;
   rdmaComp->completion_count = 0;
 
   ofi_post_nocopy_operation(
-      (char*)regAndGetMsg->destAddr,
+      (char*)(ncpyOpInfo->destPtr),
       rbuf,
-      CmiNodeOf(regAndGetMsg->srcPe),
-      regAndGetMsg->rem_key,
-      regAndGetMsg->size,
+      CmiNodeOf(ncpyOpInfo->srcPe),
+      ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo))->key,
+      ncpyOpInfo->size,
       mr,
       ofi_onesided_direct_operation_callback,
       (void *)rdmaComp,
@@ -285,49 +288,26 @@ void process_onesided_reg_and_get(struct fi_cq_tagged_entry *e, OFIRequest *req)
 
 // Perform an RDMA Get call into the local destination address from the remote source address
 void LrtsIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
+  unsigned short int *destMode) {
 
   OFIRequest *req;
-  CmiAssert(destAckSize == srcAckSize);
 
   // Register local buffer if it is not registered
   if(*destMode == CMK_BUFFER_UNREG) {
-    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)destInfo;
-    dest_info->mr = registerDirectMemory(destAddr, size);
+    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
+    dest_info->mr = registerDirectMemory(ncpyOpInfo->destPtr, ncpyOpInfo->size);
     dest_info->key = fi_mr_key(dest_info->mr);
     *destMode = CMK_BUFFER_REG;
+
+    // set registration info in the origDestLayerInfoPtr
+    ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->origDestLayerInfoPtr))->mr = dest_info->mr;
+    ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->origDestLayerInfoPtr))->key = dest_info->key;
   }
 
   if(*srcMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform PUT
-    int mdMsgSize = sizeof(CmiOfiRdmaReverseOp_t) + destAckSize + srcAckSize;
-    CmiOfiRdmaReverseOp_t *regAndPutMsg = (CmiOfiRdmaReverseOp_t *)CmiAlloc(mdMsgSize);
-    regAndPutMsg->destAddr = destAddr;
-    regAndPutMsg->destPe   = destPe;
-    regAndPutMsg->destMode = *destMode;
-    regAndPutMsg->srcAddr  = srcAddr;
-    regAndPutMsg->srcPe    = srcPe;
-    regAndPutMsg->srcMode  = *srcMode;
-    regAndPutMsg->rem_mr   = ((CmiOfiRdmaPtr_t *)destInfo)->mr;
-    regAndPutMsg->rem_key  = fi_mr_key(regAndPutMsg->rem_mr);
-    regAndPutMsg->ackSize  = destAckSize;
-    regAndPutMsg->size     = size;
-
-    memcpy((char*)regAndPutMsg + sizeof(CmiOfiRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndPutMsg + sizeof(CmiOfiRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
 #if USE_OFIREQUEST_CACHE
     req = alloc_request(context.request_cache);
 #else
@@ -337,36 +317,35 @@ void LrtsIssueRget(
 
     ZERO_REQUEST(req);
 
-    req->destNode = CmiNodeOf(srcPe);
-    req->destPE   = srcPe;
-    req->size     = mdMsgSize;
+    req->destNode = CmiNodeOf(ncpyOpInfo->srcPe);
+    req->destPE   = ncpyOpInfo->srcPe;
+    req->size     = ncpyOpInfo->ncpyOpInfoSize;
     req->callback = send_short_callback;
-    req->data.short_msg = regAndPutMsg;
+    req->data.short_msg = ncpyOpInfo;
 
-    ofi_send(regAndPutMsg,
-             mdMsgSize,
-             CmiNodeOf(srcPe),
+    ofi_send(ncpyOpInfo,
+             ncpyOpInfo->ncpyOpInfoSize,
+             CmiNodeOf(ncpyOpInfo->srcPe),
              OFI_RDMA_DIRECT_REG_AND_PUT,
              req);
   } else {
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
 
-    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)destInfo;
-    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)srcInfo;
+    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
+    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
 
-    const char *rbuf        = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)srcAddr;
+    const char *rbuf        = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)(ncpyOpInfo->srcPtr);
 
     // Allocate a completion object for tracking read completion and ack handling
     CmiOfiRdmaComp_t* rdmaComp = (CmiOfiRdmaComp_t *)malloc(sizeof(CmiOfiRdmaComp_t));
-    rdmaComp->ack_info         = ref;
+    rdmaComp->ack_info         = ncpyOpInfo;
     rdmaComp->completion_count = 0;
 
     ofi_post_nocopy_operation(
-        (char*)destAddr,
+        (char*)(ncpyOpInfo->destPtr),
         rbuf,
-        CmiNodeOf(srcPe),
+        CmiNodeOf(ncpyOpInfo->srcPe),
         src_info->key,
-        size,
+        ncpyOpInfo->size,
         dest_info->mr,
         ofi_onesided_direct_operation_callback,
         (void *)rdmaComp,
@@ -377,49 +356,26 @@ void LrtsIssueRget(
 
 // Perform an RDMA Put call into the remote destination address from the local source address
 void LrtsIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size) {
+  unsigned short int *destMode) {
 
   OFIRequest *req;
-  CmiAssert(destAckSize == srcAckSize);
 
   // Register local buffer if it is not registered
   if(*srcMode == CMK_BUFFER_UNREG) {
-    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)srcInfo;
-    src_info->mr = registerDirectMemory(srcAddr, size);
+    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
+    src_info->mr = registerDirectMemory(ncpyOpInfo->srcPtr, ncpyOpInfo->size);
     src_info->key = fi_mr_key(src_info->mr);
     *srcMode = CMK_BUFFER_REG;
+
+    // set registration info in the origSrcLayerInfoPtr
+    ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->origSrcLayerInfoPtr))->mr = src_info->mr;
+    ((CmiOfiRdmaPtr_t *)(ncpyOpInfo->origSrcLayerInfoPtr))->key = src_info->key;
   }
 
   if(*destMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform PUT
-    int mdMsgSize = sizeof(CmiOfiRdmaReverseOp_t) + srcAckSize + destAckSize;
-    CmiOfiRdmaReverseOp_t *regAndGetMsg = (CmiOfiRdmaReverseOp_t *)CmiAlloc(mdMsgSize);
-    regAndGetMsg->srcAddr = srcAddr;
-    regAndGetMsg->srcPe   = srcPe;
-    regAndGetMsg->srcMode = *srcMode;
-    regAndGetMsg->destAddr  = destAddr;
-    regAndGetMsg->destPe    = destPe;
-    regAndGetMsg->destMode  = *destMode;
-    regAndGetMsg->rem_mr   = ((CmiOfiRdmaPtr_t *)srcInfo)->mr;
-    regAndGetMsg->rem_key  = fi_mr_key(regAndGetMsg->rem_mr);
-    regAndGetMsg->ackSize  = srcAckSize;
-    regAndGetMsg->size     = size;
-
-    memcpy((char*)regAndGetMsg + sizeof(CmiOfiRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndGetMsg + sizeof(CmiOfiRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
 #if USE_OFIREQUEST_CACHE
     req = alloc_request(context.request_cache);
 #else
@@ -429,37 +385,35 @@ void LrtsIssueRput(
 
     ZERO_REQUEST(req);
 
-    req->destNode = CmiNodeOf(destPe);
-    req->destPE   = destPe;
-    req->size     = mdMsgSize;
+    req->destNode = CmiNodeOf(ncpyOpInfo->destPe);
+    req->destPE   = ncpyOpInfo->destPe;
+    req->size     = ncpyOpInfo->ncpyOpInfoSize;
     req->callback = send_short_callback;
-    req->data.short_msg = regAndGetMsg;
+    req->data.short_msg = ncpyOpInfo;
 
-    ofi_send(regAndGetMsg,
-             mdMsgSize,
-             CmiNodeOf(destPe),
+    ofi_send(ncpyOpInfo,
+             ncpyOpInfo->ncpyOpInfoSize,
+             CmiNodeOf(ncpyOpInfo->destPe),
              OFI_RDMA_DIRECT_REG_AND_GET,
              req);
   } else {
 
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
-
-    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)srcInfo;
-    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)destInfo;
+    CmiOfiRdmaPtr_t *dest_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
+    CmiOfiRdmaPtr_t *src_info = (CmiOfiRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
 
-    const char *rbuf         = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)destAddr;
+    const char *rbuf        = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)(ncpyOpInfo->destPtr);
 
     // Allocate a completion object for tracking write completion and ack handling
     CmiOfiRdmaComp_t* rdmaComp = (CmiOfiRdmaComp_t *)malloc(sizeof(CmiOfiRdmaComp_t));
-    rdmaComp->ack_info         = ref;
+    rdmaComp->ack_info         = ncpyOpInfo;
     rdmaComp->completion_count = 0;
 
     ofi_post_nocopy_operation(
-        (char*)srcAddr,
+        (char*)(ncpyOpInfo->srcPtr),
         rbuf,
-        CmiNodeOf(destPe),
+        CmiNodeOf(ncpyOpInfo->destPe),
         dest_info->key,
-        size,
+        ncpyOpInfo->size,
         src_info->mr,
         ofi_onesided_direct_operation_callback,
         (void *)rdmaComp,
index 0b3c98bc4062af820fa38c88f42536ac9778d28f..9fe4ef4b2fe3e17d6108069bf7fb6157b1e2d454 100644 (file)
@@ -187,26 +187,6 @@ typedef struct _cmi_ofi_rzv_rdma_completion {
   int  completion_count;
 }CmiOfiRdmaComp_t;
 
-/* Machine specific metadata information required to register a buffer and perform
- * an RDMA operation with a remote buffer. This metadata information is used to perform
- * registration and a PUT operation when the remote buffer wants to perform a GET with an
- * unregistered buffer. Similary, the metadata information is used to perform registration
- * and a GET operation when the remote buffer wants to perform a PUT with an unregistered
- * buffer.*/
-typedef struct _cmi_ofi_rdma_reverse_op {
-  const void *destAddr;
-  int destPe;
-  int destMode;
-  const void *srcAddr;
-  int srcPe;
-  int srcMode;
-
-  struct fid_mr *rem_mr;
-  uint64_t rem_key;
-  int ackSize;
-  int size;
-} CmiOfiRdmaReverseOp_t;
-
 // Set the machine specific information for a nocopy pointer
 void LrtsSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode);
 
index 6e8755cd7e0f11e0d0f4468ca1cf77d273fe3638..5cd0a3f522062ec5ded979007c4beb473b5c73c4 100644 (file)
@@ -186,7 +186,7 @@ void CmiIssueRgets(void *recv, int pe){
 #if CMK_ONESIDED_DIRECT_IMPL
 
 // Function Pointer to the individual Acknowledement handler function for the Direct API
-RdmaSingleAckHandlerFn ncpyAckHandlerFn;
+RdmaAckHandlerFn ncpyAckHandlerFn;
 
 typedef struct _cmi_rdma_direct_ack {
   const void *srcAddr;
@@ -200,33 +200,15 @@ typedef struct _cmi_rdma_direct_ack {
 void LrtsSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode);
 void LrtsSetRdmaNcpyAck(RdmaAckHandlerFn fn);
 void LrtsIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size);
+  unsigned short int *destMode);
+
 void LrtsIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size);
+  unsigned short int *destMode);
+
 void LrtsDeregisterMem(const void *ptr, void *info, int pe, unsigned short int mode);
 
 /* Set the machine specific information for a nocopy pointer */
@@ -234,106 +216,33 @@ void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short
   LrtsSetRdmaBufferInfo(info, ptr, size, mode);
 }
 
-void *CmiGetNcpyAck(const void *srcAddr, void *srcAck, int srcPe, const void *destAddr, void *destAck, int destPe, int ackSize) {
-  CmiRdmaDirectAck *directAck = (CmiRdmaDirectAck *)malloc(sizeof(CmiRdmaDirectAck) + 2*ackSize);
-  directAck->srcAddr = srcAddr;
-  directAck->srcPe = srcPe;
-  directAck->destAddr = destAddr;
-  directAck->destPe = destPe;
-  directAck->ackSize = ackSize;
-
-  // copy source ack
-  memcpy((char *)directAck + sizeof(CmiRdmaDirectAck), srcAck, ackSize);
-
-  // copy destination ack
-  memcpy((char *)directAck + sizeof(CmiRdmaDirectAck) + ackSize, destAck, ackSize);
-
-  return (void *)directAck;
-}
-
 void CmiInvokeNcpyAck(void *ack) {
-  CmiRdmaDirectAck *directAck = (CmiRdmaDirectAck *)ack;
-
-  // Retrieve source ack
-  void *srcAck = (char *)directAck + sizeof(CmiRdmaDirectAck);
-
-  // Retrieve destination ack
-  void *destAck = (char *)srcAck + directAck->ackSize;
-
-  ncpyAckHandlerFn(srcAck, directAck->srcPe, directAck->srcAddr);
-  ncpyAckHandlerFn(destAck, directAck->destPe, directAck->destAddr);
-
-  // free the allocated ack information
-  free(directAck);
+  ncpyAckHandlerFn(ack);
 }
 
 /* Set the ack handler function used in the Direct API */
-void CmiSetRdmaNcpyAck(RdmaSingleAckHandlerFn fn){
+void CmiSetRdmaNcpyAck(RdmaAckHandlerFn fn){
   ncpyAckHandlerFn = fn;
 }
 
 /* Perform an RDMA Get operation into the local destination address from the remote source address*/
 void CmiIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
+  unsigned short int *destMode) {
 
   // Use network RDMA for a PE on a remote host
-  LrtsIssueRget(srcAddr,
-                (char*)srcInfo + CmiGetRdmaCommonInfoSize(),
-                srcAck,
-                srcAckSize,
-                srcPe,
-                srcMode,
-                destAddr,
-                (char*)destInfo + CmiGetRdmaCommonInfoSize(),
-                destAck,
-                destAckSize,
-                destPe,
-                destMode,
-                size);
+  LrtsIssueRget(ncpyOpInfo, srcMode, destMode);
 }
 
 /* Perform an RDMA Put operation into the remote destination address from the local source address */
 void CmiIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size) {
+  unsigned short int *destMode) {
 
   // Use network RDMA for a PE on a remote host
-  LrtsIssueRput(destAddr,
-                (char*)destInfo + CmiGetRdmaCommonInfoSize(),
-                destAck,
-                destAckSize,
-                destPe,
-                destMode,
-                srcAddr,
-                (char*)srcInfo + CmiGetRdmaCommonInfoSize(),
-                srcAck,
-                srcAckSize,
-                srcPe,
-                srcMode,
-                size);
+  LrtsIssueRput(ncpyOpInfo, srcMode, destMode);
 }
 
 /* De-register registered memory for pointer */
index b7d132aab8c2acd59bfffa295420fcb790438a35..9d8553e85518761485135a37ab59cf801828850d 100644 (file)
@@ -1730,53 +1730,48 @@ static inline void processRecvWC(struct ibv_wc *recvWC,const int toBuffer){
        }
        if(header->code == INFIRDMA_DIRECT_REG_AND_PUT){
                // Register the source buffer and perform PUT
-               CmiVerbsRdmaReverseOp_t *regAndPutMsg = (CmiVerbsRdmaReverseOp_t *)(buffer->buf+sizeof(struct infiPacketHeader));
-               struct ibv_mr *mr = registerDirectMemory(regAndPutMsg->srcAddr, regAndPutMsg->size);
-
-               void *ref = CmiGetNcpyAck(regAndPutMsg->srcAddr,
-                                         (char *)regAndPutMsg + sizeof(CmiVerbsRdmaReverseOp_t)+ regAndPutMsg->ackSize, //srcAck
-                                         regAndPutMsg->srcPe,
-                                         regAndPutMsg->destAddr,
-                                         (char *)regAndPutMsg + sizeof(CmiVerbsRdmaReverseOp_t), //destAck
-                                         regAndPutMsg->destPe,
-                                         regAndPutMsg->ackSize);
+               NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(buffer->buf+sizeof(struct infiPacketHeader));
                
+               NcpyOperationInfo *newNcpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyOpInfo->ncpyOpInfoSize);
+               memcpy((char *)newNcpyOpInfo, (char *)ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize);
+               
+               resetNcpyOpInfoPointers(newNcpyOpInfo);
+               
+               struct ibv_mr *mr = registerDirectMemory(newNcpyOpInfo->srcPtr, newNcpyOpInfo->size);
                struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
                rdmaPacket->type = INFI_ONESIDED_DIRECT;
-               rdmaPacket->localBuffer = ref;
+               rdmaPacket->localBuffer = newNcpyOpInfo;
                
-               postRdma((uint64_t)regAndPutMsg->srcAddr,
+               postRdma((uint64_t)(newNcpyOpInfo->srcPtr),
                        mr->lkey,
-                       (uint64_t)regAndPutMsg->destAddr,
-                       regAndPutMsg->rem_key,
-                       regAndPutMsg->size,
-                       regAndPutMsg->destPe,
+                       (uint64_t)(newNcpyOpInfo->destPtr),
+            ((CmiVerbsRdmaPtr_t *)(newNcpyOpInfo->destLayerInfo))->key,
+                       newNcpyOpInfo->size,
+                       newNcpyOpInfo->destPe,
                        (uint64_t)rdmaPacket,
                        IBV_WR_RDMA_WRITE);
        }
        if(header->code == INFIRDMA_DIRECT_REG_AND_GET){
                // Register the destination buffer and perform GET
-               CmiVerbsRdmaReverseOp_t *regAndGetMsg = (CmiVerbsRdmaReverseOp_t *)(buffer->buf+sizeof(struct infiPacketHeader));
-               struct ibv_mr *mr = registerDirectMemory(regAndGetMsg->destAddr, regAndGetMsg->size);
-
-               void *ref = CmiGetNcpyAck(regAndGetMsg->srcAddr,
-                                         (char *)regAndGetMsg + sizeof(CmiVerbsRdmaReverseOp_t)+ regAndGetMsg->ackSize, //srcAck
-                                         regAndGetMsg->srcPe,
-                                         regAndGetMsg->destAddr,
-                                         (char *)regAndGetMsg + sizeof(CmiVerbsRdmaReverseOp_t), //destAck
-                                         regAndGetMsg->destPe,
-                                         regAndGetMsg->ackSize);
+               NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(buffer->buf+sizeof(struct infiPacketHeader));
+               
+               NcpyOperationInfo *newNcpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyOpInfo->ncpyOpInfoSize);
+               memcpy(newNcpyOpInfo, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize);
+               
+               resetNcpyOpInfoPointers(newNcpyOpInfo);
+               
+               struct ibv_mr *mr = registerDirectMemory(newNcpyOpInfo->destPtr, newNcpyOpInfo->size);
                
                struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
                rdmaPacket->type = INFI_ONESIDED_DIRECT;
-               rdmaPacket->localBuffer = ref;
+               rdmaPacket->localBuffer = newNcpyOpInfo;
                
-               postRdma((uint64_t)regAndGetMsg->destAddr,
+               postRdma((uint64_t)newNcpyOpInfo->destPtr,
                        mr->lkey,
-                       (uint64_t)regAndGetMsg->srcAddr,
-                       regAndGetMsg->rem_key,
-                       regAndGetMsg->size,
-                       regAndGetMsg->srcPe,
+                       (uint64_t)newNcpyOpInfo->srcPtr,
+                       ((CmiVerbsRdmaPtr_t *)(newNcpyOpInfo->srcLayerInfo))->key,
+                       newNcpyOpInfo->size,
+                       newNcpyOpInfo->srcPe,
                        (uint64_t)rdmaPacket,
                        IBV_WR_RDMA_READ);
        }
index 8d25edbcd01699597799f0df0d08ee3c1b562eb8..040a13c61e1a3a909b150c7ea30bf99038ce271f 100644 (file)
@@ -170,76 +170,51 @@ struct ibv_mr* registerDirectMemory(const void *addr, int size) {
 
 // Perform an RDMA Get call into the local destination address from the remote source address
 void LrtsIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
-
-  CmiAssert(srcAckSize == destAckSize);
+  unsigned short int *destMode) {
 
   // Register local buffer if it is not registered
   if(*destMode == CMK_BUFFER_UNREG) {
-    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)destInfo;
-    dest_info->mr = registerDirectMemory(destAddr, size);
+    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
+    dest_info->mr = registerDirectMemory(ncpyOpInfo->destPtr, ncpyOpInfo->size);
     dest_info->key = dest_info->mr->rkey;
     *destMode = CMK_BUFFER_REG;
+
+    // set registration info in the origDestLayerInfoPtr
+    ((CmiVerbsRdmaPtr_t *)(ncpyOpInfo->origDestLayerInfoPtr))->mr = dest_info->mr;
+    ((CmiVerbsRdmaPtr_t *)(ncpyOpInfo->origDestLayerInfoPtr))->key = dest_info->key;
   }
 
   if(*srcMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform PUT
-    int mdMsgSize = sizeof(CmiVerbsRdmaReverseOp_t) + destAckSize + srcAckSize;
-    CmiVerbsRdmaReverseOp_t *regAndPutMsg = (CmiVerbsRdmaReverseOp_t *)CmiAlloc(mdMsgSize);
-    regAndPutMsg->destAddr = destAddr;
-    regAndPutMsg->destPe   = destPe;
-    regAndPutMsg->destMode = *destMode;
-    regAndPutMsg->srcAddr  = srcAddr;
-    regAndPutMsg->srcPe    = srcPe;
-    regAndPutMsg->srcMode  = *srcMode;
-    regAndPutMsg->rem_mr   = ((CmiVerbsRdmaPtr_t *)destInfo)->mr;
-    regAndPutMsg->rem_key  = regAndPutMsg->rem_mr->rkey;
-    regAndPutMsg->ackSize  = destAckSize;
-    regAndPutMsg->size     = size;
-
-    memcpy((char*)regAndPutMsg + sizeof(CmiVerbsRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndPutMsg + sizeof(CmiVerbsRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
     infiPacket packet;
     MallocInfiPacket(packet);
 
-    packet->size = mdMsgSize;
-    packet->buf  = (char *)regAndPutMsg;
+    packet->size = ncpyOpInfo->ncpyOpInfoSize;
+    packet->buf  = (char *)ncpyOpInfo;
     packet->header.code = INFIRDMA_DIRECT_REG_AND_PUT;
     packet->ogm  = NULL;
 
-    struct ibv_mr *packetKey = METADATAFIELD(regAndPutMsg)->key;
-    OtherNode node = &nodes[CmiNodeOf(srcPe)];
-    EnqueuePacket(node, packet, mdMsgSize, packetKey);
+    struct ibv_mr *packetKey = METADATAFIELD(ncpyOpInfo)->key;
+    OtherNode node = &nodes[CmiNodeOf(ncpyOpInfo->srcPe)];
+    EnqueuePacket(node, packet, ncpyOpInfo->ncpyOpInfoSize, packetKey);
 
   } else {
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
 
     struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
     rdmaPacket->type = INFI_ONESIDED_DIRECT;
-    rdmaPacket->localBuffer = ref;
+    rdmaPacket->localBuffer = ncpyOpInfo;
 
-    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)destInfo;
-    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)srcInfo;
+    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
+    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
 
-    postRdma((uint64_t)destAddr,
+    postRdma((uint64_t)(ncpyOpInfo->destPtr),
             dest_info->key,
-            (uint64_t)srcAddr,
+            (uint64_t)(ncpyOpInfo->srcPtr),
             src_info->key,
-            size,
-            srcPe,
+            ncpyOpInfo->size,
+            ncpyOpInfo->srcPe,
             (uint64_t)rdmaPacket,
             IBV_WR_RDMA_READ);
   }
@@ -247,78 +222,50 @@ void LrtsIssueRget(
 
 // Perform an RDMA Put call into the remote destination address from the local source address
 void LrtsIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size) {
-
-  CmiAssert(srcAckSize == destAckSize);
+  unsigned short int *destMode) {
 
   // Register local buffer if it is not registered
   if(*srcMode == CMK_BUFFER_UNREG) {
-    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)srcInfo;
-    src_info->mr = registerDirectMemory(srcAddr, size);
+    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
+    src_info->mr = registerDirectMemory(ncpyOpInfo->srcPtr, ncpyOpInfo->size);
     src_info->key = src_info->mr->rkey;
     *srcMode = CMK_BUFFER_REG;
+
+    // set registration info in the origSrcLayerInfoPtr
+    ((CmiVerbsRdmaPtr_t *)(ncpyOpInfo->origSrcLayerInfoPtr))->mr = src_info->mr;
+    ((CmiVerbsRdmaPtr_t *)(ncpyOpInfo->origSrcLayerInfoPtr))->key = src_info->key;
   }
 
   if(*destMode == CMK_BUFFER_UNREG) {
     // Remote buffer is unregistered, send a message to register it and perform GET
-    int mdMsgSize = sizeof(CmiVerbsRdmaReverseOp_t) + destAckSize + srcAckSize;
-    CmiVerbsRdmaReverseOp_t *regAndGetMsg = (CmiVerbsRdmaReverseOp_t *)CmiAlloc(mdMsgSize);
-    regAndGetMsg->srcAddr  = srcAddr;
-    regAndGetMsg->srcPe    = srcPe;
-    regAndGetMsg->srcMode  = *srcMode;
-
-    regAndGetMsg->destAddr = destAddr;
-    regAndGetMsg->destPe   = destPe;
-    regAndGetMsg->destMode = *destMode;
-
-    regAndGetMsg->rem_mr   = ((CmiVerbsRdmaPtr_t *)srcInfo)->mr;
-    regAndGetMsg->rem_key  = regAndGetMsg->rem_mr->rkey;
-    regAndGetMsg->ackSize  = srcAckSize;
-    regAndGetMsg->size     = size;
-
-    memcpy((char*)regAndGetMsg + sizeof(CmiVerbsRdmaReverseOp_t), destAck, destAckSize);
-    memcpy((char*)regAndGetMsg + sizeof(CmiVerbsRdmaReverseOp_t) + srcAckSize, srcAck, srcAckSize);
-
     infiPacket packet;
     MallocInfiPacket(packet);
 
-    packet->size = mdMsgSize;
-    packet->buf  = (char *)regAndGetMsg;
+    packet->size = ncpyOpInfo->ncpyOpInfoSize;
+    packet->buf  = (char *)ncpyOpInfo;
     packet->header.code = INFIRDMA_DIRECT_REG_AND_GET;
     packet->ogm  = NULL;
 
-    struct ibv_mr *packetKey = METADATAFIELD(regAndGetMsg)->key;
-    OtherNode node = &nodes[CmiNodeOf(destPe)];
-    EnqueuePacket(node, packet, mdMsgSize, packetKey);
+    struct ibv_mr *packetKey = METADATAFIELD(ncpyOpInfo)->key;
+    OtherNode node = &nodes[CmiNodeOf(ncpyOpInfo->destPe)];
+    EnqueuePacket(node, packet, ncpyOpInfo->ncpyOpInfoSize, packetKey);
 
   } else {
-    void *ref = CmiGetNcpyAck(srcAddr, srcAck, srcPe, destAddr, destAck, destPe, srcAckSize);
-
     struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
     rdmaPacket->type = INFI_ONESIDED_DIRECT;
-    rdmaPacket->localBuffer = ref;
+    rdmaPacket->localBuffer = ncpyOpInfo;
 
-    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)srcInfo;
-    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)destInfo;
+    CmiVerbsRdmaPtr_t *src_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->srcLayerInfo);
+    CmiVerbsRdmaPtr_t *dest_info = (CmiVerbsRdmaPtr_t *)(ncpyOpInfo->destLayerInfo);
 
-    postRdma((uint64_t)srcAddr,
+    postRdma((uint64_t)(ncpyOpInfo->srcPtr),
             src_info->key,
-            (uint64_t)destAddr,
+            (uint64_t)(ncpyOpInfo->destPtr),
             dest_info->key,
-            size,
-            destPe,
+            ncpyOpInfo->size,
+            ncpyOpInfo->destPe,
             (uint64_t)rdmaPacket,
             IBV_WR_RDMA_WRITE);
   }
index 34f8e66a2ec6db03ec9bed72d23ea0078acab2a4..37dcf6eb74794db64d5dbd06bd3e435a7c08a2f1 100644 (file)
@@ -4,6 +4,7 @@
 
 #include "charm++.h"
 #include "converse.h"
+#include "cmirdmautils.h"
 
 
 #if CMK_SMP && CMK_IMMEDIATE_MSG
@@ -320,17 +321,40 @@ int getRdmaBufSize(envelope *env){
 /* Support for Direct Nocopy API */
 
 // Ack handler function which invokes the callback
-void CkRdmaAckHandler(void *cbPtr, int pe, const void *ptr) {
-  CkCallback *cb = (CkCallback *)cbPtr;
+void CkRdmaDirectAckHandler(void *ack) {
+
+  NcpyOperationInfo *info = (NcpyOperationInfo *)(ack);
+
+  if(info->ackMode == 0 || info->ackMode == 1) {
+    //Invoke the sender's callback
+    CkNcpyAck srcAck(info->srcPtr, info->srcRef);
 
 #if CMK_SMP && CMK_IMMEDIATE_MSG
-  //call to callbackgroup to call the callback when calling from comm thread
-  //this adds one more trip through the scheduler
-  _ckcallbackgroup[pe].call(*cb, sizeof(void *), (char*)&ptr);
+    //call to callbackgroup to call the callback when calling from comm thread
+    //this adds one more trip through the scheduler
+    _ckcallbackgroup[info->srcPe].call(*(CkCallback *)(info->srcAck), sizeof(CkNcpyAck), (const char *)(&srcAck));
 #else
-  //Invoke the destination callback
-  cb->send(sizeof(void *), (char *)&ptr);
+    //Invoke the destination callback
+    ((CkCallback *)(info->srcAck))->send(sizeof(CkNcpyAck), &srcAck);
 #endif
+  }
+
+  if(info->ackMode == 0 || info->ackMode == 2) {
+    //Invoke the receiver's callback
+    CkNcpyAck destAck(info->destPtr, info->destRef);
+
+#if CMK_SMP && CMK_IMMEDIATE_MSG
+    //call to callbackgroup to call the callback when calling from comm thread
+    //this adds one more trip through the scheduler
+    _ckcallbackgroup[info->destPe].call(*(CkCallback *)(info->destAck), sizeof(CkNcpyAck), (const char *)(&destAck));
+#else
+    //Invoke the destination callback
+    ((CkCallback *)(info->destAck))->send(sizeof(CkNcpyAck), &destAck);
+#endif
+  }
+
+  if(info->freeMe)
+    CmiFree(info);
 }
 
 // Returns ncpyTransferMode::MEMCPY if both the PEs are the same and memcpy can be used
@@ -366,20 +390,39 @@ void CkNcpyBuffer::cmaGet(CkNcpyBuffer &source) {
 #endif
 
 void CkNcpyBuffer::rdmaGet(CkNcpyBuffer &source) {
-  // Issue the Rget call
-  CmiIssueRget(source.ptr,
-               source.layerInfo,
-               &source.cb,
-               sizeof(CkCallback),
-               source.pe,
+
+  int layerInfoSize = CMK_NOCOPY_DIRECT_BYTES;
+  int ackSize = sizeof(CkCallback);
+
+  // Create a general object that can be used across layers
+  int ncpyObjSize = getNcpyOpInfoTotalSize(
+                      layerInfoSize,
+                      ackSize,
+                      layerInfoSize,
+                      ackSize);
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyObjSize);
+
+  setNcpyOpInfo(source.ptr,
+                (char *)(&(source.layerInfo[0])) + CmiGetRdmaCommonInfoSize(),
+                layerInfoSize,
+                (char *)(&source.cb),
+                ackSize,
+                source.pe,
+                source.ref,
+                ptr,
+                (char *)(&(layerInfo[0])) + CmiGetRdmaCommonInfoSize(),
+                layerInfoSize,
+                (char *)(&cb),
+                ackSize,
+                pe,
+                ref,
+                cnt,
+                ncpyOpInfo);
+
+  CmiIssueRget(ncpyOpInfo,
                &source.mode,
-               ptr,
-               layerInfo,
-               &cb,
-               sizeof(CkCallback),
-               CkMyPe(),
-               &mode,
-               cnt);
+               &mode);
 }
 
 // Perform a nocopy get operation into this destination using the passed source
@@ -401,20 +444,26 @@ void CkNcpyBuffer::get(CkNcpyBuffer &source){
     memcpyGet(source);
 
     //Invoke the receiver's callback
-    cb.send(sizeof(void *), &ptr);
+    CkNcpyAck srcAck(ptr, ref);
+    cb.send(sizeof(CkNcpyAck), &srcAck);
 
     //Invoke the sender's callback
-    source.cb.send(sizeof(void *), &source.ptr);
+    CkNcpyAck destAck(source.ptr, source.ref);
+    source.cb.send(sizeof(CkNcpyAck), &destAck);
+
 #if CMK_USE_CMA
   } else if(transferMode == ncpyTransferMode::CMA) {
 
     cmaGet(source);
 
     //Invoke the receiver's callback
-    cb.send(sizeof(void *), &ptr);
+    CkNcpyAck srcAck(ptr, ref);
+    cb.send(sizeof(CkNcpyAck), &srcAck);
 
     //Invoke the sender's callback
-    source.cb.send(sizeof(void *), &source.ptr);
+    CkNcpyAck destAck(source.ptr, source.ref);
+    source.cb.send(sizeof(CkNcpyAck), &destAck);
+
 #endif
   } else if (transferMode == ncpyTransferMode::RDMA) {
     rdmaGet(source);
@@ -442,20 +491,39 @@ void CkNcpyBuffer::cmaPut(CkNcpyBuffer &destination) {
 #endif
 
 void CkNcpyBuffer::rdmaPut(CkNcpyBuffer &destination) {
-  // Issue the Rput call
-  CmiIssueRput(destination.ptr,
-               destination.layerInfo,
-               &destination.cb,
-               sizeof(CkCallback),
-               destination.pe,
-               &destination.mode,
-               ptr,
-               layerInfo,
-               &cb,
-               sizeof(CkCallback),
-               CkMyPe(),
+
+  int layerInfoSize = CMK_NOCOPY_DIRECT_BYTES;
+  int ackSize = sizeof(CkCallback);
+
+  // Create a general object that can be used across layers
+  int ncpyObjSize = getNcpyOpInfoTotalSize(
+                      layerInfoSize,
+                      ackSize,
+                      layerInfoSize,
+                      ackSize);
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)CmiAlloc(ncpyObjSize);
+
+  setNcpyOpInfo(ptr,
+                (char *)(&(layerInfo[0])) + CmiGetRdmaCommonInfoSize(),
+                layerInfoSize,
+                (char *)(&cb),
+                ackSize,
+                pe,
+                ref,
+                destination.ptr,
+                (char *)(&(destination.layerInfo[0])) + CmiGetRdmaCommonInfoSize(),
+                layerInfoSize,
+                (char *)(&destination.cb),
+                ackSize,
+                destination.pe,
+                destination.ref,
+                cnt,
+                ncpyOpInfo);
+
+  CmiIssueRput(ncpyOpInfo,
                &mode,
-               cnt);
+               &destination.mode);
 }
 
 // Perform a nocopy put operation into the passed destination using this source
@@ -476,20 +544,25 @@ void CkNcpyBuffer::put(CkNcpyBuffer &destination){
     memcpyPut(destination);
 
     //Invoke the source callback
-    cb.send(sizeof(void *), &ptr);
+    CkNcpyAck srcAck(ptr, ref);
+    cb.send(sizeof(CkNcpyAck), &srcAck);
 
     //Invoke the destination callback
-    destination.cb.send(sizeof(void *), &destination.ptr);
+    CkNcpyAck destAck(destination.ptr, destination.ref);
+    destination.cb.send(sizeof(CkNcpyAck), &destAck);
 
 #if CMK_USE_CMA
   } else if(transferMode == ncpyTransferMode::CMA) {
     cmaPut(destination);
 
     //Invoke the source callback
-    cb.send(sizeof(void *), &ptr);
+    CkNcpyAck srcAck(ptr, ref);
+    cb.send(sizeof(CkNcpyAck), &srcAck);
 
     //Invoke the destination callback
-    destination.cb.send(sizeof(void *), &destination.ptr);
+    CkNcpyAck destAck(destination.ptr, destination.ref);
+    destination.cb.send(sizeof(CkNcpyAck), &destAck);
+
 #endif
   } else if (transferMode == ncpyTransferMode::RDMA) {
     rdmaPut(destination);
index 585a87b6da6e8df4298fa172d3b7aaf834294494..638b4208a8e42a2490bbcf819db5028cc5db997f 100644 (file)
@@ -105,7 +105,23 @@ int getRdmaBufSize(envelope *env);
 
 // Ack handler function which invokes the callbacks on the source and destination PEs
 void CkRdmaAckHandler(void *cookie);
-void CkRdmaAckHandler(void *cbPtr, int pe, const void *ptr);
+void CkRdmaDirectAckHandler(void *ack);
+
+// Class to represent an acknowledgement structure
+class CkNcpyAck{
+  public:
+  // pointer to the buffer
+  const void *ptr;
+
+  // reference pointer
+  // This is an optional arbitrary pointer set by the user before performing the get/put
+  // operation. It is returned back in the CkNcpyAck object.
+  const void *ref;
+
+  CkNcpyAck(const void *ptr_, const void *ref_) : ptr(ptr_), ref(ref_) {}
+};
+
+PUPbytes(CkNcpyAck);
 
 // Class to represent an RDMA buffer
 class CkNcpyBuffer{
@@ -139,15 +155,16 @@ class CkNcpyBuffer{
   // mode
   unsigned short int mode;
 
-  CkNcpyBuffer() : ptr(NULL), pe(-1), mode(CK_BUFFER_UNREG) {}
+  // reference pointer
+  const void *ref;
 
-  CkNcpyBuffer(const void *ptr_, size_t cnt_, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_UNREG)
-  {
+  CkNcpyBuffer() : ptr(NULL), pe(-1), ref(NULL), mode(CK_BUFFER_UNREG) {}
+
+  CkNcpyBuffer(const void *ptr_, size_t cnt_, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_UNREG) {
     init(ptr_, cnt_, cb_, mode_);
   }
 
-  void init(const void *ptr_, size_t cnt_, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_UNREG)
-  {
+  void init(const void *ptr_, size_t cnt_, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_UNREG) {
     ptr  = ptr_;
     cnt  = cnt_;
     cb   = cb_;
@@ -159,6 +176,14 @@ class CkNcpyBuffer{
     registerMem();
   }
 
+  void setRef(const void *ref_) {
+    ref = ref_;
+  }
+
+  const void *getRef() {
+    return ref;
+  }
+
   // Register(Pin) the memory for the buffer
   void registerMem()
   {
@@ -218,6 +243,7 @@ class CkNcpyBuffer{
 
   void pup(PUP::er &p) {
     p((char *)&ptr, sizeof(ptr));
+    p((char *)&ref, sizeof(ref));
     p|cnt;
     p|cb;
     p|pe;
index fdbb1ae38b190881ba71f0f3f4e4d75d6d9665fd..1875cbcce8b4445a1f15065f00e61db80293a95a 100644 (file)
@@ -1284,7 +1284,7 @@ void _initCharm(int unused_argc, char **argv)
        CkMessageWatcherInit(argv,CkpvAccess(_coreState));
        
        // Set the ack handler function used for the direct nocopy api
-       CmiSetRdmaNcpyAck(CkRdmaAckHandler);
+       CmiSetRdmaNcpyAck(CkRdmaDirectAckHandler);
 
        /**
          The rank-0 processor of each node calls the 
diff --git a/src/conv-core/conv-header.h b/src/conv-core/conv-header.h
new file mode 100644 (file)
index 0000000..9d46254
--- /dev/null
@@ -0,0 +1,55 @@
+#ifndef _CONV_HEADER_H
+#define _CONV_HEADER_H
+
+#include "conv-config.h"
+
+/******** CMI: TYPE DEFINITIONS ********/
+
+typedef CMK_TYPEDEF_INT2      CmiInt2;
+typedef CMK_TYPEDEF_INT4      CmiInt4;
+typedef CMK_TYPEDEF_INT8      CmiInt8;
+typedef CMK_TYPEDEF_UINT2     CmiUInt2;
+typedef CMK_TYPEDEF_UINT4     CmiUInt4;
+typedef CMK_TYPEDEF_UINT8     CmiUInt8;
+#if CMK___int128_t_DEFINED
+typedef __int128_t            CmiInt16;
+typedef __uint128_t           CmiUInt16;
+#elif CMK___int128_DEFINED
+typedef __int128              CmiInt16;
+typedef __uint128     CmiUInt16;
+#endif
+
+#if defined(CMK_CUSTOM_FP_FORMAT)
+typedef CMK_TYPEDEF_FLOAT4    CmiFloat4;
+typedef CMK_TYPEDEF_FLOAT8    CmiFloat8;
+#else
+typedef float                 CmiFloat4;
+typedef double                CmiFloat8;
+#endif
+
+typedef void  *CmiCommHandle;
+typedef void (*CmiHandler)(void *msg);
+typedef void (*CmiHandlerEx)(void *msg,void *userPtr);
+
+
+
+typedef struct CMK_MSG_HEADER_BASIC CmiMsgHeaderBasic;
+typedef struct CMK_MSG_HEADER_EXT   CmiMsgHeaderExt;
+
+#define CmiMsgHeaderSizeBytes (sizeof(CmiMsgHeaderBasic))
+#define CmiExtHeaderSizeBytes (sizeof(CmiMsgHeaderExt))
+
+/* all common extra fields in BigSim message header */
+#define CMK_BIGSIM_FIELDS  CmiInt4 nd,n; double rt; CmiInt2 tID, hID; char t, flag; CmiInt2 ref; CmiInt4 msgID, srcPe;
+
+#ifndef CmiReservedHeaderSize
+typedef struct CMK_MSG_HEADER_BIGSIM_   CmiBlueGeneMsgHeader;
+#define CmiBlueGeneMsgHeaderSizeBytes (sizeof(CmiBlueGeneMsgHeader))
+#if CMK_BIGSIM_CHARM
+#  define CmiReservedHeaderSize   CmiBlueGeneMsgHeaderSizeBytes
+#else
+#  define CmiReservedHeaderSize   CmiExtHeaderSizeBytes
+#endif
+#endif
+
+#endif
index 050230de97be11050f8df55ca0dea9ee64be8f88..dde3d4df6598cdc69a6d873de75d01e04a6a8391 100644 (file)
@@ -17,60 +17,48 @@ int CmiGetRdmaCommonInfoSize() {
 /* Support for generic implementation */
 
 // Function Pointer to Acknowledement handler function for the Direct API
-RdmaSingleAckCallerFn ncpyAckHandlerFn;
+RdmaAckCallerFn ncpyAckHandlerFn;
 
 // An Rget initiator PE sends this message to the target PE that will be the source of the data
-typedef struct _getRequestMsg {
+typedef struct _converseRdmaMsg {
   char cmicore[CmiMsgHeaderSizeBytes];
-  int srcPe; /* Source processor */
-  int destPe; /* Destination processor */
-  int size; /* size of the source buffer */
-  char *srcAddr; /* Source Address */
-  char *destAddr; /* Destination Address */
-  int ackSize;  /* Number of bytes occupied by the ack */
-} getRequestMsg;
-
-// This is a header for RDMA payloads transferred as normal converse messages,
-// delivered to the PE holding the destination buffer (Rget initiator or Rput target)
-typedef struct _rdmaPayloadMsg {
-  char cmicore[CmiMsgHeaderSizeBytes];
-  int pe; /* Source processor */
-  int size; /* size of the buffer */
-  char *destAddr; /* Destination Address */
-  char *ref; /* Reference Address used for invoking acks*/
-} rdmaPayloadMsg;
+} ConverseRdmaMsg;
 
 static int get_request_handler_idx;
 static int put_data_handler_idx;
 
 // Invoked when this PE has to send a large array for an Rget
-static void getRequestHandler(getRequestMsg *reqMsg){
-  void *srcAck = (char *)reqMsg + sizeof(getRequestMsg);
-  void *destAck = (char *)srcAck + reqMsg->ackSize;
+static void getRequestHandler(ConverseRdmaMsg *getReqMsg){
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)((char *)(getReqMsg) + sizeof(ConverseRdmaMsg));
+
+  resetNcpyOpInfoPointers(ncpyOpInfo);
+
+  ncpyOpInfo->freeMe = 0;
+
   // Get is implemented internally using a call to Put
-  CmiIssueRput(reqMsg->destAddr,
+  CmiIssueRput(ncpyOpInfo,
                NULL,
-               destAck,
-               reqMsg->ackSize,
-               reqMsg->destPe,
-               NULL,
-               reqMsg->srcAddr,
-               NULL,
-               srcAck,
-               reqMsg->ackSize,
-               reqMsg->srcPe,
-               NULL,
-               reqMsg->size);
+               NULL);
 }
 
 // Invoked when this PE receives a large array as the target of an Rput or the initiator of an Rget
-static void putDataHandler(rdmaPayloadMsg *recvMsg) {
+static void putDataHandler(ConverseRdmaMsg *payloadMsg) {
+
+  NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)((char *)payloadMsg + sizeof(ConverseRdmaMsg));
+
+  resetNcpyOpInfoPointers(ncpyOpInfo);
+
   // copy the received messsage into the user's destination address
-  memcpy(recvMsg->destAddr, (char *)recvMsg + sizeof(rdmaPayloadMsg), recvMsg->size);
+  memcpy((char *)ncpyOpInfo->destPtr,
+         (char *)payloadMsg + sizeof(ConverseRdmaMsg) + ncpyOpInfo->ncpyOpInfoSize,
+         ncpyOpInfo->size);
 
   // Invoke the destination ack
-  void *destAck = (char *)recvMsg + sizeof(rdmaPayloadMsg) + recvMsg->size;
-  ncpyAckHandlerFn(destAck, recvMsg->pe, recvMsg->destAddr);
+  ncpyOpInfo->ackMode = 2;
+  ncpyOpInfo->freeMe  = 0;
+  ncpyAckHandlerFn(ncpyOpInfo);
+  //ncpyAckHandlerFn(destAck, recvMsg->pe, recvMsg->destAddr);
 }
 
 // Rget/Rput operations are implemented as normal converse messages
@@ -80,79 +68,62 @@ void CmiOnesidedDirectInit(void) {
   put_data_handler_idx = CmiRegisterHandler((CmiHandler)putDataHandler);
 }
 
-void CmiSetRdmaNcpyAck(RdmaSingleAckCallerFn fn) {
+void CmiSetRdmaNcpyAck(RdmaAckCallerFn fn) {
   ncpyAckHandlerFn = fn;
 }
 
 void CmiIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size) {
-
-  // Send a getRequestMsg to other PE requesting it to send the array
-  getRequestMsg *getReqMsg = (getRequestMsg *)CmiAlloc(sizeof(getRequestMsg) + srcAckSize + destAckSize);
-  getReqMsg->srcPe = srcPe;
-  getReqMsg->destPe = destPe;
-  getReqMsg->size = size;
-  getReqMsg->srcAddr = (char *)srcAddr;
-  getReqMsg->destAddr = (char *)destAddr;
-
-  CmiAssert(srcAckSize == destAckSize);
-  getReqMsg->ackSize = srcAckSize;
-
-  // copy the source ack into the getReqMsg
-  memcpy((char *)getReqMsg + sizeof(getRequestMsg), srcAck, srcAckSize);
-
-  // copy the destination ack into the getReqMsg
-  memcpy((char *)getReqMsg + sizeof(getRequestMsg) + srcAckSize, destAck, destAckSize);
+  unsigned short int *destMode) {
+
+  int ncpyOpInfoSize = ncpyOpInfo->ncpyOpInfoSize;
+
+  // Send a ConverseRdmaMsg to other PE requesting it to send the array
+  ConverseRdmaMsg *getReqMsg = (ConverseRdmaMsg *)CmiAlloc(sizeof(ConverseRdmaMsg) + ncpyOpInfoSize);
+
+  // copy the additional Info into the getReqMsg
+  memcpy((char *)getReqMsg + sizeof(ConverseRdmaMsg),
+         (char *)ncpyOpInfo,
+         ncpyOpInfoSize);
 
   CmiSetHandler(getReqMsg, get_request_handler_idx);
-  CmiSyncSendAndFree(srcPe, sizeof(getRequestMsg) + srcAckSize + destAckSize, getReqMsg);
+  CmiSyncSendAndFree(ncpyOpInfo->srcPe, sizeof(ConverseRdmaMsg) + ncpyOpInfoSize, getReqMsg);
+
+  // free original ncpyOpinfo
+  CmiFree(ncpyOpInfo);
 }
 
 void CmiIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size) {
+  unsigned short int *destMode) {
 
-  // Send a rdmaPayloadMsg to the other PE sending the array
-  rdmaPayloadMsg *recvMsg = (rdmaPayloadMsg *)CmiAlloc(sizeof(rdmaPayloadMsg) + size + destAckSize);
+  int ncpyOpInfoSize = ncpyOpInfo->ncpyOpInfoSize;
+  int size = ncpyOpInfo->size;
 
-  // copy the large array into the recvMsg
-  memcpy((char *)recvMsg + sizeof(rdmaPayloadMsg), srcAddr, size);
+  // Send a ConverseRdmaMsg to the other PE sending the array
+  ConverseRdmaMsg *payloadMsg = (ConverseRdmaMsg *)CmiAlloc(sizeof(ConverseRdmaMsg) + ncpyOpInfoSize + size);
 
-  // copy the destination ack into the recvMsg
-  memcpy((char *)recvMsg + sizeof(rdmaPayloadMsg) + size, destAck, destAckSize);
+  // copy the ncpyOpInfo into the recvMsg
+  memcpy((char *)payloadMsg + sizeof(ConverseRdmaMsg),
+         (char *)ncpyOpInfo,
+         ncpyOpInfoSize);
+
+  // copy the large array into the recvMsg
+  memcpy((char *)payloadMsg + sizeof(ConverseRdmaMsg) + ncpyOpInfoSize,
+         ncpyOpInfo->srcPtr,
+         size);
 
   // Invoke the source ack
-  ncpyAckHandlerFn(srcAck, srcPe, srcAddr);
+  ncpyOpInfo->ackMode = 1;
 
-  recvMsg->pe = destPe;
-  recvMsg->size = size;
-  recvMsg->destAddr = (char *)destAddr;
+  ncpyAckHandlerFn(ncpyOpInfo);
 
-  CmiSetHandler(recvMsg, put_data_handler_idx);
-  CmiSyncSendAndFree(destPe, sizeof(rdmaPayloadMsg) + size + destAckSize, recvMsg);
+  CmiSetHandler(payloadMsg, put_data_handler_idx);
+  CmiSyncSendAndFree(ncpyOpInfo->destPe,
+                     sizeof(ConverseRdmaMsg) + ncpyOpInfoSize + size,
+                     payloadMsg);
 }
 
 void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode) {}
index 935044831b839de98cfe3101abc8759143c8859f..e3b25a686c1e551c060b99f9ad48a777690a7326 100644 (file)
@@ -1,6 +1,8 @@
 #ifndef _CONV_RDMA_H
 #define _CONV_RDMA_H
 
+#include "cmirdmautils.h"
+
 typedef void (*RdmaSingleAckCallerFn)(void *cbPtr, int pe, const void *ptr);
 typedef void (*RdmaAckCallerFn)(void *token);
 
@@ -24,7 +26,7 @@ void CmiSetRdmaCommonInfo(void *info, const void *ptr, int size);
 int CmiGetRdmaCommonInfoSize(void);
 
 void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode);
-void CmiSetRdmaNcpyAck(RdmaSingleAckCallerFn fn);
+void CmiSetRdmaNcpyAck(RdmaAckCallerFn fn);
 
 /* CmiIssueRget initiates an RDMA read operation, transferring 'size' bytes of data from the address space of 'srcPe' to local address, 'destAddr'.
  * When the runtime invokes srcAck on the source (target), it indicates safety to overwrite or free the srcAddr buffer.
@@ -32,19 +34,9 @@ void CmiSetRdmaNcpyAck(RdmaSingleAckCallerFn fn);
  * destAddr buffer.
  */
 void CmiIssueRget(
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  int size);
+  unsigned short int *destMode);
 
 /* CmiIssueRput initiates an RDMA write operation, transferring 'size' bytes of data from the local address, 'srcAddr' to the address space of 'destPe'.
  * When the runtime invokes srcAck on the source (initiator), it indicates safety to overwrite or free the srcAddr buffer.
@@ -53,19 +45,9 @@ void CmiIssueRget(
  */
 
 void CmiIssueRput(
-  const void* destAddr,
-  void *destInfo,
-  void *destAck,
-  int destAckSize,
-  int destPe,
-  unsigned short int *destMode,
-  const void* srcAddr,
-  void *srcInfo,
-  void *srcAck,
-  int srcAckSize,
-  int srcPe,
+  NcpyOperationInfo *ncpyOpInfo,
   unsigned short int *srcMode,
-  int size);
+  unsigned short int *destMode);
 
 void CmiDeregisterMem(const void *ptr, void *info, int pe, unsigned short int mode);
 
index 2b8a442ba598bd7741c0829706a67636ec31a5ea..3956105a9518ed2dda4bb9134bce6f1ddfa239b5 100644 (file)
@@ -58,7 +58,7 @@
 #define CMI_NOOPTIMIZE
 #endif
 
-#include "conv-config.h"
+#include "conv-header.h"
 
 #define CMIALIGN(x,n)       (size_t)((~((size_t)n-1))&((x)+(n-1)))
 /*#define ALIGN8(x)        (size_t)((~7)&((x)+7)) */
@@ -677,53 +677,6 @@ extern void CmiInitHwlocTopology(void);
    for this node is longer than this many bytes. */
 int CmiLongSendQueue(int forNode,int longerThanBytes);
 
-/******** CMI: TYPE DEFINITIONS ********/
-
-typedef CMK_TYPEDEF_INT2      CmiInt2;
-typedef CMK_TYPEDEF_INT4      CmiInt4;
-typedef CMK_TYPEDEF_INT8      CmiInt8;
-typedef CMK_TYPEDEF_UINT2     CmiUInt2;
-typedef CMK_TYPEDEF_UINT4     CmiUInt4;
-typedef CMK_TYPEDEF_UINT8     CmiUInt8;
-#if CMK___int128_t_DEFINED
-typedef __int128_t            CmiInt16;
-typedef __uint128_t           CmiUInt16;
-#elif CMK___int128_DEFINED
-typedef __int128              CmiInt16;
-typedef __uint128     CmiUInt16;
-#endif
-
-#if defined(CMK_CUSTOM_FP_FORMAT)
-typedef CMK_TYPEDEF_FLOAT4    CmiFloat4;
-typedef CMK_TYPEDEF_FLOAT8    CmiFloat8;
-#else
-typedef float                 CmiFloat4;
-typedef double                CmiFloat8;
-#endif
-
-typedef void  *CmiCommHandle;
-typedef void (*CmiHandler)(void *msg);
-typedef void (*CmiHandlerEx)(void *msg,void *userPtr);
-
-typedef struct CMK_MSG_HEADER_BASIC CmiMsgHeaderBasic;
-typedef struct CMK_MSG_HEADER_EXT   CmiMsgHeaderExt;
-
-#define CmiMsgHeaderSizeBytes (sizeof(CmiMsgHeaderBasic))
-#define CmiExtHeaderSizeBytes (sizeof(CmiMsgHeaderExt))
-
-/* all common extra fields in BigSim message header */
-#define CMK_BIGSIM_FIELDS  CmiInt4 nd,n; double rt; CmiInt2 tID, hID; char t, flag; CmiInt2 ref; CmiInt4 msgID, srcPe;
-
-#ifndef CmiReservedHeaderSize
-typedef struct CMK_MSG_HEADER_BIGSIM_   CmiBlueGeneMsgHeader;
-#define CmiBlueGeneMsgHeaderSizeBytes (sizeof(CmiBlueGeneMsgHeader))
-#if CMK_BIGSIM_CHARM
-#  define CmiReservedHeaderSize   CmiBlueGeneMsgHeaderSizeBytes
-#else
-#  define CmiReservedHeaderSize   CmiExtHeaderSizeBytes
-#endif
-#endif
-
 /******** CMI, CSD: MANY LOW-LEVEL OPERATIONS ********/
 
 typedef struct {
index 695306af0d51615869d81d2646e8e8d3f8b791a5..c8ed8820c1d3e7c0862fa386c9e7d648478321b0 100644 (file)
@@ -232,7 +232,8 @@ CVHEADERS=cpthreads.h converse.h conv-trace.h conv-random.h conv-qd.h \
       memory-isomalloc.h debug-conv.h debug-conv++.h conv-autoconfig.h \
       conv-common.h conv-config.sh conv-config.h conv-mach.h conv-mach.sh \
       blue.h blue-conv.h bgconverse.h cmipool.h mempool.h cmiqueue.h \
-      cmidirect.h cmidirectmanytomany.h cmitls.h lrtslock.h conv-rdma.h lrts-common.h
+      cmidirect.h cmidirectmanytomany.h cmitls.h lrtslock.h conv-rdma.h lrts-common.h  \
+      conv-header.h
 
 # The .c files are there to be #included by clients whole
 # This is a bit unusual, but makes client linking simpler.
@@ -240,7 +241,7 @@ UTILHEADERS=pup.h pupf.h pup_c.h pup_stl.h pup_mpi.h pup_toNetwork.h pup_toNetwo
        ckimage.h ckdll.h ckhashtable.h ckbitvector.h cklists.h ckliststring.h \
        cksequence.h ckstatistics.h ckvector3d.h conv-lists.h ckcomplex.h \
        sockRoutines.h sockRoutines.c cmimemcpy.h simd.h SSE-Double.h SSE-Float.h \
-       crc32.h ckBIconfig.h rand48_replacement.h ckregex.h spanningTree.h
+       crc32.h ckBIconfig.h rand48_replacement.h ckregex.h spanningTree.h cmirdmautils.h
 
 CKHEADERS=ck.h ckstream.h objid.h envelope.h init.h qd.h charm.h charm++.h \
          ckfutures.h ckIgetControl.h cktiming.h debug-charm.h\
@@ -458,7 +459,7 @@ LIBCONV_UTIL=pup_util.o pup_toNetwork.o pup_toNetwork4.o            \
        ckimage.o ckdll.o ckhashtable.o sockRoutines.o                  \
        conv-lists.o persist-comm.o mempool.o graph.o \
        crc32.o  lz4.o partitioning_strategies.o hilbert.o \
-       spanningTree.o
+       spanningTree.o cmirdmautils.o
 
 LIBCONV_UTILF=pup_f.o
 
@@ -664,7 +665,7 @@ LIBCK_CORE=trace-common.o tracec.o tracef.o init.o register.o qd.o ck.o \
           NullLB.o LBSimulation.o modifyScheduler.o \
           charmProjections.o cktiming.o ckbitvector.o \
            pathHistory.o controlPoints.o arrayRedistributor.o cp_effects.o \
-                                        trace-controlPoints.o mpi-interoperate.o ckregex.o
+                                        trace-controlPoints.o mpi-interoperate.o ckregex.o cmirdmautils.o
 
 charm-core: converse $(L)/libck.a $(L)/libckf.a $(L)/libckmain.a $(L)/libckmainf.a
 charm-target: loadbalancers default_libs $(L)/libmpi-mainmodule.a tmgr
diff --git a/src/util/cmirdmautils.c b/src/util/cmirdmautils.c
new file mode 100644 (file)
index 0000000..5b2a053
--- /dev/null
@@ -0,0 +1,77 @@
+#include "cmirdmautils.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int getNcpyOpInfoTotalSize(
+  int srcLayerSize,
+  int srcAckSize,
+  int destLayerSize,
+  int destAckSize) {
+  return sizeof(NcpyOperationInfo) + srcLayerSize + destLayerSize + srcAckSize + destAckSize;
+}
+
+void setNcpyOpInfo(
+    const void *srcPtr,
+    char *srcLayerInfo,
+    int srcLayerSize,
+    char *srcAck,
+    int srcAckSize,
+    int srcPe,
+    const void *srcRef,
+    const void *destPtr,
+    char *destLayerInfo,
+    int destLayerSize,
+    char *destAck,
+    int destAckSize,
+    int destPe,
+    const void *destRef,
+    int size,
+    NcpyOperationInfo *ncpyOpInfo) {
+
+  // memcpy srcLayerInfo
+  memcpy((char *)ncpyOpInfo + sizeof(NcpyOperationInfo), srcLayerInfo, srcLayerSize);
+  ncpyOpInfo->srcLayerInfo = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
+  // memcpy srcAckInfo
+  memcpy(ncpyOpInfo->srcLayerInfo + srcLayerSize, srcAck, srcAckSize);
+  ncpyOpInfo->srcAck = ncpyOpInfo->srcLayerInfo + srcLayerSize;
+
+  // memcpy destLayerInfo
+  memcpy(ncpyOpInfo->srcAck + srcAckSize, destLayerInfo, destLayerSize);
+  ncpyOpInfo->destLayerInfo = ncpyOpInfo->srcAck + srcAckSize;
+
+  // memcpy destAck Info
+  memcpy(ncpyOpInfo->destLayerInfo + destLayerSize, destAck, destAckSize);
+  ncpyOpInfo->destAck = ncpyOpInfo->destLayerInfo + destLayerSize;
+
+  ncpyOpInfo->srcPtr = srcPtr;
+  ncpyOpInfo->srcPe = srcPe;
+  ncpyOpInfo->srcRef = srcRef;
+  ncpyOpInfo->srcLayerSize = srcLayerSize;
+  ncpyOpInfo->srcAckSize = srcAckSize;
+  ncpyOpInfo->origSrcLayerInfoPtr = srcLayerInfo;
+
+  ncpyOpInfo->destPtr = destPtr;
+  ncpyOpInfo->destPe = destPe;
+  ncpyOpInfo->destRef = destRef;
+  ncpyOpInfo->destLayerSize = destLayerSize;
+  ncpyOpInfo->destAckSize = destAckSize;
+  ncpyOpInfo->origDestLayerInfoPtr = destLayerInfo;
+
+  ncpyOpInfo->ackMode = 0;
+  ncpyOpInfo->freeMe  = 1;
+
+  ncpyOpInfo->ncpyOpInfoSize = sizeof(NcpyOperationInfo) + srcLayerSize + destLayerSize + srcAckSize + destAckSize;
+  ncpyOpInfo->size = size;
+}
+
+
+void resetNcpyOpInfoPointers(NcpyOperationInfo *ncpyOpInfo) {
+  ncpyOpInfo->srcLayerInfo = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
+
+  ncpyOpInfo->srcAck = (char *)(ncpyOpInfo->srcLayerInfo) + ncpyOpInfo->srcLayerSize;
+
+  ncpyOpInfo->destLayerInfo = (char *)(ncpyOpInfo->srcAck) + ncpyOpInfo->srcAckSize;
+
+  ncpyOpInfo->destAck = (char *)(ncpyOpInfo->destLayerInfo) + ncpyOpInfo->destLayerSize;
+}
diff --git a/src/util/cmirdmautils.h b/src/util/cmirdmautils.h
new file mode 100644 (file)
index 0000000..c70e3d2
--- /dev/null
@@ -0,0 +1,70 @@
+#ifndef _CKRDMAUTILS_H
+#define _CKRDMAUTILS_H
+
+#include "conv-header.h"
+
+// Structure that can be used across layers
+typedef struct ncpystruct{
+
+  // Used in the MPI layer
+#if CMK_CONVERSE_MPI
+  char core[CmiMsgHeaderSizeBytes];
+  int tag;
+#endif
+
+  const void *srcPtr;
+  int srcPe;
+  char *origSrcLayerInfoPtr;
+  char *srcLayerInfo;
+  int srcLayerSize;
+  char *srcAck;
+  int srcAckSize;
+  const void *srcRef;
+
+  const void *destPtr;
+  int destPe;
+  char *origDestLayerInfoPtr;
+  char *destLayerInfo;
+  int destLayerSize;
+  char *destAck;
+  int destAckSize;
+  const void *destRef;
+
+  // Variables used for ack handling
+  int ackMode; // 0 for call both src and dest acks
+               // 1 for call just src ack
+               // 2 for call just dest ack
+  int freeMe; // 1 for free, 0 for do not free
+
+  int ncpyOpInfoSize;
+  int size;
+
+}NcpyOperationInfo;
+
+int getNcpyOpInfoTotalSize(
+  int srcLayerSize,
+  int srcAckSize,
+  int destLayerSize,
+  int destAckSize);
+
+void setNcpyOpInfo(
+    const void *srcPtr,
+    char *srcLayerInfo,
+    int srcLayerSize,
+    char *srcAck,
+    int srcAckSize,
+    int srcPe,
+    const void *srcRef,
+    const void *destPtr,
+    char *destLayerInfo,
+    int destLayerSize,
+    char *destAck,
+    int destAckSize,
+    int destPe,
+    const void *destRef,
+    int size,
+    NcpyOperationInfo *ncpyOpInfo);
+
+
+void resetNcpyOpInfoPointers(NcpyOperationInfo *ncpyOpInfo);
+#endif