Zerocopy Direct API: Elevate CMA API usage to the charm layer 09/4209/6
authorNitin Bhat <nbhat4@illinois.edu>
Fri, 18 May 2018 20:37:20 +0000 (15:37 -0500)
committerNitin Bhat <nbhat4@illinois.edu>
Mon, 2 Jul 2018 22:24:50 +0000 (17:24 -0500)
This patch also includes other refactoring for the Zerocopy API that
is required for upcoming changes in order to make its usage more flexible

Change-Id: I66ee2250848992f073771941bbf582d3ae884c44

src/arch/util/machine-rdma.h
src/ck-core/ckrdma.C
src/ck-core/ckrdma.h
src/conv-core/conv-rdma.c
src/conv-core/conv-rdma.h

index 244716530947cad85352e11ff5d9def72341817d..6e8755cd7e0f11e0d0f4468ca1cf77d273fe3638 100644 (file)
@@ -288,19 +288,6 @@ void CmiIssueRget(
   unsigned short int *destMode,
   int size) {
 
-#if CMK_USE_CMA
-  // check if remote PE is on the same physical node
-  if(cma_works && CmiPeOnSamePhysicalNode(srcPe, destPe)) {
-
-    CmiIssueRgetUsingCMA(srcAddr, srcInfo, srcPe,
-                         destAddr, destInfo, destPe,
-                         size);
-    // directy invoke the acks
-    ncpyAckHandlerFn(srcAck, srcPe, srcAddr);
-    ncpyAckHandlerFn(destAck, destPe, destAddr);
-    return;
-  }
-#endif
   // Use network RDMA for a PE on a remote host
   LrtsIssueRget(srcAddr,
                 (char*)srcInfo + CmiGetRdmaCommonInfoSize(),
@@ -333,20 +320,6 @@ void CmiIssueRput(
   unsigned short int *srcMode,
   int size) {
 
-#if CMK_USE_CMA
-  // check if remote PE is on the same physical node
-  if(cma_works && CmiPeOnSamePhysicalNode(srcPe, destPe)) {
-
-    CmiIssueRputUsingCMA(destAddr, destInfo, destPe,
-                         srcAddr, srcInfo, srcPe,
-                         size);
-
-    // directy invoke the acks
-    ncpyAckHandlerFn(srcAck, srcPe, srcAddr);
-    ncpyAckHandlerFn(destAck, destPe, destAddr);
-    return;
-  }
-#endif
   // Use network RDMA for a PE on a remote host
   LrtsIssueRput(destAddr,
                 (char*)destInfo + CmiGetRdmaCommonInfoSize(),
index 52b1fbdc17af4065495c9d0a689837ff0d30fe96..34f8e66a2ec6db03ec9bed72d23ea0078acab2a4 100644 (file)
@@ -333,45 +333,53 @@ void CkRdmaAckHandler(void *cbPtr, int pe, const void *ptr) {
 #endif
 }
 
-// Perform a nocopy put operation into the passed destination using this source
-void CkNcpyBuffer::put(CkNcpyBuffer &destination){
-  if(mode == CK_BUFFER_NOREG || destination.mode == CK_BUFFER_NOREG) {
-    CkAbort("Cannot perform RDMA operations in CK_BUFFER_NOREG mode\n");
-  }
-
-  // Check that the count for both the counters matches
-  CkAssert(cnt <= destination.cnt);
-
-  // Check that this object is local when put is called
-  CkAssert(CkNodeOf(pe) == CkMyNode());
-
-  if(CkNodeOf(destination.pe) == CkMyNode()) {
-    // memcpy the data from the source buffer into the destination buffer
-    memcpy((void *)destination.ptr, ptr, cnt);
-
-    //Invoke the source callback
-    cb.send(sizeof(void *), &ptr);
+// Returns ncpyTransferMode::MEMCPY if both the PEs are the same and memcpy can be used
+// Returns ncpyTransferMode::CMA if both the PEs are in the same physical node and CMA can be used
+// Returns ncpyTransferMode::RDMA if RDMA needs to be used
+ncpyTransferMode findTransferMode(int srcPe, int destPe) {
+  if(CmiNodeOf(srcPe)==CmiNodeOf(destPe))
+    return ncpyTransferMode::MEMCPY;
+#if CMK_USE_CMA
+  else if(CmiDoesCMAWork() && CmiPeOnSamePhysicalNode(srcPe, destPe))
+    return ncpyTransferMode::CMA;
+#endif
+  else
+    return ncpyTransferMode::RDMA;
+}
 
-    //Invoke the destination callback
-    destination.cb.send(sizeof(void *), &destination.ptr);
+// Get Methods
+void CkNcpyBuffer::memcpyGet(CkNcpyBuffer &source) {
+  // memcpy the data from the source buffer into the destination buffer
+  memcpy((void *)ptr, source.ptr, cnt);
+}
 
-  } else {
+#if CMK_USE_CMA
+void CkNcpyBuffer::cmaGet(CkNcpyBuffer &source) {
+  CmiIssueRgetUsingCMA(source.ptr,
+                       source.layerInfo,
+                       source.pe,
+                       ptr,
+                       layerInfo,
+                       pe,
+                       cnt);
+}
+#endif
 
-    // Issue the Rput call
-    CmiIssueRput(destination.ptr,
-                 destination.layerInfo,
-                 &destination.cb,
-                 sizeof(CkCallback),
-                 destination.pe,
-                 &destination.mode,
-                 ptr,
-                 layerInfo,
-                 &cb,
-                 sizeof(CkCallback),
-                 CkMyPe(),
-                 &mode,
-                 cnt);
-  }
+void CkNcpyBuffer::rdmaGet(CkNcpyBuffer &source) {
+  // Issue the Rget call
+  CmiIssueRget(source.ptr,
+               source.layerInfo,
+               &source.cb,
+               sizeof(CkCallback),
+               source.pe,
+               &source.mode,
+               ptr,
+               layerInfo,
+               &cb,
+               sizeof(CkCallback),
+               CkMyPe(),
+               &mode,
+               cnt);
 }
 
 // Perform a nocopy get operation into this destination using the passed source
@@ -380,39 +388,112 @@ void CkNcpyBuffer::get(CkNcpyBuffer &source){
     CkAbort("Cannot perform RDMA operations in CK_BUFFER_NOREG mode\n");
   }
 
-  // Check that the count for both the counters matches
+  // Check that the source buffer fits into the destination buffer
   CkAssert(source.cnt <= cnt);
 
   // Check that this object is local when get is called
   CkAssert(CkNodeOf(pe) == CkMyNode());
 
-  //Check if it is a within-process sending
-  if(CmiNodeOf(source.pe) == CkMyNode()) {
+  ncpyTransferMode transferMode = findTransferMode(source.pe, pe);
 
-    // memcpy the data from the source buffer into the destination buffer
-    memcpy((void *)ptr, source.ptr, cnt);
+  //Check if it is a within-process sending
+  if(transferMode == ncpyTransferMode::MEMCPY) {
+    memcpyGet(source);
 
     //Invoke the receiver's callback
     cb.send(sizeof(void *), &ptr);
 
     //Invoke the sender's callback
     source.cb.send(sizeof(void *), &source.ptr);
+#if CMK_USE_CMA
+  } else if(transferMode == ncpyTransferMode::CMA) {
+
+    cmaGet(source);
 
+    //Invoke the receiver's callback
+    cb.send(sizeof(void *), &ptr);
+
+    //Invoke the sender's callback
+    source.cb.send(sizeof(void *), &source.ptr);
+#endif
+  } else if (transferMode == ncpyTransferMode::RDMA) {
+    rdmaGet(source);
   } else {
+    CkAbort("Invalid ncpyTransferMode");
+  }
+}
 
-    // Issue the Rget call
-    CmiIssueRget(source.ptr,
-                 source.layerInfo,
-                 &source.cb,
-                 sizeof(CkCallback),
-                 source.pe,
-                 &source.mode,
-                 ptr,
-                 layerInfo,
-                 &cb,
-                 sizeof(CkCallback),
-                 CkMyPe(),
-                 &mode,
-                 cnt);
+// Put Methods
+void CkNcpyBuffer::memcpyPut(CkNcpyBuffer &destination) {
+  // memcpy the data from the source buffer into the destination buffer
+  memcpy((void *)destination.ptr, ptr, cnt);
+}
+
+#if CMK_USE_CMA
+void CkNcpyBuffer::cmaPut(CkNcpyBuffer &destination) {
+  CmiIssueRputUsingCMA(destination.ptr,
+                       destination.layerInfo,
+                       destination.pe,
+                       ptr,
+                       layerInfo,
+                       pe,
+                       cnt);
+}
+#endif
+
+void CkNcpyBuffer::rdmaPut(CkNcpyBuffer &destination) {
+  // Issue the Rput call
+  CmiIssueRput(destination.ptr,
+               destination.layerInfo,
+               &destination.cb,
+               sizeof(CkCallback),
+               destination.pe,
+               &destination.mode,
+               ptr,
+               layerInfo,
+               &cb,
+               sizeof(CkCallback),
+               CkMyPe(),
+               &mode,
+               cnt);
+}
+
+// Perform a nocopy put operation into the passed destination using this source
+void CkNcpyBuffer::put(CkNcpyBuffer &destination){
+  if(mode == CK_BUFFER_NOREG || destination.mode == CK_BUFFER_NOREG) {
+    CkAbort("Cannot perform RDMA operations in CK_BUFFER_NOREG mode\n");
+  }
+  // Check that the source buffer fits into the destination buffer
+  CkAssert(cnt <= destination.cnt);
+
+  // Check that this object is local when put is called
+  CkAssert(CkNodeOf(pe) == CkMyNode());
+
+  ncpyTransferMode transferMode = findTransferMode(pe, destination.pe);
+
+  //Check if it is a within-process sending
+  if(transferMode == ncpyTransferMode::MEMCPY) {
+    memcpyPut(destination);
+
+    //Invoke the source callback
+    cb.send(sizeof(void *), &ptr);
+
+    //Invoke the destination callback
+    destination.cb.send(sizeof(void *), &destination.ptr);
+
+#if CMK_USE_CMA
+  } else if(transferMode == ncpyTransferMode::CMA) {
+    cmaPut(destination);
+
+    //Invoke the source callback
+    cb.send(sizeof(void *), &ptr);
+
+    //Invoke the destination callback
+    destination.cb.send(sizeof(void *), &destination.ptr);
+#endif
+  } else if (transferMode == ncpyTransferMode::RDMA) {
+    rdmaPut(destination);
+  } else {
+    CkAbort("Invalid ncpyTransferMode");
   }
 }
index 0a1437f4046fd4fcde373f8abe1c2f8ba575fa37..585a87b6da6e8df4298fa172d3b7aaf834294494 100644 (file)
@@ -187,9 +187,19 @@ class CkNcpyBuffer{
 
   void setMode(unsigned short int mode_) { mode = mode_; }
 
-  void put(CkNcpyBuffer &destination);
+  void memcpyGet(CkNcpyBuffer &source);
+  void memcpyPut(CkNcpyBuffer &destination);
+
+#if CMK_USE_CMA
+  void cmaGet(CkNcpyBuffer &source);
+  void cmaPut(CkNcpyBuffer &destination);
+#endif
+
+  void rdmaGet(CkNcpyBuffer &source);
+  void rdmaPut(CkNcpyBuffer &destination);
 
   void get(CkNcpyBuffer &source);
+  void put(CkNcpyBuffer &destination);
 
   // Deregister(Unpin) the memory that is registered for the buffer
   void deregisterMem() {
@@ -218,4 +228,6 @@ class CkNcpyBuffer{
 
 };
 
+enum class ncpyTransferMode : char { MEMCPY, CMA, RDMA };
+
 #endif
index b9fd727b9edbba6a89a0bd36a784dd86aff02103..050230de97be11050f8df55ca0dea9ee64be8f88 100644 (file)
@@ -3,10 +3,6 @@
  */
 #include "converse.h"
 
-#if CMK_USE_CMA
-extern int cma_works;
-#endif
-
 // Methods required to keep the Nocopy Direct API functional on non-LRTS layers
 #if !CMK_USE_LRTS
 void CmiSetRdmaCommonInfo(void *info, const void *ptr, int size) {
@@ -103,21 +99,6 @@ void CmiIssueRget(
   unsigned short int *destMode,
   int size) {
 
-#if CMK_USE_CMA
-  // check if remote PE is on the same physical node
-  if(cma_works && CmiPeOnSamePhysicalNode(srcPe, destPe)) {
-
-    CmiIssueRgetUsingCMA(srcAddr, srcInfo, srcPe,
-                         destAddr, destInfo, destPe,
-                         size);
-
-    // directy invoke the acks
-    ncpyAckHandlerFn(srcAck, srcPe, srcAddr);
-    ncpyAckHandlerFn(destAck, destPe, destAddr);
-    return;
-  }
-#endif
-
   // Send a getRequestMsg to other PE requesting it to send the array
   getRequestMsg *getReqMsg = (getRequestMsg *)CmiAlloc(sizeof(getRequestMsg) + srcAckSize + destAckSize);
   getReqMsg->srcPe = srcPe;
@@ -154,21 +135,6 @@ void CmiIssueRput(
   unsigned short int *srcMode,
   int size) {
 
-#if CMK_USE_CMA
-  // check if remote PE is on the same physical node
-  if(cma_works && CmiPeOnSamePhysicalNode(srcPe, destPe)) {
-
-    CmiIssueRputUsingCMA(destAddr, destInfo, destPe,
-                         srcAddr, srcInfo, srcPe,
-                         size);
-
-    // directy invoke the acks
-    ncpyAckHandlerFn(srcAck, srcPe, srcAddr);
-    ncpyAckHandlerFn(destAck, destPe, destAddr);
-    return;
-  }
-#endif
-
   // Send a rdmaPayloadMsg to the other PE sending the array
   rdmaPayloadMsg *recvMsg = (rdmaPayloadMsg *)CmiAlloc(sizeof(rdmaPayloadMsg) + size + destAckSize);
 
index 4078279e862ceccd4d8e77a2acc69ae93366b634..935044831b839de98cfe3101abc8759143c8859f 100644 (file)
@@ -92,6 +92,8 @@ void CmiIssueRputUsingCMA(
 // Allocation from pool
 void *CmiRdmaAlloc(int size);
 
+int CmiDoesCMAWork(void);
+
 #if !CMK_ONESIDED_DIRECT_IMPL
 // Function declaration used for the generic implementation of the Nocopy Direct API
 void CmiOnesidedDirectInit(void);