Keep old interface of Cmidirect and added gemini cmidirect
authorYanhua Sun <yanhuas@jyc1.(none)>
Wed, 8 Feb 2012 06:18:31 +0000 (00:18 -0600)
committerYanhua Sun <yanhuas@jyc1.(none)>
Wed, 8 Feb 2012 06:18:31 +0000 (00:18 -0600)
src/arch/gemini_gni/machine-cmidirect.c
src/arch/gemini_gni/machine.c
src/conv-core/cmidirect.h
tests/charm++/pingpong/pingpong.C

index f7479426ffe11c797f4a4877acfcf8b9b55aee8e..ad4a73b0da6149a06af51ccfdffb267784899efa 100644 (file)
   Yanhua Sun, 2/5/2012
 */
 
-#define     CMI_DIRECT_DEBUG    1
+//#define     CMI_DIRECT_DEBUG    0
 #include "cmidirect.h"
 
-static void printHandle(CmiDirectUserHandle *userHandle)
+static void printHandle(CmiDirectUserHandle *userHandle, char *s)
 {
-#if CMI_DIRECT_DEBUG
-    CmiPrintf( "[%d] sender (%p, %lld, %lld), remote(%p, %lld, %lld)\n", CmiMyPe(), userHandle->senderBuf, userHandle->senderMdh.qword1, userHandle->senderMdh.qword2, 
-        userHandle->recverBuf, userHandle->recverMdh.qword1, userHandle->recverMdh.qword2);
-#endif
+    CmiPrintf( "[%d]%s(%p)(%p,%p,%p)==>(%p,%p,%p)(%d)(%p,%p)\n", CmiMyPe(), s, userHandle, userHandle->localBuf, userHandle->localMdh.qword1, userHandle->localMdh.qword2, 
+        userHandle->remoteBuf, userHandle->remoteMdh.qword1, userHandle->remoteMdh.qword2, userHandle->transSize, userHandle->callbackFnPtr, userHandle->callbackData );
 }
 
 /**
  To be called on the receiver to create a handle and return its number
 **/
-CmiDirectUserHandle* CmiDirect_createHandle(int senderNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue) {
+CmiDirectUserHandle CmiDirect_createHandle(int localNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue) {
 
     gni_return_t            status = GNI_RC_SUCCESS;
-    CmiDirectUserHandle *userHandle = malloc(sizeof(CmiDirectUserHandle));
-    userHandle->handle=1; 
-    userHandle->senderNode=senderNode;
-    userHandle->recverNode=_Cmi_mynode;
-    userHandle->recverBufSize=recvBufSize;
-    userHandle->recverBuf=recvBuf;
-    userHandle->initialValue=initialValue;
-    userHandle->callbackFnPtr=callbackFnPtr;
-    userHandle->callbackData=callbackData;
-
-    if(recvBufSize < SMSG_MAX_MSG)
+    CmiDirectUserHandle userHandle;
+    userHandle.handle=1; 
+    userHandle.localNode=localNode;
+    userHandle.remoteNode=_Cmi_mynode;
+    userHandle.transSize=recvBufSize;
+    userHandle.remoteBuf=recvBuf;
+    userHandle.initialValue=initialValue;
+    userHandle.callbackFnPtr=callbackFnPtr;
+    userHandle.callbackData=callbackData;
+
+    if(recvBufSize <= SMSG_MAX_MSG)
     {
-        status = MEMORY_REGISTER(onesided_hnd, nic_hndl, userHandle->recverBuf, recvBufSize, &userHandle->recverMdh, &omdh);
+        status = MEMORY_REGISTER(onesided_hnd, nic_hndl, userHandle.remoteBuf, recvBufSize, &(userHandle.remoteMdh), &omdh);
     }
-    else if(IsMemHndlZero((GetMemHndl(userHandle->recverBuf)))){
-        status = registerMempool(userHandle->recverBuf);
-        userHandle->recverMdh = GetMemHndl(userHandle->recverBuf);
+    else if(IsMemHndlZero((GetMemHndl(userHandle.remoteBuf)))){
+        status = registerMempool(userHandle.remoteBuf);
+        userHandle.remoteMdh = GetMemHndl(userHandle.remoteBuf);
     } else
-        userHandle->recverMdh = GetMemHndl(userHandle->recverBuf);
+        userHandle.remoteMdh = GetMemHndl(userHandle.remoteBuf);
     if(status != GNI_RC_SUCCESS) {
-        userHandle->recverMdh.qword1 = 0;
-        userHandle->recverMdh.qword2 = 0;
+        userHandle.remoteMdh.qword1 = 0;
+        userHandle.remoteMdh.qword2 = 0;
     }
 
 #if CMI_DIRECT_DEBUG
-    printHandle(userHandle);
+    printHandle(userHandle, "Create Handler");
 #endif
     return userHandle;
 }
 
 /****
- To be called on the sender to attach the sender's buffer to this handle
+ To be called on the local to attach the local's buffer to this handle
 ******/
 
 void CmiDirect_assocLocalBuffer(CmiDirectUserHandle *userHandle,void *sendBuf,int sendBufSize) {
 
     /* one-sided primitives would require registration of memory */
-    /* with two-sided primitives we just record the sender buf in the handle */
     gni_return_t            status = GNI_RC_SUCCESS;
-    userHandle->senderBuf=sendBuf;
-    if(userHandle->recverBufSize < SMSG_MAX_MSG)
+    
+    userHandle->localBuf=sendBuf;
+    if(userHandle->transSize <= SMSG_MAX_MSG)
     {
-        status = MEMORY_REGISTER(onesided_hnd, nic_hndl, userHandle->senderBuf, userHandle->recverBufSize, &userHandle->senderMdh, &omdh);
+        status = MEMORY_REGISTER(onesided_hnd, nic_hndl, userHandle->localBuf, userHandle->transSize, &userHandle->localMdh, &omdh);
     }
-    else if(IsMemHndlZero((GetMemHndl(userHandle->senderBuf)))){
-        status = registerMempool(userHandle->senderBuf);
-        userHandle->senderMdh = GetMemHndl(userHandle->senderBuf);
+    else if(IsMemHndlZero((GetMemHndl(userHandle->localBuf)))){
+        status = registerMempool(userHandle->localBuf);
+        userHandle->localMdh = GetMemHndl(userHandle->localBuf);
     } else
-        userHandle->senderMdh = GetMemHndl(userHandle->senderBuf);
+        userHandle->localMdh = GetMemHndl(userHandle->localBuf);
+   
+    if(status != GNI_RC_SUCCESS) {
+        userHandle->localMdh.qword1 = 0;
+        userHandle->localMdh.qword2 = 0;
+    }
 
 #if CMI_DIRECT_DEBUG
-    CmiPrintf("Assciate");
-    printHandle(userHandle);
+    printHandle(userHandle, "Associate Handler");
 #endif
 }
 
 /****
-To be called on the sender to do the actual data transfer
+To be called on the local to do the actual data transfer
 ******/
 void CmiDirect_put(CmiDirectUserHandle *userHandle) {
 
-#if CMI_DIRECT_DEBUG
-    CmiPrintf("[%d] call put\n", CmiMyPe());
-    printHandle(userHandle);
-#endif
+    gni_post_descriptor_t *pd;
+
 #if USE_LRTS_MEMPOOL
-    if (userHandle->recverNode== CmiMyNode()) {
-        CmiMemcpy(userHandle->recverBuf,userHandle->senderBuf,userHandle->recverBufSize);
+    if (userHandle->remoteNode== CmiMyNode()) {
+        CmiMemcpy(userHandle->remoteBuf,userHandle->localBuf,userHandle->transSize);
         (*(userHandle->callbackFnPtr))(userHandle->callbackData);
     } else {
-        gni_post_descriptor_t *pd;
         gni_return_t status;
         RDMA_REQUEST        *rdma_request_msg;
         MallocPostDesc(pd);
-        if(userHandle->recverBufSize <= 2048)
+        if(userHandle->transSize <= LRTS_GNI_RDMA_THRESHOLD)
             pd->type            = GNI_POST_FMA_PUT;
         else
             pd->type            = GNI_POST_RDMA_PUT;
         pd->cq_mode         = GNI_CQMODE_GLOBAL_EVENT;
         pd->dlvr_mode       = GNI_DLVMODE_PERFORMANCE;
-        pd->length          = userHandle->recverBufSize;
-        pd->local_addr      = (uint64_t) (userHandle->senderBuf);
-        pd->local_mem_hndl  = userHandle->senderMdh; 
-        pd->remote_addr     = (uint64_t)(userHandle->recverBuf);
-        pd->remote_mem_hndl = userHandle->recverMdh;
+        pd->length          = userHandle->transSize;
+        pd->local_addr      = (uint64_t) (userHandle->localBuf);
+        pd->local_mem_hndl  = userHandle->localMdh; 
+        pd->remote_addr     = (uint64_t)(userHandle->remoteBuf);
+        pd->remote_mem_hndl = userHandle->remoteMdh;
         pd->src_cq_hndl     = 0;
         pd->rdma_mode       = 0;
-        pd->first_operand   = (uint64_t) userHandle-> callbackFnPtr;
-        pd->second_operand  = (uint64_t) userHandle-> callbackData;
+        pd->first_operand   = (uint64_t) (userHandle->callbackFnPtr);
+        pd->second_operand  = (uint64_t) (userHandle->callbackData);
         pd->amo_cmd         = 1;
         if(pd->type == GNI_POST_RDMA_PUT) 
-            status = GNI_PostRdma(ep_hndl_array[userHandle->recverNode], pd);
+            status = GNI_PostRdma(ep_hndl_array[userHandle->remoteNode], pd);
         else
-            status = GNI_PostFma(ep_hndl_array[userHandle->recverNode],  pd);
-        printDesc(pd);
+            status = GNI_PostFma(ep_hndl_array[userHandle->remoteNode],  pd);
         if(status == GNI_RC_ERROR_RESOURCE|| status == GNI_RC_ERROR_NOMEM )
         {
             MallocRdmaRequest(rdma_request_msg);
-            rdma_request_msg->destNode = userHandle->recverNode;
+            rdma_request_msg->destNode = userHandle->remoteNode;
             rdma_request_msg->pd = pd;
 #if CMK_SMP
             PCQueuePush(sendRdmaBuf, (char*)rdma_request_msg);
@@ -138,7 +135,7 @@ void CmiDirect_put(CmiDirectUserHandle *userHandle) {
             }
 #endif
         }else
-            GNI_RC_CHECK("AFter posting", status);
+            GNI_RC_CHECK("CMI_Direct_PUT", status);
     }
 #else
     CmiPrintf("Normal Send in CmiDirect Put\n");
@@ -146,44 +143,49 @@ void CmiDirect_put(CmiDirectUserHandle *userHandle) {
 #endif
 
 #if CMI_DIRECT_DEBUG
-    CmiPrintf("[%d] RDMA put addr %p %d to recverNode %d receiver addr %p callback %p callbackdata %p\n",CmiMyPe(),userHandle->senderBuf,userHandle->recverBufSize, userHandle->recverNode,userHandle->recverBuf, userHandle->callbackFnPtr, userHandle->callbackData);
+    printHandle(userHandle, "After Direct_put");
+    CmiPrintf("[%d] RDMA put %d,%d bytes addr %p to remoteNode %d:%p \n\n",CmiMyPe(), userHandle->transSize, pd->length, (void*)(pd->local_addr), userHandle->remoteNode, (void*) (pd->remote_addr));
 #endif
 
 }
 
-// needs to figure out what is sender/recver
+// needs to figure out what is local/remote
 void CmiDirect_get(CmiDirectUserHandle *userHandle) {
+
+    gni_post_descriptor_t *pd;
+
 #if USE_LRTS_MEMPOOL
-    if (userHandle->recverNode== _Cmi_mynode) {
-        CmiMemcpy(userHandle->recverBuf,userHandle->senderBuf,userHandle->recverBufSize);
+    if (userHandle->remoteNode== CmiMyNode()) {
+        CmiMemcpy(userHandle->remoteBuf,userHandle->localBuf,userHandle->transSize);
         (*(userHandle->callbackFnPtr))(userHandle->callbackData);
     } else {
-        gni_post_descriptor_t *pd;
         gni_return_t status;
         RDMA_REQUEST        *rdma_request_msg;
         MallocPostDesc(pd);
-        if(userHandle->recverBufSize <= 2048)
+        if(userHandle->transSize <= LRTS_GNI_RDMA_THRESHOLD)
             pd->type            = GNI_POST_FMA_GET;
         else
             pd->type            = GNI_POST_RDMA_GET;
         pd->cq_mode         = GNI_CQMODE_GLOBAL_EVENT;
         pd->dlvr_mode       = GNI_DLVMODE_PERFORMANCE;
-        pd->length          = userHandle->recverBufSize;
-        pd->local_addr      = (uint64_t) (userHandle->recverBuf);
-        pd->local_mem_hndl  = userHandle->recverMdh; 
-        pd->remote_addr     = (uint64_t)(userHandle->senderBuf);
-        pd->remote_mem_hndl = userHandle->senderMdh;
+        pd->length          = userHandle->transSize;
+        pd->local_addr      = (uint64_t) (userHandle->localBuf);
+        pd->local_mem_hndl  = userHandle->localMdh; 
+        pd->remote_addr     = (uint64_t)(userHandle->remoteBuf);
+        pd->remote_mem_hndl = userHandle->remoteMdh;
         pd->src_cq_hndl     = 0;
         pd->rdma_mode       = 0;
-
-        if(pd->type == GNI_POST_RDMA_PUT) 
-            status = GNI_PostRdma(ep_hndl_array[userHandle->senderNode], pd);
+        pd->first_operand   = (uint64_t) (userHandle->callbackFnPtr);
+        pd->second_operand  = (uint64_t) (userHandle->callbackData);
+        pd->amo_cmd         = 2;
+        if(pd->type == GNI_POST_RDMA_GET) 
+            status = GNI_PostRdma(ep_hndl_array[userHandle->remoteNode], pd);
         else
-            status = GNI_PostFma(ep_hndl_array[userHandle->senderNode],  pd);
+            status = GNI_PostFma(ep_hndl_array[userHandle->remoteNode],  pd);
         if(status == GNI_RC_ERROR_RESOURCE|| status == GNI_RC_ERROR_NOMEM )
         {
             MallocRdmaRequest(rdma_request_msg);
-            rdma_request_msg->destNode = userHandle->senderNode;
+            rdma_request_msg->destNode = userHandle->remoteNode;
             rdma_request_msg->pd = pd;
 #if CMK_SMP
             PCQueuePush(sendRdmaBuf, (char*)rdma_request_msg);
@@ -197,17 +199,18 @@ void CmiDirect_get(CmiDirectUserHandle *userHandle) {
             }
 #endif
         }else
-            GNI_RC_CHECK("AFter posting", status);
+            GNI_RC_CHECK("CMI_Direct_GET", status);
     }
 #else
-    CmiPrintf("Normal Send in CmiDirect Put\n");
+    CmiPrintf("Normal Send in CmiDirect Get\n");
     CmiAbort("");
 #endif
 
 #if CMI_DIRECT_DEBUG
-        CmiPrintf("[%d] RDMA put addr %p %d to recverNode %d receiver addr %p callback %p callbackdata %p\n",CmiMyPe(),userHandle->senderBuf,userHandle->recverBufSize, userHandle->recverNode,userHandle->recverBuf, userHandle->callbackFnPtr, userHandle->callbackData);
+    CmiPrintf("[%d] RDMA get %d,%d bytes addr %p to remoteNode %d:%p \n\n",CmiMyPe(), userHandle->transSize, pd->length, (void*)(pd->local_addr), userHandle->remoteNode, (void*) (pd->remote_addr));
 #endif
 
+
 }
 
 /**** up to the user to safely call this */
@@ -218,6 +221,7 @@ void CmiDirect_deassocLocalBuffer(CmiDirectUserHandle *userHandle) {
 
 /**** up to the user to safely call this */
 void CmiDirect_destroyHandle(CmiDirectUserHandle *userHandle) {
+    free(userHandle);
 }
 
 /**** Should not be called the first time *********/
index 993d4d095cb911403b1a75f620698141f258886e..84ade66c75e7d67852d180faf5d1c0b4ed683406 100644 (file)
@@ -417,7 +417,7 @@ static RDMA_REQUEST         *rdma_freelist = NULL;
 /* reuse gni_post_descriptor_t */
 static gni_post_descriptor_t *post_freelist=0;
 
-#if !CMK_SMP
+#if  !CMK_SMP
 #define FreePostDesc(d)       \
     (d)->next_descr = post_freelist;\
     post_freelist = d;
@@ -1328,9 +1328,7 @@ static void PumpNetworkSmsg()
 
 static void printDesc(gni_post_descriptor_t *pd)
 {
-    printf(" addr=%p, ", pd->local_addr); 
-    printf(" remote addr=%p, ", pd->remote_addr);
-    printf(" local %lld %lld, remote %lld, %lld\n", pd->local_mem_hndl.qword1, pd->local_mem_hndl.qword2, pd->remote_mem_hndl.qword1, pd->remote_mem_hndl.qword2); 
+    printf(" Descriptor (%p===>%p)(%d)\n", pd->local_addr, pd->remote_addr, pd->length); 
 }
 
 // for BIG_MSG called on receiver side for receiving control message
@@ -1390,7 +1388,7 @@ static void getLargeMsgRequest(void* header, uint64_t inst_id )
     }
     pd->first_operand = ALIGN64(size);                   //  total length
 
-    if(request_msg->total_length < LRTS_GNI_RDMA_THRESHOLD)
+    if(request_msg->total_length <= LRTS_GNI_RDMA_THRESHOLD)
         pd->type            = GNI_POST_FMA_GET;
     else
         pd->type            = GNI_POST_RDMA_GET;
@@ -1406,6 +1404,7 @@ static void getLargeMsgRequest(void* header, uint64_t inst_id )
     pd->remote_mem_hndl = request_msg->source_mem_hndl;
     pd->src_cq_hndl     = 0;//post_tx_cqh;     /* smsg_tx_cqh;  */
     pd->rdma_mode       = 0;
+    pd->amo_cmd         = 0;
 
     //memory registration success
     if(status == GNI_RC_SUCCESS)
@@ -1473,7 +1472,7 @@ static void getLargeMsgRequest(void* header, uint64_t inst_id )
     }
 
     MallocPostDesc(pd);
-    if(request_msg->length < LRTS_GNI_RDMA_THRESHOLD) 
+    if(request_msg->length <= LRTS_GNI_RDMA_THRESHOLD) 
         pd->type            = GNI_POST_FMA_GET;
     else
         pd->type            = GNI_POST_RDMA_GET;
@@ -1489,6 +1488,7 @@ static void getLargeMsgRequest(void* header, uint64_t inst_id )
     pd->remote_mem_hndl = request_msg->source_mem_hndl;
     pd->src_cq_hndl     = 0;//post_tx_cqh;     /* smsg_tx_cqh;  */
     pd->rdma_mode       = 0;
+    pd->amo_cmd         = 0;
 
     //memory registration successful
     if(status == GNI_RC_SUCCESS)
@@ -1638,6 +1638,7 @@ static void PumpLocalRdmaTransactions()
                 else
 #endif
                     FreeControlMsg(ack_msg_tmp);
+
             }
 #if CMK_PERSISTENT_COMM
             if (tmp_pd->type == GNI_POST_RDMA_GET || tmp_pd->type == GNI_POST_FMA_GET)
@@ -1656,7 +1657,7 @@ static void PumpLocalRdmaTransactions()
                     TRACE_COMM_CREATION(CpvAccess(projTraceStart), (void*)tmp_pd->local_addr);
 #endif
                     handleOneRecvedMsg(tmp_pd->length, (void*)tmp_pd->local_addr); 
-                }else {
+                }else if(msg_tag == BIG_MSG_TAG){
                   void *msg = (void*)tmp_pd->local_addr-(tmp_pd->cqwrite_value-1)*ONE_SEG;
                   CmiSetMsgSeq(msg, CmiGetMsgSeq(msg)+1);
                   if (tmp_pd->first_operand <= ONE_SEG*CmiGetMsgSeq(msg)) {
index 5e7467dbe66d731985c3b082acf6e7523bb51336..bed3ccc1440a6ebc372b2b46761adea441e4c60b 100644 (file)
@@ -19,16 +19,16 @@ typedef struct {
 */
 #ifdef CMK_BLUEGENEP
 #include "dcmf.h"
-#elif CMK_CONVERSE_GEMINI_UGNI
+#elif  CMK_CONVERSE_GEMINI_UGNI
 #include "gni_pub.h"
 #endif
 typedef struct infiDirectUserHandle{
     int handle;
+#ifdef CMK_BLUEGENEP
     int senderNode;
     int recverNode;
     void *recverBuf;
     int recverBufSize;
-#ifdef CMK_BLUEGENEP
     void *senderBuf;
     void (*callbackFnPtr)(void *);
     void *callbackData;
@@ -39,16 +39,24 @@ typedef struct infiDirectUserHandle{
     DCMF_Memregion_t DCMF_senderMemregion;
     DCMF_Callback_t DCMF_notify_cb;
 #elif  CMK_CONVERSE_GEMINI_UGNI
-    void *senderBuf;
+    int localNode;
+    int remoteNode;
+    void *remoteBuf;
+    int transSize;
+    void *localBuf;
     void (*callbackFnPtr)(void *);
     void *callbackData;
-    gni_mem_handle_t    senderMdh;
-    gni_mem_handle_t    recverMdh;
+    gni_mem_handle_t    localMdh;
+    gni_mem_handle_t    remoteMdh;
 #else
+    int senderNode;
+    int recverNode;
+    void *recverBuf;
+    int recverBufSize;
        char recverKey[64];
 #endif
        double initialValue;
-} CmiDirectUserHandle;
+}CmiDirectUserHandle;
 
 
 /* functions */
@@ -59,7 +67,7 @@ extern "C" {
 /**
  To be called on the receiver to create a handle and return its number
 **/
-CmiDirectUserHandle* CmiDirect_createHandle(int senderNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue);
+struct infiDirectUserHandle CmiDirect_createHandle(int senderNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue);
 
 /****
  To be called on the sender to attach the sender's buffer to this handle
index 7c8ff53e9ae802ab1c5f756db652184cec6b3667..8994d23285ea09d731d510b4b12db60228141f1e 100644 (file)
@@ -4,12 +4,12 @@
 #define NITER 1000
 #define PAYLOAD 100
 
-#if ! CMK_SMP            /* only test RDMA when non-SMP */
-
-#if defined(CMK_DIRECT) || defined(CMK_USE_IBVERBS)
+#ifdef CMK_DIRECT 
 #define USE_RDMA 1
 #endif
 
+#ifdef CMK_USE_IBVERBS 
+#define USE_RDMA 1
 #endif
 
 #ifdef USE_RDMA
@@ -62,7 +62,7 @@ int payload;
 #define P1 0
 #define P2 1%CkNumPes()
 
-class main : public CBase_main
+class main : public Chare
 {
   int phase;
   CProxy_Ping1 arr1;
@@ -76,8 +76,9 @@ public:
   main(CkMigrateMessage *m) {}
   main(CkArgMsg* m)
   {
-      CkAbort("Run this program on 1 or 2 processors only.\n");
+    delete m;
     if(CkNumPes()>2) {
+      CkAbort("Run this program on 1 or 2 processors only.\n");
     }
 
     iterations=NITER;
@@ -110,7 +111,6 @@ public:
     arrF.doneInserting();
     phase=0;
     mainProxy.maindone();
-    delete m;
   };
 
   void maindone(void)
@@ -140,7 +140,7 @@ public:
         break;
 #else
       case 6:
-        ngid[0].startRDMA();
+         ngid[0].startRDMA();
          break;
 #endif
       default:
@@ -149,7 +149,7 @@ public:
   };
 };
 
-class PingG : public CBase_PingG
+class PingG : public Group
 {
   CProxyElement_PingG *pp;
   int niter;
@@ -190,17 +190,19 @@ public:
 };
 
 
-class PingN : public CBase_PingN
+class PingN : public NodeGroup
 {
+  CProxyElement_PingN *pp;
   int niter;
   int me, nbr;
 #ifdef USE_RDMA 
-  CmiDirectUserHandle *shandle,*rhandle;
+  struct infiDirectUserHandle shandle,rhandle;
   char *rbuff;
   char *sbuff;
 #endif
   double start_time, end_time;
 public:
+  CProxyElement_PingN *myProxy;
   PingN()
   {
     me = CkMyNode();    
@@ -210,16 +212,17 @@ public:
     // upstream and downstream which makes this an artificially simple
     // calculation.
 
+    pp = new CProxyElement_PingN(thisgroup,nbr);
+    myProxy = new CProxyElement_PingN(thisgroup,me);
     niter = 0;
 #ifdef USE_RDMA 
-    rbuff=(char *) CmiAlloc(payload*sizeof(char));
-    sbuff=(char *) CmiAlloc(payload*sizeof(char));
+    rbuff=(char *) malloc(payload*sizeof(char));
+    sbuff=(char *) malloc(payload*sizeof(char));
     bzero(sbuff,payload);
-    bzero(rbuff,payload);
     // setup persistent comm sender and receiver side
     double OOB=9999999999.0;
     rhandle=CmiDirect_createHandle(nbr,rbuff,payload*sizeof(char),PingN::Wrapper_To_CallBack,(void *) this,OOB);
-    thisProxy[nbr].recvHandle((char*) rhandle,sizeof(CmiDirectUserHandle));
+    (*pp).recvHandle((char*) &rhandle,sizeof(struct infiDirectUserHandle));
 #endif
   }
   PingN(CkMigrateMessage *m) {}
@@ -227,24 +230,22 @@ public:
   {
 
 #ifdef USE_RDMA 
-    CmiDirectUserHandle *_shandle=(CmiDirectUserHandle *) ptr;
-    shandle=_shandle;
-    CmiDirect_assocLocalBuffer(shandle,sbuff,payload);
-    if(CkMyNode() == 1)
-        startRDMA();
+    struct infiDirectUserHandle *_shandle=(struct infiDirectUserHandle *) ptr;
+    shandle=*_shandle;
+    CmiDirect_assocLocalBuffer(&shandle,sbuff,payload);
 #endif
   }
   void start(void)
   {
     start_time = CkWallTimer();
-    thisProxy[nbr].recv(new (payload) PingMsg);
+    (*pp).recv(new (payload) PingMsg);
   }
   void startRDMA(void)
   {
     niter=0;
     start_time = CkWallTimer();
 #ifdef USE_RDMA 
-    CmiDirect_put(shandle);
+    CmiDirect_put(&shandle);
 #else
     CkAbort("do not call startRDMA if you don't actually have RDMA");
 #endif
@@ -262,10 +263,10 @@ public:
         delete msg;
         mainProxy.maindone();
       } else {
-        thisProxy[nbr].recv(msg);
+        (*pp).recv(msg);
       }
     } else {
-      thisProxy[nbr].recv(msg);
+      (*pp).recv(msg);
     }
   }
   static void Wrapper_To_CallBack(void* pt2Object){
@@ -276,14 +277,14 @@ public:
     if(CkNumNodes() == 0){
       mySelf->recvRDMA();
     }else{
-      (mySelf->thisProxy)[CkMyNode()].recvRDMA();   
+      (*mySelf->myProxy).recvRDMA();   
     }
   }
   // not an entry method, called via Wrapper_To_Callback
   void recvRDMA()
   {
 #ifdef USE_RDMA 
-    CmiDirect_ready(rhandle);
+    CmiDirect_ready(&rhandle);
 #endif
     if(me==0) {
       niter++;
@@ -295,14 +296,14 @@ public:
         mainProxy.maindone();
       } else {
 #ifdef USE_RDMA 
-       CmiDirect_put(shandle);
+       CmiDirect_put(&shandle);
 #else
        CkAbort("do not call startRDMA if you don't actually have RDMA");
 #endif
       }
     } else {
 #ifdef USE_RDMA 
-      CmiDirect_put(shandle);
+      CmiDirect_put(&shandle);
 #else
       CkAbort("do not call startRDMA if you don't actually have RDMA");
 #endif
@@ -312,7 +313,7 @@ public:
 };
 
 
-class Ping1 : public CBase_Ping1
+class Ping1 : public ArrayElement1D
 {
   CProxy_Ping1 *pp;
   int niter;
@@ -366,7 +367,7 @@ public:
   }
 };
 
-class Ping2 : public CBase_Ping2
+class Ping2 : public ArrayElement2D
 {
   CProxy_Ping2 *pp;
   int niter;
@@ -401,7 +402,7 @@ public:
   }
 };
 
-class Ping3 : public CBase_Ping3
+class Ping3 : public ArrayElement3D
 {
   CProxy_Ping3 *pp;
   int niter;
@@ -475,7 +476,7 @@ public:
   }
 };
 
-class PingC : public CBase_PingC
+class PingC : public Chare
 {
   CProxy_PingC *pp;
   int niter;