Zerocopy Entry Method API: use Zerocopy Direct API internally 59/4259/51
authorNitin Bhat <nbhat4@illinois.edu>
Tue, 2 Oct 2018 19:43:52 +0000 (14:43 -0500)
committerNitin Bhat <nbhat4@illinois.edu>
Tue, 11 Dec 2018 20:25:33 +0000 (14:25 -0600)
This patches cleans up all the zerocopy entry method api code and
eliminates the redundant machine layer implementations.
It also gives the Entry Method API already developed capabilities
of the Direct API such as CMA, operation modes and use of pre-registered
buffers.

Change-Id: I6ac812bc31f8c0615ad8cf75ea943620e3495e4f

37 files changed:
examples/charm++/zerocopy/entry_method_api/simpleZeroCopy/simpleZeroCopy.C
examples/charm++/zerocopy/entry_method_api/stencil3d/stencil3d.C
src/arch/common/conv-mach-common.h
src/arch/gni/machine-onesided.c
src/arch/gni/machine-onesided.h
src/arch/gni/machine.C
src/arch/mpi/machine-onesided.c
src/arch/mpi/machine-onesided.h
src/arch/mpi/machine.C
src/arch/ofi/machine-onesided.c
src/arch/ofi/machine-onesided.h
src/arch/ofi/machine.C
src/arch/ofi/request.h
src/arch/pamilrts/machine-onesided.c
src/arch/pamilrts/machine-onesided.h
src/arch/util/machine-rdma.h
src/arch/verbs/machine-ibverbs.C
src/arch/verbs/machine-onesided.c
src/arch/verbs/machine-onesided.h
src/ck-core/charm++.h
src/ck-core/ck.C
src/ck-core/ck.h
src/ck-core/ckrdma.C
src/ck-core/ckrdma.h
src/ck-core/ckrdmawrapper.h [deleted file]
src/ck-core/init.C
src/conv-core/conv-rdma.c
src/conv-core/conv-rdma.h
src/conv-core/convcore.C
src/scripts/Make.depends
src/scripts/Makefile
src/util/cmirdmautils.c
src/util/cmirdmautils.h
src/xlat-i/sdag/CEntry.C
src/xlat-i/sdag/CSdagConstruct.C
src/xlat-i/xi-Entry.C
src/xlat-i/xi-Parameter.C

index f5d95d65d7dcef8431c92516c35120884f0684ed..0a248529395ea7e4d0a739de79ac04380af73766 100644 (file)
@@ -153,9 +153,9 @@ class zerocopyObject : public CBase_zerocopyObject{
     zerocopyObject(CkMigrateMessage *m){}
 
     void zerocopySent(CkDataMsg *m){
-      //to get access to the array sent via zerocopy
-      void *ptr = *((void **)(m->data));
-      free(ptr);
+      // Get access to the array information sent via zerocopy
+      CkNcpyBuffer *src = (CkNcpyBuffer *)(m->data);
+      free((void *)(src->ptr));
       delete m;
 
       if(++mixedZeroCopySentCounter == 2)
index d32537ef58eed672bf3acb7d1a2b8456ba78b475..555ab99102d6e10ac6d1a99f5f0cec3f4a2a1c42 100644 (file)
@@ -150,7 +150,8 @@ class Stencil: public CBase_Stencil {
 
     // callback function called on completion of sending ghosts
     void completedSendingGhost(CkDataMsg *msg){
-      void *ptr = *((void **)(msg->data)); // we don't free the ghost array as it is reused across iterations
+      CkNcpyBuffer *src = (CkNcpyBuffer *)(msg->data);
+      void *ptr = (void *)(src->ptr); // we don't free the ghost array as it is reused across iterations
       delete msg;
       counter++;
       // Advance to next step on completion of sending ghosts to the 6 neighbors
index ac1644664ff01bc519e2c1b8da10d44a95eb3a59..98490eb35f0beb0655478aee1636b8a4202a40b7 100644 (file)
@@ -39,3 +39,26 @@ enum ncpyRegModes {
 // default of CMK_COMMON_NOCOPY_DIRECT_BYTES assumes no CMA support
 // Refined for lrts layers with CMA support inside lrts-common.h
 #define CMK_COMMON_NOCOPY_DIRECT_BYTES 0
+
+// Enum for the type of zerocopy operation
+// TODO: Convert to typed enum post C++ conversion
+enum ncpyOperationMode {
+  CMK_DIRECT_API          = 0,
+  CMK_EM_API              = 1,
+  CMK_EM_API_REVERSE      = 2
+};
+
+// Enum for the method of acknowledglement handling after the completion of a zerocopy operation
+// TODO: Convert to typed enum post C++ conversion
+enum ncpyAckMode {
+  CMK_SRC_DEST_ACK       = 0,
+  CMK_SRC_ACK            = 1,
+  CMK_DEST_ACK           = 2
+};
+
+// Enum to determine if a NcpyOperationInfo can be freed upon completion
+// TODO: Convert to a bool variable post C++ conversion
+enum ncpyFreeNcpyOpInfoMode {
+  CMK_FREE_NCPYOPINFO           = 0,
+  CMK_DONT_FREE_NCPYOPINFO           = 1
+};
index b16aebf06adc2afa5b87b1b1a4c0c246a5f5f595..fbd1a30f5eb102f522f41d48a99b3a6efc6c2308 100644 (file)
@@ -22,56 +22,6 @@ void _initOnesided()
   MACH_DEBUG(CmiPrintf("[%d]_initOneSided: Initialized CQ and lock\n", CmiMyPe()));
 }
 
-int checkFourByteAligned(void *recv){
-  CmiGNIRzvRdmaRecv_t* recvInfo = (CmiGNIRzvRdmaRecv_t*)recv;
-  int i, size;
-  for(i = 0; i < recvInfo->numOps; ++i)
-  {
-    CmiGNIRzvRdmaRecvOp_t * recvOp = &recvInfo->rdmaOp[i];
-    uint64_t remote_addr = recvOp->remote_addr;
-    uint64_t local_addr = recvOp->local_addr;
-    int length = recvOp->size;
-    if(((local_addr % 4)==0) && ((remote_addr % 4)==0) && ((length % 4)==0))
-      continue;
-    MACH_DEBUG(CmiPrintf("[%d][%d][%d] Unaligned, should use PUT\n", CmiMyPe(), CmiMyNode(), CmiMyRank()));
-    return 0;
-  }
-  MACH_DEBUG(CmiPrintf("[%d][%d][%d] Aligned, should use GET\n", CmiMyPe(), CmiMyNode(), CmiMyRank()));
-  return 1;
-}
-
-void rdma_sendMdBackForPut( CmiGNIRzvRdmaRecv_t* recvInfo, int src_pe){
-  int size = LrtsGetRdmaRecvInfoSize(recvInfo->numOps);
-  send_smsg_message(&smsg_queue, CmiNodeOf(src_pe), recvInfo, size, RDMA_PUT_MD_TAG, 0, NULL, NONCHARM_SMSG_DONT_FREE, 0);
-  MACH_DEBUG(CmiPrintf("[%d]rdma_sendMdBackForPut: Sent md back to %d for PUT\n", CmiMyPe(), src_pe));
-}
-
-void  rdma_sendAck (CmiGNIRzvRdmaRecvOp_t* recvOpInfo, int src_pe)
-{
-  CmiGNIAckOp_t *ack_data = (CmiGNIAckOp_t *)malloc(sizeof(CmiGNIAckOp_t));
-  ack_data->ack = (CmiRdmaAck *)recvOpInfo->src_info;
-  ack_data->mem_hndl = recvOpInfo->remote_mem_hndl;
-  MACH_DEBUG(CmiPrintf("[%d]rdma_sendAck: Sent rdma ack to %d\n", CmiMyPe(), src_pe));
-  gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(src_pe), ack_data, sizeof(CmiGNIAckOp_t), RDMA_ACK_TAG, 0, NULL, NONCHARM_SMSG, 1);
-#if !CMK_SMSGS_FREE_AFTER_EVENT
-  if(status == GNI_RC_SUCCESS) {
-    free(ack_data);
-  }
-#endif
-}
-
-void  rdma_sendMsgForPutCompletion (CmiGNIRzvRdmaRecv_t* recvInfo, int destNode)
-{
-  int size = LrtsGetRdmaRecvInfoSize(recvInfo->numOps);
-  gni_return_t status = send_smsg_message(&smsg_queue, destNode, recvInfo, size, RDMA_PUT_DONE_TAG, 0, NULL, NONCHARM_SMSG, 1);
-#if !CMK_SMSGS_FREE_AFTER_EVENT
-  if(status == GNI_RC_SUCCESS) {
-    free(recvInfo);
-  }
-#endif
-  MACH_DEBUG(CmiPrintf("[%d]rdma_sendMsgForPutCompletion: Sent md back to node:%d to indicate PUT completion\n", CmiMyPe(), destNode));
-}
-
 gni_return_t post_rdma(uint64_t remote_addr, gni_mem_handle_t remote_mem_hndl,
     uint64_t local_addr, gni_mem_handle_t local_mem_hndl,
     int length, uint64_t post_id, int destNode, int type, unsigned short int mode)
@@ -94,7 +44,6 @@ gni_return_t post_rdma(uint64_t remote_addr, gni_mem_handle_t remote_mem_hndl,
   pd->amo_cmd             = (gni_fma_cmd_type_t)0;
 
   switch(mode) {
-    case INDIRECT_SEND                :  // Using entry method api
     case DIRECT_SEND_RECV             :  // Using direct api GET or PUT
     case DIRECT_SEND_RECV_UNALIGNED   :  // Using direct api GET,
                                          // which resulted into a PUT because of alignment
@@ -119,86 +68,6 @@ gni_return_t post_rdma(uint64_t remote_addr, gni_mem_handle_t remote_mem_hndl,
   return status;
 }
 
-void LrtsIssueRputs(void *recv, int node)
-{
-  CmiGNIRzvRdmaRecv_t* recvInfo = (CmiGNIRzvRdmaRecv_t *)recv;
-  gni_return_t status;
-  int i;
-
-  MACH_DEBUG(CmiPrintf("Started LrtsIssueRputs, Issued from %d to node:%d\n", CmiMyPe(), node));
-
-  for(i = 0; i < recvInfo->numOps; ++i){
-    CmiGNIRzvRdmaRecvOp_t *recvOp = &recvInfo->rdmaOp[i];
-    gni_mem_handle_t remote_mem_hndl = recvOp->remote_mem_hndl;
-    gni_mem_handle_t local_mem_hndl = recvOp->local_mem_hndl;
-    uint64_t remote_addr = recvOp->remote_addr;
-    uint64_t buffer = recvOp->local_addr;
-    int length = recvOp->size;
-
-    uint64_t opAddress = (uint64_t)(recvOp);
-
-    status = post_rdma(remote_addr, remote_mem_hndl, buffer, local_mem_hndl,
-        length, opAddress, node, GNI_POST_RDMA_PUT, INDIRECT_SEND);
-  }
-}
-
-void LrtsIssueRgets(void *recv, int pe)
-{
-  CmiGNIRzvRdmaRecv_t* recvInfo = (CmiGNIRzvRdmaRecv_t*)recv;
-  gni_return_t status;
-  int i;
-
-  MACH_DEBUG(CmiPrintf("Started LrtsIssueRgets, Issued from %d to %d\n", CmiMyPe(), pe));
-
-  if(checkFourByteAligned(recv)){
-    for(i = 0; i < recvInfo->numOps; ++i)
-    {
-      CmiGNIRzvRdmaRecvOp_t * recvOp = &recvInfo->rdmaOp[i];
-      gni_mem_handle_t remote_mem_hndl = recvOp->remote_mem_hndl;
-      uint64_t remote_addr = recvOp->remote_addr;
-      uint64_t buffer = recvOp->local_addr;
-      int length = recvOp->size;
-      uint64_t opAddress = (uint64_t)(recvOp);
-
-      /* Register the local buffer with the NIC */
-      gni_mem_handle_t local_mem_hndl;
-
-      status = GNI_MemRegister(nic_hndl, buffer, length, NULL,  GNI_MEM_READWRITE, -1, &local_mem_hndl);
-      GNI_RC_CHECK("Error! Exceeded Allowed Pinned Memory Limit! GNI_MemRegister on Receiver Buffer (destination) Failed before GET", status);
-
-      recvOp->local_mem_hndl = local_mem_hndl;
-      status = post_rdma(remote_addr, remote_mem_hndl, buffer, local_mem_hndl,
-          length, opAddress, CmiNodeOf(pe), GNI_POST_RDMA_GET, INDIRECT_SEND);
-    }
-  }
-  //use RPUT because of 4-byte alignment not being conformed
-  else{
-
-    //send metadata message back to the sender for performing RPUT
-    for(i = 0; i < recvInfo->numOps; ++i)
-    {
-      CmiGNIRzvRdmaRecvOp_t * recvOp = &recvInfo->rdmaOp[i];
-      uint64_t remote_addr = recvOp->remote_addr;
-      uint64_t buffer = recvOp->local_addr;
-      int length = recvOp->size;
-
-      /* Register the local buffer with the NIC */
-      gni_mem_handle_t local_mem_hndl;
-      status = GNI_MemRegister(nic_hndl, buffer, length, NULL,  GNI_MEM_READWRITE, -1, &local_mem_hndl);
-      GNI_RC_CHECK("Error! Exceeded Allowed Pinned Memory Limit! GNI_MemRegister on Receiver Buffer (destination) Failed before PUT", status);
-
-      //Switch local and remote handles and buffers as recvInfo will be sent to the sender for PUT
-      recvOp->local_mem_hndl = recvOp->remote_mem_hndl;
-      recvOp->remote_mem_hndl = local_mem_hndl;
-      recvOp->local_addr = remote_addr;
-      recvOp->remote_addr = buffer;
-    }
-
-    //send metadata message to receiver to perform a PUT
-    rdma_sendMdBackForPut(recvInfo, pe);
-  }
-}
-
 /*
  * This code largely overlaps with code in PumpLocalTransactions,
  * any changes to PumpLocalTransactions should be reflected here
@@ -231,28 +100,7 @@ void PumpOneSidedRDMATransactions(gni_cq_handle_t rdma_cq, CmiNodeLock rdma_cq_l
 
       if(tmp_pd->type == GNI_POST_RDMA_GET){
 
-        if(tmp_pd->first_operand == INDIRECT_SEND) {
-          // Invoke the method handler if used for indirect api
-          CmiGNIRzvRdmaRecvOp_t * recvOpInfo = (CmiGNIRzvRdmaRecvOp_t *)tmp_pd->second_operand;
-          CmiGNIRzvRdmaRecv_t * recvInfo = (CmiGNIRzvRdmaRecv_t *)((char *)recvOpInfo
-              - recvOpInfo->opIndex * sizeof(CmiGNIRzvRdmaRecvOp_t)
-              - sizeof(CmiGNIRzvRdmaRecv_t));
-
-          // Deregister registered receiver memory used for GET
-          status = GNI_MemDeregister(nic_hndl, &(recvOpInfo->local_mem_hndl));
-          GNI_RC_CHECK("GNI_MemDeregister on Receiver for GET operation", status);
-
-          rdma_sendAck(recvOpInfo, recvInfo->srcPE);
-          recvInfo->comOps++;
-          if(recvInfo->comOps == recvInfo->numOps)
-          {
-            char * msg = (char *)recvInfo->msg;
-            int msg_size = CmiGetMsgSize(msg);
-            handleOneRecvedMsg(msg_size, msg);
-            MACH_DEBUG(CmiPrintf("[%d]PumpOneSidedTransaction: Final Ack sent to %d\n", CmiMyPe(), CMI_DEST_RANK(msg)));
-
-          }
-        } else if (tmp_pd->first_operand == DIRECT_SEND_RECV) {
+        if (tmp_pd->first_operand == DIRECT_SEND_RECV) {
           // Call the ack handler function if used for direct api
           CmiInvokeNcpyAck((void *)tmp_pd->second_operand);
         } else {
@@ -262,30 +110,7 @@ void PumpOneSidedRDMATransactions(gni_cq_handle_t rdma_cq, CmiNodeLock rdma_cq_l
       }
       else if(tmp_pd->type == GNI_POST_RDMA_PUT){
 
-        if(tmp_pd->first_operand == INDIRECT_SEND) {
-          // Invoke the method handler if used for indirect api
-          CmiGNIRzvRdmaRecvOp_t * recvOpInfo = (CmiGNIRzvRdmaRecvOp_t *)tmp_pd->second_operand;
-          CmiGNIRzvRdmaRecv_t * recvInfo = (CmiGNIRzvRdmaRecv_t *)((char *)recvOpInfo
-              - recvOpInfo->opIndex * sizeof(CmiGNIRzvRdmaRecvOp_t)
-              - sizeof(CmiGNIRzvRdmaRecv_t));
-          recvInfo->comOps++;
-          // Deregister registered sender memory used for PUT
-          status = GNI_MemDeregister(nic_hndl, &(recvOpInfo->local_mem_hndl));
-          GNI_RC_CHECK("GNI_MemDeregister on Sender for PUT operation", status);
-
-          if(recvInfo->comOps == recvInfo->numOps)
-          {
-            // send message to the receiver to signal PUT completion so that
-            // the receiver can call the message handler
-            rdma_sendMsgForPutCompletion(recvInfo, recvInfo->destNode);
-          }
-          //call ack on the sender
-          CmiRdmaAck* ack = (CmiRdmaAck *)(recvOpInfo->src_info);
-          ack->fnPtr(ack->token);
-          //free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
-          free(ack);
-        } else if (tmp_pd->first_operand == DIRECT_SEND_RECV) {
-
+        if (tmp_pd->first_operand == DIRECT_SEND_RECV) {
           // Call the ack handler function if used for direct api
           CmiInvokeNcpyAck((void *)tmp_pd->second_operand);
 
@@ -356,22 +181,27 @@ void LrtsIssueRget(NcpyOperationInfo *ncpyOpInfo) {
   if(ncpyOpInfo->isSrcRegistered == 0) {
     // Remote buffer is unregistered, send a message to register it and perform PUT
 
+    // set OpMode for reverse operation
+    setReverseModeForNcpyOpInfo(ncpyOpInfo);
+
 #if CMK_SMP
     // send the small message to the other node through the comm thread
     buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize,
                           CmiNodeOf(ncpyOpInfo->srcPe),
                           RDMA_REG_AND_PUT_MD_DIRECT_TAG);
 #else // non-smp mode
+
+    int msgMode = (ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) ? CHARM_SMSG : SMSG_DONT_FREE;
+
     // send the small message directly
     gni_return_t status = send_smsg_message(&smsg_queue,
                             CmiNodeOf(ncpyOpInfo->srcPe),
                             ncpyOpInfo,
                             ncpyOpInfo->ncpyOpInfoSize,
                             RDMA_REG_AND_PUT_MD_DIRECT_TAG,
-                            0, NULL, CHARM_SMSG, 1);
-    GNI_RC_CHECK("Sending REG & PUT metadata msg failed!", status);
+                            0, NULL, msgMode, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
-    if(status == GNI_RC_SUCCESS) {
+    if(status == GNI_RC_SUCCESS && ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) {
       CmiFree(ncpyOpInfo);
     }
 #endif // end of !CMK_SMSGS_FREE_AFTER_EVENT
@@ -406,15 +236,20 @@ void LrtsIssueRget(NcpyOperationInfo *ncpyOpInfo) {
 
     } else {
       // send all the data to the source to perform a put
+
+      // set OpMode for reverse operation
+      setReverseModeForNcpyOpInfo(ncpyOpInfo);
+
 #if CMK_SMP
       // send the small message to the other node through the comm thread
       buffer_small_msgs(&smsg_queue, ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, CmiNodeOf(ncpyOpInfo->srcPe), RDMA_PUT_MD_DIRECT_TAG);
 #else // nonsmp mode
       // send the small message directly
-      gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(ncpyOpInfo->srcPe), ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, RDMA_PUT_MD_DIRECT_TAG, 0, NULL, CHARM_SMSG, 1);
-      GNI_RC_CHECK("Sending PUT metadata msg failed!", status);
+      int msgMode = (ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) ? CHARM_SMSG : SMSG_DONT_FREE;
+
+      gni_return_t status = send_smsg_message(&smsg_queue, CmiNodeOf(ncpyOpInfo->srcPe), ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, RDMA_PUT_MD_DIRECT_TAG, 0, NULL, SMSG_DONT_FREE, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
-      if(status == GNI_RC_SUCCESS) {
+      if(status == GNI_RC_SUCCESS && ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) {
         CmiFree(ncpyOpInfo);
       }
 #endif // end of !CMK_SMSGS_FREE_AFTER_EVENT
index 07b146b43a22ccf904edba1389f4ffce3bec1e9e..bc873cec4c265784e2bd28ac5bb5591d9e7562b1 100644 (file)
 
 #include "mempool.h"
 
-typedef struct _cmi_gni_ack_op {
-  CmiRdmaAck * ack;
-  gni_mem_handle_t mem_hndl;
-} CmiGNIAckOp_t;
-
-typedef struct _cmi_gni_rzv_rdma_op {
-  int size;
-  uint64_t remote_addr;
-  gni_mem_handle_t mem_hndl;
-} CmiGNIRzvRdmaOp_t;
-
-typedef struct _cmi_gni_rzv_rdma {
-  int numOps;
-  int srcPE;
-  CmiGNIRzvRdmaOp_t rdmaOp[0];
-} CmiGNIRzvRdma_t;
-
-typedef struct _cmi_gni_rzv_rdma_recv_op {
-  int size;
-  uint64_t remote_addr;
-  uint64_t local_addr;
-  gni_mem_handle_t local_mem_hndl;
-  gni_mem_handle_t remote_mem_hndl;
-  int opIndex;
-  void * src_info;
-} CmiGNIRzvRdmaRecvOp_t;
-
-
-typedef struct _cmi_gni_rzv_rdma_recv {
-  int numOps;
-  int comOps;
-  int destNode;
-  int srcPE;
-  void* msg;
-  CmiGNIRzvRdmaRecvOp_t rdmaOp[0];
-} CmiGNIRzvRdmaRecv_t;
-
 void _initOnesided();
 
-void  rdma_sendAck (CmiGNIRzvRdmaRecvOp_t* recvOpInfo, int src_pe);
-void  rdma_sendMsgForPutCompletion (CmiGNIRzvRdmaRecv_t* recvInfo, int dest_pe);
-
-void LrtsIssueRgets(void *recv, int pe);
-void LrtsIssueRputs(void *recv, int pe);
-
-int LrtsGetRdmaOpInfoSize(){
-  return sizeof(CmiGNIRzvRdmaOp_t);
-}
-int LrtsGetRdmaGenInfoSize(){
-  return sizeof(CmiGNIRzvRdma_t);
-}
-int LrtsGetRdmaInfoSize(int numOps){
-  return sizeof(CmiGNIRzvRdma_t) + numOps * sizeof(CmiGNIRzvRdmaOp_t);
-}
-
-int LrtsGetRdmaOpRecvInfoSize(){
-  return sizeof(CmiGNIRzvRdmaRecvOp_t);
-}
-
-int LrtsGetRdmaGenRecvInfoSize(){
-  return sizeof(CmiGNIRzvRdmaRecv_t);
-}
-
-int LrtsGetRdmaRecvInfoSize(int numOps){
-  return sizeof(CmiGNIRzvRdmaRecv_t) + numOps * sizeof(CmiGNIRzvRdmaRecvOp_t);
-}
-
-void LrtsSetRdmaRecvInfo(void *rdmaRecv, int numOps, void *msg, void *rdmaSend, int msgSize){
-  CmiGNIRzvRdmaRecv_t *rdmaRecvInfo = (CmiGNIRzvRdmaRecv_t *)rdmaRecv;
-  CmiGNIRzvRdma_t *rdmaSendInfo = (CmiGNIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvInfo->srcPE = rdmaSendInfo->srcPE;
-  rdmaRecvInfo->destNode = CmiMyNode();
-  rdmaRecvInfo->numOps = numOps;
-  rdmaRecvInfo->comOps = 0;
-  rdmaRecvInfo->msg = msg;
-}
-
-void LrtsSetRdmaRecvOpInfo(void *rdmaRecvOp, void *buffer, void *src_ref, int size, int opIndex, void *rdmaSend){
-  CmiGNIRzvRdmaRecvOp_t *rdmaRecvOpInfo = (CmiGNIRzvRdmaRecvOp_t *)rdmaRecvOp;
-  CmiGNIRzvRdma_t *rdmaSendInfo = (CmiGNIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvOpInfo->src_info = src_ref;
-  rdmaRecvOpInfo->size = rdmaSendInfo->rdmaOp[opIndex].size;
-  rdmaRecvOpInfo->opIndex = opIndex;
-  rdmaRecvOpInfo->local_addr = (uint64_t)buffer;
-
-  rdmaRecvOpInfo->remote_addr = rdmaSendInfo->rdmaOp[opIndex].remote_addr;
-  rdmaRecvOpInfo->remote_mem_hndl = rdmaSendInfo->rdmaOp[opIndex].mem_hndl;
-}
-
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps){
-  CmiGNIRzvRdma_t *rdma = (CmiGNIRzvRdma_t*)dest;
-  rdma->srcPE = CmiMyPe();
-  rdma->numOps = numOps;
-}
-
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  gni_mem_handle_t mem_hndl;
-  gni_return_t status = GNI_RC_SUCCESS;
-
-  status = GNI_MemRegister(nic_hndl, (uint64_t)ptr,  (uint64_t)size, NULL,  GNI_MEM_READ_ONLY, -1, &mem_hndl);
-  GNI_RC_CHECK("Error! Exceeded Allowed Pinned Memory Limit! GNI_MemRegister on Sender Buffer (source) Failed before sending metadata message", status);
-
-  CmiGNIRzvRdmaOp_t *rdmaOp = (CmiGNIRzvRdmaOp_t *)dest;
-  rdmaOp->remote_addr = (uint64_t)ptr;
-  rdmaOp->size = size;
-  rdmaOp->mem_hndl = mem_hndl;
-}
-
 void PumpOneSidedRDMATransactions(gni_cq_handle_t rdma_cq, CmiNodeLock rdma_cq_lock);
 
-
 /* Support for Nocopy Direct API */
 
 /* Type of RDMA transaction
    This is used in handling each type of RDMA transaction appropriately on completion
  */
 enum CMK_RDMA_TYPE {
-  INDIRECT_SEND=1,
-  DIRECT_SEND_RECV,
+  DIRECT_SEND_RECV=1,
   DIRECT_SEND_RECV_UNALIGNED
 };
 
index 3e0fd388e4f2a7ba69c4d8553b5c28fe24984468..c4ab6bf3c2061636713ca8ef234d861f42837e7a 100644 (file)
@@ -105,7 +105,7 @@ static int   RDMA_pending = 0;
 enum CMK_SMSG_TYPE {
   CHARM_SMSG=1,
   NONCHARM_SMSG,
-  NONCHARM_SMSG_DONT_FREE
+  SMSG_DONT_FREE
 };
 
 #define USE_LRTS_MEMPOOL                  1
@@ -2323,37 +2323,6 @@ static void PumpNetworkSmsg()
             }
 #endif
 #if CMK_ONESIDED_IMPL
-            case RDMA_ACK_TAG:
-            {
-                CmiGNIAckOp_t *ack_data = (CmiGNIAckOp_t *)header;
-                CmiRdmaAck * ack = ack_data->ack;
-                gni_mem_handle_t mem_hndl = ack_data->mem_hndl;
-                GNI_SmsgRelease(ep_hndl_array[inst_id]);
-                CMI_GNI_UNLOCK(smsg_mailbox_lock)
-
-                // call the fnPtr to handle the ack
-                ack->fnPtr(ack->token);
-
-                // free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
-                free(ack);
-
-                // Deregister registered sender memory used for GET
-                status = GNI_MemDeregister(nic_hndl, &mem_hndl);
-                GNI_RC_CHECK("GNI_MemDeregister on Sender for GET operation", status);
-                break;
-            }
-            case RDMA_PUT_MD_TAG:
-            {
-                CmiGNIRzvRdmaRecv_t *recvInfo = (CmiGNIRzvRdmaRecv_t *)header;
-                recvInfoSize = LrtsGetRdmaRecvInfoSize(recvInfo->numOps);
-                CmiGNIRzvRdmaRecv_t *newRecvInfo = (CmiGNIRzvRdmaRecv_t *)malloc(recvInfoSize);
-                memcpy(newRecvInfo, recvInfo, recvInfoSize);
-                GNI_SmsgRelease(ep_hndl_array[inst_id]);
-                CMI_GNI_UNLOCK(smsg_mailbox_lock)
-
-                LrtsIssueRputs((void *)newRecvInfo, recvInfo->destNode);
-                break;
-            }
             case RDMA_PUT_MD_DIRECT_TAG:
             {
                 NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)header;
@@ -2365,6 +2334,8 @@ static void PumpNetworkSmsg()
                 GNI_SmsgRelease(ep_hndl_array[inst_id]);
                 CMI_GNI_UNLOCK(smsg_mailbox_lock)
 
+                resetNcpyOpInfoPointers(newNcpyOpInfo);
+
                 post_rdma((uint64_t)newNcpyOpInfo->destPtr,
                           ((CmiGNIRzvRdmaPtr_t *)((char *)(newNcpyOpInfo->destLayerInfo) + CmiGetRdmaCommonInfoSize()))->mem_hndl,
                           (uint64_t)newNcpyOpInfo->srcPtr,
@@ -2377,36 +2348,7 @@ static void PumpNetworkSmsg()
 
                 break;
             }
-            case RDMA_PUT_DONE_TAG:
-            {
-                CmiGNIRzvRdmaRecv_t *recvInfo = (CmiGNIRzvRdmaRecv_t *)header;
-
-                /* copy the received message (recvInfo) into newRecvInfo as
-                 * recvInfo is invalid after SmsgRelease call
-                 */
-                recvInfoSize = LrtsGetRdmaRecvInfoSize(recvInfo->numOps);
-                CmiGNIRzvRdmaRecv_t *newRecvInfo = (CmiGNIRzvRdmaRecv_t *)malloc(recvInfoSize);
-                memcpy(newRecvInfo, recvInfo, recvInfoSize);
-                GNI_SmsgRelease(ep_hndl_array[inst_id]);
-                CMI_GNI_UNLOCK(smsg_mailbox_lock)
-
-                char *msg = (char *)newRecvInfo->msg;
-                int size = CmiGetMsgSize(msg);
-
-                handleOneRecvedMsg(size, msg);
-
-                int i;
-                for(i=0; i<newRecvInfo->numOps;i++){
-                    CmiGNIRzvRdmaRecvOp_t * recvOp = &newRecvInfo->rdmaOp[i];
-                    // Deregister registered receiver memory used for PUT
-                    status = GNI_MemDeregister(nic_hndl, &(recvOp->remote_mem_hndl));
-                    GNI_RC_CHECK("GNI_MemDeregister on Receiver for PUT operation", status);
-                }
-                // free newRecvInfo as it is no longer used
-                free(newRecvInfo);
-                break;
-            }
-            case RDMA_PUT_DONE_DIRECT_TAG:
+           case RDMA_PUT_DONE_DIRECT_TAG:
             {
                 // Direct API when PUT is used instead of a GET
                 // This tag implies the completion of an indirect PUT operation used for the Direct API
@@ -2443,8 +2385,11 @@ static void PumpNetworkSmsg()
 
                 resetNcpyOpInfoPointers(newNcpyOpInfo);
 
+                // Free the NcpyOperationInfo in a reverse API operation
+                newNcpyOpInfo->freeMe = CMK_FREE_NCPYOPINFO;
+
                 // Register source buffer
-                ((CmiGNIRzvRdmaPtr_t *)((char *)(newNcpyOpInfo->srcLayerInfo) + CmiGetRdmaCommonInfoSize()))->mem_hndl = 
+                ((CmiGNIRzvRdmaPtr_t *)((char *)(newNcpyOpInfo->srcLayerInfo) + CmiGetRdmaCommonInfoSize()))->mem_hndl =
                                             registerDirectMem(newNcpyOpInfo->srcPtr,
                                                                 newNcpyOpInfo->srcSize,
                                                                 GNI_MEM_READ_ONLY);
@@ -3232,7 +3177,7 @@ static void PumpLocalTransactions(gni_cq_handle_t my_tx_cqh, CmiNodeLock my_cq_l
             case NONCHARM_SMSG:
                 free(addr);
                 break;
-            case NONCHARM_SMSG_DONT_FREE:
+            case SMSG_DONT_FREE:
                 break;
             default:
                 CmiAbort("Invalid SmsgsIndex");
@@ -3371,7 +3316,7 @@ INLINE_KEYWORD gni_return_t _sendOneBufferedSmsg(SMSG_QUEUE *queue, MSG_LIST *pt
 {
     CONTROL_MSG         *control_msg_tmp;
     gni_return_t        status = GNI_RC_ERROR_RESOURCE;
-    int                 numRdmaOps, recvInfoSize, msgSize;
+    int                 numRdmaOps, recvInfoSize, msgSize, msgMode;
     NcpyOperationInfo *ncpyOpInfo;
 
     MACHSTATE5(8, "noempty-smsg  %d (%d,%d,%d) tag=%d \n", ptr->destNode, buffered_send_msg, buffered_recv_msg, register_memory_size, ptr->tag); 
@@ -3440,37 +3385,12 @@ INLINE_KEYWORD gni_return_t _sendOneBufferedSmsg(SMSG_QUEUE *queue, MSG_LIST *pt
 #endif
         break;
 #endif
-    case RDMA_ACK_TAG:
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, sizeof(CmiGNIAckOp_t), ptr->tag, 1, ptr, NONCHARM_SMSG, 1);
-#if !CMK_SMSGS_FREE_AFTER_EVENT
-        if(status == GNI_RC_SUCCESS) {
-          free(ptr->msg);
-        }
-#endif
-        break;
-
-    case RDMA_PUT_MD_TAG:
-        numRdmaOps = ((CmiGNIRzvRdmaRecv_t *)(ptr->msg))->numOps;
-        recvInfoSize = LrtsGetRdmaRecvInfoSize(numRdmaOps);
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, recvInfoSize, ptr->tag, 1, ptr, NONCHARM_SMSG_DONT_FREE, 0);
-        break;
-
-     case RDMA_PUT_DONE_TAG:
-        numRdmaOps = ((CmiGNIRzvRdmaRecv_t *)(ptr->msg))->numOps;
-        recvInfoSize = LrtsGetRdmaRecvInfoSize(numRdmaOps);
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, recvInfoSize, ptr->tag, 1, ptr, NONCHARM_SMSG, 1);
-#if !CMK_SMSGS_FREE_AFTER_EVENT
-        if(status == GNI_RC_SUCCESS) {
-          free(ptr->msg);
-        }
-#endif
-        break;
-
      case RDMA_PUT_MD_DIRECT_TAG:
         ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, CHARM_SMSG, 1);
+        msgMode = (ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) ? CHARM_SMSG : SMSG_DONT_FREE;
+        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, msgMode, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
-        if(status == GNI_RC_SUCCESS) {
+        if(status == GNI_RC_SUCCESS && msgMode) {
           CmiFree(ptr->msg);
         }
 #endif
@@ -3480,9 +3400,10 @@ INLINE_KEYWORD gni_return_t _sendOneBufferedSmsg(SMSG_QUEUE *queue, MSG_LIST *pt
      case RDMA_REG_AND_PUT_MD_DIRECT_TAG:
         //msgSize = sizeof(CmiGNIRzvRdmaReverseOp_t) + 2*(((CmiGNIRzvRdmaReverseOp_t *)(ptr->msg))->ackSize);
         ncpyOpInfo = (NcpyOperationInfo *)(ptr->msg);
-        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, CHARM_SMSG, 1);
+        msgMode = (ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO) ? CHARM_SMSG : SMSG_DONT_FREE;
+        status = send_smsg_message(queue, ptr->destNode, ptr->msg, ncpyOpInfo->ncpyOpInfoSize, ptr->tag, 1, ptr, msgMode, 1);
 #if !CMK_SMSGS_FREE_AFTER_EVENT
-        if(status == GNI_RC_SUCCESS) {
+        if(status == GNI_RC_SUCCESS && msgMode) {
           CmiFree(ptr->msg);
         }
 #endif
index e5dc48970863d8fbef9909c3ddfb65a0b8414939..0f2ec654e75a7431f0704e01f5362c8248878ecc 100644 (file)
@@ -3,22 +3,6 @@
   Nitin Bhat 03/07/2017
 */
 
-//Post MPI_Irecvs to receive the RDMA buffers
-void LrtsIssueRgets(void *recv, int pe){
-  int i;
-  CmiMPIRzvRdmaRecvList_t *recvInfo = (CmiMPIRzvRdmaRecvList_t *)recv;
-  MPI_Request reqBufferRecv;
-  int srcPe = recvInfo->srcPe;
-
-  for(i=0; i<recvInfo->numOps; i++){
-    void *buffer = recvInfo->rdmaOp[i].buffer;
-    int size = recvInfo->rdmaOp[i].size;
-    int srcTag = recvInfo->rdmaOp[i].tag;
-
-    MPIPostOneBuffer(buffer, (char *)(&(recvInfo->rdmaOp[i])), size, srcPe, srcTag, ONESIDED_BUFFER_RECV);  
-  }
-}
-
 // Post MPI_Isend or MPI_Irecv to send/recv the RDMA buffer
 // Used by both non-SMP and SMP modes
 void MPISendOrRecvOneBuffer(SMSG_LIST *smsg, int tag){
index 654b782c8d34d4bcbdbb6e51bd704addaacb188d..b7ce0ac7ad1158b827dbba4c14147fdc5cb4e86a 100644 (file)
@@ -1,45 +1,4 @@
-
-typedef struct _cmi_mpi_rzv_rdma_op{
-  int size;
-  int tag;
-} CmiMPIRzvRdmaOp_t;
-
-typedef struct _cmi_mpi_rzv_rdma{
-  int numOps;
-  int srcPe;
-  CmiMPIRzvRdmaOp_t rdmaOp[0];
-} CmiMPIRzvRdma_t;
-
-typedef struct _cmi_mpi_rzv_rdma_recv_op {
-  void *buffer;
-  void *src_info;
-  int size;
-  int tag;
-  int opIndex;
-  MPI_Request req;
-} CmiMPIRzvRdmaRecvOp_t;
-
-//Receiver's rdma buffer information is stored as a list to wait for completion of the recv requests
-typedef struct _cmi_mpi_rzv_rdma_recv_list {
-  int srcPe;
-  int numOps;
-  int counter;
-  int msgLen;
-  void *msg;
-  CmiMPIRzvRdmaRecvOp_t rdmaOp[0];
-} CmiMPIRzvRdmaRecvList_t;
-
-//Used by MPI Send
-typedef struct _cmi_mpi_rzv_rdma_opinfo {
-  void *ack;
-  int tag;
-} CmiMPIRzvRdmaOpInfo_t;
-
 //List of rdma receiver buffer information that is used for waiting for completion
-CpvDeclare(CmiMPIRzvRdmaRecvList_t *, recvRdmaBuffers);
-CpvDeclare(CmiMPIRzvRdmaRecvList_t *, endRdmaBuffer);
-CpvDeclare(int, RdmaRecvQueueLen);
-
 void MPIPostOneBuffer(const void *buffer, void *ref, int size, int pe, int tag, int type);
 
 int getNewMPITag(void){
@@ -67,84 +26,6 @@ int getNewMPITag(void){
   return newRdmaTag;
 }
 
-int LrtsGetRdmaOpInfoSize(void){
-  return sizeof(CmiMPIRzvRdmaOp_t);
-}
-
-int LrtsGetRdmaGenInfoSize(void){
-  return sizeof(CmiMPIRzvRdma_t);
-}
-
-int LrtsGetRdmaInfoSize(int numOps){
-  return sizeof(CmiMPIRzvRdma_t) + numOps * sizeof(CmiMPIRzvRdmaOp_t);
-}
-
-int LrtsGetRdmaOpRecvInfoSize(void){
-  return sizeof(CmiMPIRzvRdmaRecvOp_t);
-}
-
-int LrtsGetRdmaGenRecvInfoSize(void){
-  return sizeof(CmiMPIRzvRdmaRecvList_t);
-}
-
-int LrtsGetRdmaRecvInfoSize(int numOps){
-  return sizeof(CmiMPIRzvRdmaRecvList_t) + numOps * sizeof(CmiMPIRzvRdmaRecvOp_t);
-}
-
-void LrtsSetRdmaRecvInfo(void *rdmaRecv, int numOps, void *msg, void *rdmaSend, int msgSize){
-  CmiMPIRzvRdmaRecvList_t *rdmaRecvInfo = (CmiMPIRzvRdmaRecvList_t *)rdmaRecv;
-  CmiMPIRzvRdma_t *rdmaSendInfo = (CmiMPIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvInfo->numOps = numOps;
-  rdmaRecvInfo->counter = 0;
-  rdmaRecvInfo->srcPe = rdmaSendInfo->srcPe;
-  rdmaRecvInfo->msg = msg;
-  rdmaRecvInfo->msgLen = msgSize;
-}
-
-void LrtsSetRdmaRecvOpInfo(void *rdmaRecvOp, void *buffer, void *src_ref, int size, int opIndex, void *rdmaSend){
-  CmiMPIRzvRdmaRecvOp_t *rdmaRecvOpInfo = (CmiMPIRzvRdmaRecvOp_t *)rdmaRecvOp;
-  CmiMPIRzvRdma_t *rdmaSendInfo = (CmiMPIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvOpInfo->buffer = buffer;
-  rdmaRecvOpInfo->size = size;
-  rdmaRecvOpInfo->src_info = src_ref;
-
-  rdmaRecvOpInfo->opIndex = opIndex;
-  rdmaRecvOpInfo->tag = rdmaSendInfo->rdmaOp[opIndex].tag;
-}
-
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps){
-  CmiMPIRzvRdma_t *rdma = (CmiMPIRzvRdma_t *)dest;
-  rdma->numOps = numOps;
-  rdma->srcPe = CmiMyPe();
-}
-
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  CmiMPIRzvRdmaOp_t *rdmaOp = (CmiMPIRzvRdmaOp_t *)dest;
-  rdmaOp->size = size;
-
-  //Generate a new tag to be used for the RDMA buffer
-  rdmaOp->tag = getNewMPITag();
-
-  CmiMPIRzvRdmaOpInfo_t *rdmaOpInfo = (CmiMPIRzvRdmaOpInfo_t *) malloc(sizeof(CmiMPIRzvRdmaOpInfo_t));
-  rdmaOpInfo->ack = ack;
-  rdmaOpInfo->tag = rdmaOp->tag;
-
-  // Post the RDMA buffer with the generated tag using MPI_Isend. Post MPI_Isend directly for non-smp or through the comm thread for smp mode
-  MPIPostOneBuffer(ptr, (void *)rdmaOpInfo, size, destPE, rdmaOp->tag, ONESIDED_BUFFER_SEND);
-}
-
-// Structure used for the Nocopy Direct API to request an MPI rank to post a buffer
-typedef struct _cmi_mpi_rzv_rdma_post_info {
-  void *buffer;
-  int size;
-  int tag;
-  int ackSize;
-  int srcPe;
-  int destPe;
-}CmiMPIRzvRdmaPostInfo_t;
-
 // Set the machine specific information for a nocopy pointer (Empty method to maintain API consistency)
 void LrtsSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode){
 }
index 2e54d1e0d939cb47b9e7a47f568a0a10c3b94319..216bf9de7ad9d9d49363d810fe8ac9f8fcdffa9e 100644 (file)
@@ -610,6 +610,7 @@ static void ReleasePostedMessages(void) {
     SMSG_LIST *msg_tmp=CpvAccess(sent_msgs);
     SMSG_LIST *prev=0;
     SMSG_LIST *temp;
+
     int done;
     MPI_Status sts;
 
@@ -641,41 +642,28 @@ static void ReleasePostedMessages(void) {
             if(CpvAccess(end_sent) == msg_tmp) {
               CpvAccess(end_sent) = prev;
             }
-            //if rdma msg, call the callback
-            if(msg_tmp->type == ONESIDED_BUFFER_SEND) {
-                CmiMPIRzvRdmaOpInfo_t *rdmaOpInfo = (CmiMPIRzvRdmaOpInfo_t *)msg_tmp->ref;
-                CmiRdmaAck *ack = (CmiRdmaAck *) rdmaOpInfo->ack;
-                ack->fnPtr(ack->token);
-
-                //free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
-                free(ack);
-            } else if(msg_tmp->type == ONESIDED_BUFFER_RECV) {
-
-                // get hold of CmiMPIRzvRdmaRecvList_t
-                CmiMPIRzvRdmaRecvOp_t *rdmaRecvOpInfo = (CmiMPIRzvRdmaRecvOp_t *)msg_tmp->ref;
-
-                CmiMPIRzvRdmaRecvList_t *rdmaRecvInfo = (CmiMPIRzvRdmaRecvList_t *)((char *)(rdmaRecvOpInfo)
-                                                        - rdmaRecvOpInfo->opIndex * LrtsGetRdmaOpRecvInfoSize()
-                                                        - LrtsGetRdmaGenRecvInfoSize());
-
-                rdmaRecvInfo->counter++;
-                if(rdmaRecvInfo->counter == rdmaRecvInfo->numOps) {
-                    handleOneRecvedMsg(rdmaRecvInfo->msgLen, (char *)rdmaRecvInfo->msg);
-                }
-            } else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV) {
+            if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV) {
                 // MPI_Irecv posted as a part of the Direct API was completed
                 NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(msg_tmp->ref);
+
                 // Invoke the destination ack
-                ncpyOpInfo->ackMode = 2;
+                ncpyOpInfo->ackMode = CMK_DEST_ACK; // Only invoke destination ack
+
+                // On the destination the NcpyOperationInfo is freed for the Direct API
+                // but not freed for the Entry Method API as it a part of the parameter marshalled message
+                // and is enentually freed by the RTS
                 CmiInvokeNcpyAck(ncpyOpInfo);
 
             } else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_SEND) {
                 // MPI_Isend posted as a part of the Direct API was completed
                 NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(msg_tmp->ref);
                 // Invoke the source ack
-                ncpyOpInfo->ackMode = 1;
-                CmiInvokeNcpyAck(ncpyOpInfo);
+                ncpyOpInfo->ackMode = CMK_SRC_ACK; // Only invoke the source ack
+
+                // Free the NcpyOperationInfo on the source
+                ncpyOpInfo->freeMe = CMK_FREE_NCPYOPINFO;
 
+                CmiInvokeNcpyAck(ncpyOpInfo);
             }
             else if(msg_tmp->type == POST_DIRECT_SEND || msg_tmp->type == POST_DIRECT_RECV) {
                 // do nothing as the received message is a NcpyOperationInfo object
@@ -711,13 +699,6 @@ static int PumpMsgs(void) {
     MPI_Status sts;
     int recd=0;
 
-#if CMK_ONESIDED_IMPL
-    CmiMPIRzvRdmaRecvList_t *recvBufferTmp = CpvAccess(recvRdmaBuffers);
-    CmiMPIRzvRdmaRecvList_t *prev = 0;
-    CmiMPIRzvRdmaRecvList_t *temp;
-    int opDone, allOpsDone, i;
-#endif
-
 #if CMI_EXERT_RECV_CAP || CMI_DYNAMIC_EXERT_CAP
     int recvCnt=0;
 #endif
@@ -1088,15 +1069,7 @@ static int SendMsgBuf(void) {
 #endif
 
 #if CMK_ONESIDED_IMPL
-            if(msg_tmp->type == ONESIDED_BUFFER_SEND) {
-                CmiMPIRzvRdmaOpInfo_t *rdmaOpInfo = (CmiMPIRzvRdmaOpInfo_t *)(msg_tmp->ref);
-                MPISendOrRecvOneBuffer(msg_tmp, rdmaOpInfo->tag);
-            }
-            else if(msg_tmp->type == ONESIDED_BUFFER_RECV) {
-                CmiMPIRzvRdmaRecvOp_t *rdmaRecvOpInfo = (CmiMPIRzvRdmaRecvOp_t *)(msg_tmp->ref);
-                MPISendOrRecvOneBuffer(msg_tmp, rdmaRecvOpInfo->tag);
-            }
-            else if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV || msg_tmp->type == ONESIDED_BUFFER_DIRECT_SEND) {
+            if(msg_tmp->type == ONESIDED_BUFFER_DIRECT_RECV || msg_tmp->type == ONESIDED_BUFFER_DIRECT_SEND) {
                 NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(msg_tmp->ref);
                 MPISendOrRecvOneBuffer(msg_tmp, ncpyOpInfo->tag);
             }
@@ -1814,16 +1787,6 @@ void LrtsPostCommonInit(int everReturn) {
     CpvAccess(end_sent) = NULL;
     CpvAccess(MsgQueueLen) = 0;
 
-#if CMK_ONESIDED_IMPL
-    //List for storing the receiver's rdma request information
-    CpvInitialize(CmiMPIRzvRdmaRecvList_t *, recvRdmaBuffers);
-    CpvInitialize(CmiMPIRzvRdmaRecvList_t *, endRdmaBuffer);
-    CpvInitialize(int, RdmaRecvQueueLen);
-    CpvAccess(recvRdmaBuffers) = NULL;
-    CpvAccess(endRdmaBuffer) = NULL;
-    CpvAccess(RdmaRecvQueueLen) = 0;
-#endif
-
 #if CMI_MACH_TRACE_USEREVENTS && CMK_TRACE_ENABLED && !CMK_TRACE_IN_CHARM
     CpvInitialize(double, projTraceStart);
     /* only PE 0 needs to care about registration (to generate sts file). */
index 8fdf9c8f501edde02650fbaa8cc370d531e631f6..08ba60108d6acc566a388d532bce4a17bb50f874 100644 (file)
@@ -1,24 +1,4 @@
-void ofi_onesided_all_ops_done(char *msg) {
-  int stdlen = ((CmiMsgHeaderBasic *) msg)->size;
-  // Invoke the message handler
-  handleOneRecvedMsg(stdlen, msg);
-}
-
-void process_onesided_completion_ack(struct fi_cq_tagged_entry *e, OFIRequest *req)
-{
-  struct fid *mr;
-  OfiRdmaAck_t *ofiAck = (OfiRdmaAck_t *)req->data.rma_ncpy_ack;
-  CmiRdmaAck *ack = (CmiRdmaAck *)(ofiAck->src_ref);
-  // Invoke the ack handler function
-  ack->fnPtr(ack->token);
-
-  // Deregister the buffer
-  mr = (struct fid *)(ofiAck->src_mr);
-  CmiAssert(mr);
-  fi_close(mr);
-}
-
-void registerDirectMemory(void *info, const void *addr, int size) {
+struct fid_mr* registerDirectMemory(void *info, const void *addr, int size) {
   CmiOfiRdmaPtr_t *rdmaInfo = (CmiOfiRdmaPtr_t *)info;
   uint64_t requested_key = 0;
   int ret;
@@ -42,58 +22,6 @@ void registerDirectMemory(void *info, const void *addr, int size) {
 }
 
 
-static inline void ofi_onesided_send_ack_callback(struct fi_cq_tagged_entry *e, OFIRequest *req)
-{
-#if USE_OFIREQUEST_CACHE
-  free_request(req);
-#else
-  CmiFree(req);
-#endif
-}
-
-static inline void ofi_onesided_read_callback(struct fi_cq_tagged_entry *e, OFIRequest *req)
-{
-  CmiOfiRdmaRecvOp_t *rdmaRecvOpInfo = (CmiOfiRdmaRecvOp_t *)(req->data.rma_ncpy_info);
-  rdmaRecvOpInfo->completion_count--;
-  struct fid* memregion;
-
-  if(0 == rdmaRecvOpInfo->completion_count) {
-    CmiOfiRdmaRecv_t *recvInfo = (CmiOfiRdmaRecv_t *)((char *)rdmaRecvOpInfo
-                        - rdmaRecvOpInfo->opIndex * sizeof(CmiOfiRdmaRecvOp_t)
-                        - sizeof(CmiOfiRdmaRecv_t));
-
-    req->callback      = ofi_onesided_send_ack_callback;
-
-    // Send an acknowledgement message to the sender to indicate the completion of the RDMA operation
-    ofi_send(&(rdmaRecvOpInfo->ack),
-             sizeof(OfiRdmaAck_t),
-             rdmaRecvOpInfo->src_nodeNo,
-             OFI_RDMA_OP_ACK,
-             req);
-
-    recvInfo->comOps++;
-
-    // Store the memregion for registration
-    memregion = (struct fid *)rdmaRecvOpInfo->mr;
-
-    if(recvInfo->comOps == recvInfo->numOps) {
-      // All the RDMA operations for one entry method have completed
-      ofi_onesided_all_ops_done((char*)recvInfo->msg);
-    }
-
-    // Deregister the memory region
-    if(memregion)
-      fi_close(memregion);
-
-  } else {
-#if USE_OFIREQUEST_CACHE
-    free_request(req);
-#else
-    CmiFree(req);
-#endif
-  }
-}
-
 void ofi_post_nocopy_operation(
   char *lbuf,
   const char *rbuf,
@@ -171,26 +99,6 @@ void ofi_post_nocopy_operation(
   }
 }
 
-void LrtsIssueRgets(void *recv, int pe) {
-  CmiOfiRdmaRecv_t* recvInfo = (CmiOfiRdmaRecv_t *)recv;
-  int i;
-  for(i = 0; i < recvInfo->numOps; i++) {
-    CmiOfiRdmaRecvOp_t *rdmaRecvOpInfo = &(recvInfo->rdmaOp[i]);
-    const char *rbuf        = (FI_MR_SCALABLE == context.mr_mode) ? 0 : (const char*)rdmaRecvOpInfo->src_buf;
-    ofi_post_nocopy_operation(
-        (char *)rdmaRecvOpInfo->buf,
-        rbuf,
-        rdmaRecvOpInfo->src_nodeNo,
-        rdmaRecvOpInfo->src_key,
-        rdmaRecvOpInfo->len,
-        rdmaRecvOpInfo->mr,
-        ofi_onesided_read_callback,
-        (void *)rdmaRecvOpInfo,
-        &(rdmaRecvOpInfo->completion_count),
-        OFI_READ_OP);
-  }
-}
-
 /* Support for Nocopy Direct API */
 // Method called on the completion of a direct onesided operation
 
@@ -226,8 +134,8 @@ void process_onesided_reg_and_put(struct fi_cq_tagged_entry *e, OFIRequest *req)
   NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(data);
   resetNcpyOpInfoPointers(ncpyOpInfo);
 
-  // Do not free as this message
-  ncpyOpInfo->freeMe = 0;
+  // Free the NcpyOperationInfo in a reverse API operation
+  ncpyOpInfo->freeMe = CMK_FREE_NCPYOPINFO;
 
   registerDirectMemory(ncpyOpInfo->srcLayerInfo + CmiGetRdmaCommonInfoSize(),
                        ncpyOpInfo->srcPtr,
@@ -263,6 +171,9 @@ void process_onesided_reg_and_get(struct fi_cq_tagged_entry *e, OFIRequest *req)
   NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(data);
   resetNcpyOpInfoPointers(ncpyOpInfo);
 
+  // Free this message
+  ncpyOpInfo->freeMe = CMK_FREE_NCPYOPINFO;
+
   registerDirectMemory(ncpyOpInfo->destLayerInfo + CmiGetRdmaCommonInfoSize(),
                        ncpyOpInfo->destPtr,
                        ncpyOpInfo->srcSize);
@@ -304,6 +215,14 @@ void LrtsIssueRget(NcpyOperationInfo *ncpyOpInfo) {
 
     ZERO_REQUEST(req);
 
+    // set OpMode for reverse operation
+    setReverseModeForNcpyOpInfo(ncpyOpInfo);
+
+    if(ncpyOpInfo->freeMe == CMK_FREE_NCPYOPINFO)
+      req->freeMe   = 1;// free ncpyOpInfo
+    else
+      req->freeMe   = 0;// do not free ncpyOpInfo
+
     req->destNode = CmiNodeOf(ncpyOpInfo->srcPe);
     req->destPE   = ncpyOpInfo->srcPe;
     req->size     = ncpyOpInfo->ncpyOpInfoSize;
index 9fe4ef4b2fe3e17d6108069bf7fb6157b1e2d454..f4e970e78c30e26f8f69183f7396fc2d4542a28f 100644 (file)
@@ -5,162 +5,6 @@
 // Signature of the callback function to be called on rdma direct operation completion
 typedef void (*ofiCallbackFn)(struct fi_cq_tagged_entry *e, OFIRequest *req);
 
-// Structure for sender side machine specific information for a buffer
-typedef struct _cmi_ofi_rdma_op {
-  int         len;
-  uint64_t    key;
-  uint64_t    mr;
-  int         nodeNo;
-  void        *ref;
-  const void  *buf;
-} CmiOfiRdmaSendOp_t;
-
-// Structure for sender side machine specific information for multiple buffers
-typedef struct _cmi_ofi_rdma{
-  int            numOps;
-  CmiOfiRdmaSendOp_t rdmaOp[0];
-} CmiOfiRdmaSend_t;
-
-// Structure for representing the acknowledgement information for a buffer
-// This sturcture variable is sent by the receiver side on completion of an RDMA operation
-typedef struct _ofi_rdma_ack {
-  void     *src_ref;
-  uint64_t src_mr;
-} OfiRdmaAck_t;
-
-// Structure for sender side machine specific information for a buffer
-typedef struct _cmi_ofi_rdma_recv_op {
-  int           len;
-  uint64_t      src_key;
-  int           src_nodeNo;
-  const void    *src_buf;
-  OfiRdmaAck_t    ack;
-  int           completion_count;
-  int           opIndex;
-  const void    *buf;
-  struct        fid_mr *mr;
-} CmiOfiRdmaRecvOp_t;
-
-// Structure for receiver side machine specific information for multiple buffers
-typedef struct _cmi_ofi_rdma_recv {
-  int numOps;
-  int comOps;
-  void *msg;
-  CmiOfiRdmaRecvOp_t rdmaOp[0];
-} CmiOfiRdmaRecv_t;
-
-/* Sender Side Functions */
-int LrtsGetRdmaOpInfoSize(){
-  return sizeof(CmiOfiRdmaSendOp_t);
-}
-
-int LrtsGetRdmaGenInfoSize(){
-  return sizeof(CmiOfiRdmaSend_t);
-}
-
-int LrtsGetRdmaInfoSize(int numOps){
-  return sizeof(CmiOfiRdmaSend_t) + numOps * sizeof(CmiOfiRdmaSendOp_t);
-}
-
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps){
-  CmiOfiRdmaSend_t *rdma = (CmiOfiRdmaSend_t*)dest;
-  rdma->numOps = numOps;
-}
-
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  struct fid_mr *mr;
-  uint64_t requested_key = 0;
-  int ret;
-
-  /* Register the source buffer */
-  if(FI_MR_SCALABLE == context.mr_mode) {
-    requested_key = __sync_fetch_and_add(&(context.mr_counter), 1);
-  }
-  ret = fi_mr_reg(context.domain,
-                  ptr,
-                  size,
-                  MR_ACCESS_PERMISSIONS,
-                  0ULL,
-                  requested_key,
-                  0ULL,
-                  &mr,
-                  NULL);
-
-  if (ret) {
-    CmiAbort("LrtsSetRdmaInfo: fi_mr_reg failed!\n");
-  }
-
-  CmiOfiRdmaSendOp_t *rdmaSendOpInfo = (CmiOfiRdmaSendOp_t *)dest;
-  rdmaSendOpInfo->nodeNo  = CmiMyNodeGlobal();
-  rdmaSendOpInfo->buf     = ptr;
-  rdmaSendOpInfo->len     = size;
-  rdmaSendOpInfo->key     = fi_mr_key(mr);
-  rdmaSendOpInfo->mr      = (uint64_t)mr;
-  rdmaSendOpInfo->ref     = ack;
-}
-
-/* Receiver Side Functions */
-int LrtsGetRdmaOpRecvInfoSize(){
-  return sizeof(CmiOfiRdmaRecvOp_t);
-}
-
-int LrtsGetRdmaGenRecvInfoSize(){
-  return sizeof(CmiOfiRdmaRecv_t);
-}
-
-int LrtsGetRdmaRecvInfoSize(int numOps){
-  return sizeof(CmiOfiRdmaRecv_t) + numOps * sizeof(CmiOfiRdmaRecvOp_t);
-}
-
-void LrtsSetRdmaRecvInfo(void *rdmaRecv, int numOps, void *msg, void *rdmaSend, int msgSize){
-  CmiOfiRdmaRecv_t *rdmaRecvInfo = (CmiOfiRdmaRecv_t *)rdmaRecv;
-  CmiOfiRdmaSend_t *rdmaSendInfo = (CmiOfiRdmaSend_t *)rdmaSend;
-
-  rdmaRecvInfo->numOps = numOps;
-  rdmaRecvInfo->comOps = 0;
-  rdmaRecvInfo->msg = msg;
-}
-
-void LrtsSetRdmaRecvOpInfo(void *rdmaRecvOp, void *buffer, void *src_ref, int size, int opIndex, void *rdmaSend){
-  int ret, len;
-  struct fid_mr *mr = NULL;
-  CmiOfiRdmaRecvOp_t *rdmaRecvOpInfo = (CmiOfiRdmaRecvOp_t *)rdmaRecvOp;
-  CmiOfiRdmaSend_t *rdmaSendInfo = (CmiOfiRdmaSend_t *)rdmaSend;
-  CmiOfiRdmaSendOp_t rdmaSendOpInfo = rdmaSendInfo->rdmaOp[opIndex];
-
-  len = rdmaSendOpInfo.len;
-
-  rdmaRecvOpInfo->len              = len;
-  rdmaRecvOpInfo->src_key          = rdmaSendOpInfo.key;
-  rdmaRecvOpInfo->src_nodeNo       = rdmaSendOpInfo.nodeNo;
-  rdmaRecvOpInfo->src_buf          = rdmaSendOpInfo.buf;
-  rdmaRecvOpInfo->ack.src_ref      = rdmaSendOpInfo.ref;
-  rdmaRecvOpInfo->ack.src_mr       = rdmaSendOpInfo.mr;
-  rdmaRecvOpInfo->completion_count = 0;
-  rdmaRecvOpInfo->opIndex          = opIndex;
-  rdmaRecvOpInfo->buf              = buffer;
-
-  if (FI_MR_BASIC == context.mr_mode) {
-    /* Register local MR to be read into */
-    ret = fi_mr_reg(context.domain,
-                    buffer,
-                    len,
-                    MR_ACCESS_PERMISSIONS,
-                    0ULL,
-                    0ULL,
-                    0ULL,
-                    &mr,
-                    NULL);
-
-    if (ret) {
-      CmiAbort("LrtsSetRdmaRecvOpInfo: fi_mr_reg failed!\n");
-    }
-  }
-  rdmaRecvOpInfo->mr            = mr;
-}
-
-
-inline void ofi_onesided_all_ops_done(char *msg);
 inline void process_onesided_completion_ack(struct fi_cq_tagged_entry *e, OFIRequest *req);
 inline void process_onesided_reg_and_put(struct fi_cq_tagged_entry *e, OFIRequest *req);
 inline void process_onesided_reg_and_get(struct fi_cq_tagged_entry *e, OFIRequest *req);
index dfc55e2de236526edc46cf3d0222395730372789..92705e1d265a955f627f464eebad79c4eca65aba 100644 (file)
@@ -136,7 +136,6 @@ CpvDeclare(mempool_type*, mempool);
 #define OFI_RDMA_DIRECT_REG_AND_PUT 0x4ULL
 #define OFI_RDMA_DIRECT_REG_AND_GET 0x5ULL
 
-#define OFI_RDMA_OP_ACK 0x7ULL
 #define OFI_OP_NAMES 0x8ULL
 
 #define OFI_READ_OP 1
@@ -674,7 +673,9 @@ void send_short_callback(struct fi_cq_tagged_entry *e, OFIRequest *req)
     msg = (char *)req->data.short_msg;
     CmiAssert(msg);
     MACHSTATE1(3, "--> msg=%p", msg);
-    CmiFree(msg);
+
+    if(req->freeMe)
+      CmiFree(msg);
 
 #if USE_OFIREQUEST_CACHE
     free_request(req);
@@ -1145,9 +1146,6 @@ void recv_callback(struct fi_cq_tagged_entry *e, OFIRequest *req)
     case OFI_OP_ACK:
         process_long_send_ack(e, req);
         break;
-    case OFI_RDMA_OP_ACK:
-        process_onesided_completion_ack(e, req);
-        break;
     case OFI_RDMA_DIRECT_REG_AND_PUT:
         process_onesided_reg_and_put(e, req);
         break;
index 420553071fcbe9282731140f099948467eccb7ec..3a30e8da22c4df1d6acd0cdfbdfe840fdf966a7b 100644 (file)
@@ -24,6 +24,7 @@
         (r)->size             = 0; \
         (r)->mode             = 0; \
         (r)->data.recv_buffer = 0; \
+        (r)->freeMe           = 1; \
     } while (0)
 
 typedef enum request_state
@@ -67,6 +68,7 @@ typedef struct OFIRequest
     int                destPE;
     int                size;
     int                mode;
+    int                freeMe;
     struct fid_mr      *mr;
     union
     {
index 0704a41f38c013a165c34872e7aad009bfbf8738..5955b283de4aab54fa5e155a6fac1abffb438b94 100644 (file)
@@ -11,13 +11,6 @@ void _initOnesided( pami_context_t *contexts, int nc){
   pami_dispatch_callback_function pfn;
   int i = 0;
   for (i = 0; i < nc; ++i) {
-    pfn.p2p = ack_rdma_pkt_dispatch;
-    PAMI_Dispatch_set (contexts[i],
-      CMI_PAMI_ONESIDED_ACK_DISPATCH,
-      pfn,
-      NULL,
-      options);
-
     pfn.p2p = rdma_direct_get_dispatch;
     PAMI_Dispatch_set (contexts[i],
       CMI_PAMI_DIRECT_GET_DISPATCH,
@@ -27,24 +20,6 @@ void _initOnesided( pami_context_t *contexts, int nc){
  }
 }
 
-//function called on completion of the rdma operation
-void rzv_rdma_recv_done   (pami_context_t     ctxt,
-    void             * clientdata,
-    pami_result_t      result)
-{
-  CmiPAMIRzvRdmaRecvOp_t* recvOpInfo = (CmiPAMIRzvRdmaRecvOp_t *)clientdata;
-  CmiPAMIRzvRdmaRecv_t* recvInfo = (CmiPAMIRzvRdmaRecv_t *)(
-                                        ((char *)recvOpInfo)
-                                      - recvOpInfo->opIndex * sizeof(CmiPAMIRzvRdmaRecvOp_t)
-                                      - sizeof(CmiPAMIRzvRdmaRecv_t));
-
-  rdma_sendAck(ctxt, recvOpInfo, recvInfo->src_ep);
-  recvInfo->comOps++;
-  if(recvInfo->comOps == recvInfo->numOps){
-    recv_done(ctxt, recvInfo->msg, PAMI_SUCCESS);
-  }
-}
-
 //function to perform Rget
 pami_result_t getData(
   pami_context_t       context,
@@ -131,71 +106,6 @@ pami_result_t putData(
   return res;
 }
 
-
-void LrtsIssueRgets(void *recv, int pe){
-
-  CmiPAMIRzvRdmaRecv_t* recvInfo = (CmiPAMIRzvRdmaRecv_t*)recv;
-
-#if CMK_SMP && CMK_ENABLE_ASYNC_PROGRESS
-  int c = CmiMyNode() % cmi_pami_numcontexts;
-  pami_context_t my_context = cmi_pami_contexts[c];
-#else
-  pami_context_t my_context= MY_CONTEXT();
-#endif
-
-  INCR_ORECVS();
-
-
-  pami_endpoint_t      origin;
-  PAMI_Endpoint_create (cmi_pami_client, (pami_task_t)CmiNodeOf(pe), recvInfo->dstContext, &origin);
-  recvInfo->src_ep = origin;
-
-  int i;
-  for(i=0; i<recvInfo->numOps; i++){
-    void *buffer = recvInfo->rdmaOp[i].buffer;
-    int offset = recvInfo->rdmaOp[i].offset;
-    int size = recvInfo->rdmaOp[i].size;
-    MACHSTATE5(3, "[%d]LrtsIssueRgets, recv:%p, origin: %u, dst_context: %d, offset: %d\n", CmiMyPe(), recvInfo, origin, recvInfo->dstContext, offset);
-    getData(my_context, origin, buffer, &recvInfo->rdmaOp[i],
-          recvInfo->dstContext, rzv_rdma_recv_done, offset, &recvInfo->mregion, size);
-  }
-}
-
-//function to send the acknowledgement to the sender
-void  rdma_sendAck (
-    pami_context_t      context,
-    CmiPAMIRzvRdmaRecvOp_t* recvOpInfo,
-    int src_ep)
-{
-  pami_send_immediate_t parameters;
-  parameters.dispatch        = CMI_PAMI_ONESIDED_ACK_DISPATCH;
-  parameters.header.iov_base = &recvOpInfo->src_info;
-  parameters.header.iov_len  = sizeof(void *);
-  parameters.data.iov_base   = NULL;
-  parameters.data.iov_len    = 0;
-  parameters.dest            = src_ep;
-  PAMI_Send_immediate (context, &parameters);
-}
-
-
-// function called on the sender on receiving acknowledgement from the receiver to signal the completion of the rdma operation
-void ack_rdma_pkt_dispatch (
-    pami_context_t       context,
-    void               * clientdata,
-    const void         * header_addr,
-    size_t               header_size,
-    const void         * pipe_addr,
-    size_t               pipe_size,
-    pami_endpoint_t      origin,
-    pami_recv_t         * recv)
-{
-  CmiAssert(sizeof(void *) == header_size);
-  CmiRdmaAck *ack = *((CmiRdmaAck **) header_addr);
-  ack->fnPtr(ack->token);
-  //free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
-  free(ack);
-}
-
 /* Support for Nocopy Direct API */
 
 // Function called on completion of the receive operation for ncpyOpInfo object
index 593c71031a37dcb06648d8d16da226a67a791827..ffb49cd23af88d41cead01f64de346664a53eef7 100644 (file)
@@ -1,124 +1,5 @@
 void _initOnesided( pami_context_t *contexts, int nc);
 
-typedef struct _cmi_pami_rzv_rdma_op {
-  int offset;
-  int size;
-} CmiPAMIRzvRdmaOp_t;
-
-typedef struct _cmi_pami_rzv_rdma{
-  int numOps;
-  size_t dstContext;
-  pami_memregion_t    mregion;
-  CmiPAMIRzvRdmaOp_t rdmaOp[0];
-} CmiPAMIRzvRdma_t;
-
-typedef struct _cmi_pami_rzv_rdma_recv_op {
-  void           * buffer;
-  void           * src_info;
-  int              offset;
-  int              size;
-  int              opIndex;
-} CmiPAMIRzvRdmaRecvOp_t;
-
-typedef struct _cmi_pami_rzv_rdma_recv {
-  int           src_ep;
-  int           numOps;
-  int           comOps;
-  void*         msg;
-  size_t           dstContext;
-  pami_memregion_t    mregion;
-  CmiPAMIRzvRdmaRecvOp_t rdmaOp[0];
-}CmiPAMIRzvRdmaRecv_t;
-
-void ack_rdma_pkt_dispatch (pami_context_t       context,
-    void               * clientdata,
-    const void         * header_addr,
-    size_t               header_size,
-    const void         * pipe_addr,
-    size_t               pipe_size,
-    pami_endpoint_t      origin,
-    pami_recv_t         * recv);
-
-void  rdma_sendAck (
-    pami_context_t      context,
-    CmiPAMIRzvRdmaRecvOp_t* recvOpInfo,
-    int src_ep);
-
-void rzv_rdma_recv_done   (
-    pami_context_t     ctxt,
-    void             * clientdata,
-    pami_result_t      result);
-
-int LrtsGetRdmaOpInfoSize(void){
-  return sizeof(CmiPAMIRzvRdmaOp_t);
-}
-int LrtsGetRdmaGenInfoSize(void){
-  return sizeof(CmiPAMIRzvRdma_t);
-}
-int LrtsGetRdmaInfoSize(int numOps){
-  return sizeof(CmiPAMIRzvRdma_t) + numOps * sizeof(CmiPAMIRzvRdmaOp_t);
-}
-
-int LrtsGetRdmaOpRecvInfoSize(void){
-  return sizeof(CmiPAMIRzvRdmaRecvOp_t);
-}
-
-int LrtsGetRdmaGenRecvInfoSize(void){
-  return sizeof(CmiPAMIRzvRdmaRecv_t);
-}
-
-int LrtsGetRdmaRecvInfoSize(int numOps){
-  return sizeof(CmiPAMIRzvRdmaRecv_t) + numOps * sizeof(CmiPAMIRzvRdmaRecvOp_t);
-}
-
-void LrtsSetRdmaRecvInfo(void *rdmaRecv, int numOps, void *msg, void *rdmaSend, int msgSize){
-
-  CmiPAMIRzvRdmaRecv_t *rdmaRecvInfo = (CmiPAMIRzvRdmaRecv_t *)rdmaRecv;
-  CmiPAMIRzvRdma_t *rdmaSendInfo = (CmiPAMIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvInfo->numOps = numOps;
-  rdmaRecvInfo->comOps = 0;
-  rdmaRecvInfo->msg = msg;
-
-  rdmaRecvInfo->dstContext = rdmaSendInfo->dstContext;
-  memcpy(&rdmaRecvInfo->mregion, &rdmaSendInfo->mregion, sizeof(pami_memregion_t));
-}
-
-void LrtsSetRdmaRecvOpInfo(void *rdmaRecvOp, void *buffer, void *src_ref, int size, int opIndex, void *rdmaSend){
-  CmiPAMIRzvRdmaRecvOp_t *rdmaRecvOpInfo = (CmiPAMIRzvRdmaRecvOp_t *)rdmaRecvOp;
-  CmiPAMIRzvRdma_t *rdmaSendInfo = (CmiPAMIRzvRdma_t *)rdmaSend;
-
-  rdmaRecvOpInfo->buffer = buffer;
-  rdmaRecvOpInfo->src_info = src_ref;
-  rdmaRecvOpInfo->size = size;
-  rdmaRecvOpInfo->opIndex = opIndex;
-
-  rdmaRecvOpInfo->offset = rdmaSendInfo->rdmaOp[opIndex].offset;
-}
-
-
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps){
-
-  CmiPAMIRzvRdma_t *rdma = (CmiPAMIRzvRdma_t*)dest;
-  int rank = CmiRankOf(destPE);
-
-#if CMK_PAMI_MULTI_CONTEXT &&  CMK_NODE_QUEUE_AVAILABLE
-  size_t dst_context = (rank != DGRAM_NODEMESSAGE) ? (rank>>LTPS) : (rand_r(&r_seed) % cmi_pami_numcontexts);
-#else
-  size_t dst_context = 0;
-#endif
-
-  rdma->dstContext = dst_context;
-  memcpy(&rdma->mregion, &cmi_pami_memregion[0].mregion, sizeof(pami_memregion_t));
-  rdma->numOps = numOps;
-}
-
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  CmiPAMIRzvRdmaOp_t *rdmaOp = (CmiPAMIRzvRdmaOp_t *)dest;
-  rdmaOp->offset = (size_t)(ptr) - (size_t)cmi_pami_memregion[0].baseVA;
-  rdmaOp->size = size;
-}
-
 /* Support for Nocopy Direct API */
 
 // Machine specific information for a nocopy pointer
index 49e5d5974d2ccdd792401fda155704266eecac06..e9b47967a12fee78e9645d88ca33c0692ca5b069 100644 (file)
@@ -1,17 +1,9 @@
 #ifndef _MACHINE_RDMA_H_
 #define _MACHINE_RDMA_H_
 
-typedef void (*RdmaSingleAckHandlerFn)(void *cbPtr, int pe, const void *ptr);
 /*Function Pointer to Acknowledgement Handler*/
 typedef void (*RdmaAckHandlerFn)(void *token);
 
-/*Acknowledgement constisting of handler and token*/
-typedef struct _cmi_rdma_ack{
-  // Function Pointer to Acknowledgment handler function for the Indirect API
-  RdmaAckHandlerFn fnPtr;
-  void *token;
-} CmiRdmaAck;
-
 /* Support for Nocopy Direct API */
 typedef struct _cmi_common_rdma_info {
 #if CMK_USE_CMA
@@ -91,102 +83,10 @@ void CmiIssueRputUsingCMA(
 }
 #endif
 
-#if CMK_ONESIDED_IMPL
-/*Lrts Function declarations*/
-
-/*Sender Functions*/
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps);
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE);
-int LrtsGetRdmaOpInfoSize(void);
-int LrtsGetRdmaGenInfoSize(void);
-int LrtsGetRdmaInfoSize(int numOps);
-void LrtsSetRdmaRecvInfo(void *dest, int numOps, void *charmMsg, void *rdmaInfo, int msgSize);
-
-/*Receiver Functions*/
-void LrtsSetRdmaRecvOpInfo(void *dest, void *buffer, void *src_ref, int size, int opIndex, void *rdmaRecv);
-int LrtsGetRdmaOpRecvInfoSize(void);
-int LrtsGetRdmaGenRecvInfoSize(void);
-int LrtsGetRdmaRecvInfoSize(int numOps);
-void LrtsIssueRgets(void *recv, int pe);
-/* Converse Machine Interface Functions*/
-
-/* Sender Side Functions */
-
-/* Set the machine layer info generic to RDMA ops*/
-void CmiSetRdmaInfo(void *dest, int destPE, int numOps){
-  LrtsSetRdmaInfo(dest, destPE, numOps);
-}
-
-/* Set the machine layer info specific to RDMA op*/
-void CmiSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  LrtsSetRdmaOpInfo(dest, ptr, size, ack, destPE);
-}
-
-/* Getter for size help upper layers allocate space for machine layer info
- * while allocating the message*/
-
-/* Get the size occupied by the machine layer info specific to RDMA op*/
-int CmiGetRdmaOpInfoSize(void){
-  return LrtsGetRdmaOpInfoSize();
-}
-
-/* Get the size occupied by the macine layer info generic to RDMA ops*/
-int CmiGetRdmaGenInfoSize(void){
-  return LrtsGetRdmaGenInfoSize();
-}
-
-/* Get the total size occupied by the machine layer info (specific + generic)*/
-int CmiGetRdmaInfoSize(int numOps){
-  return LrtsGetRdmaInfoSize(numOps);
-}
-
-/* Set the ack function handler and token*/
-void *CmiSetRdmaAck(RdmaAckHandlerFn fn, void *token){
-  CmiRdmaAck *ack = (CmiRdmaAck *)malloc(sizeof(CmiRdmaAck));
-  ack->fnPtr = fn;
-  ack->token = token;
-  return ack;
-}
-
-
-/* Receiver side functions */
-
-/* Set the receiver specific machine layer info generic to RDMA ops*/
-void CmiSetRdmaRecvInfo(void *dest, int numOps, void *charmMsg, void *rdmaInfo, int msgSize){
-  LrtsSetRdmaRecvInfo(dest, numOps, charmMsg, rdmaInfo, msgSize);
-}
-
-/* Set the receiver specific machine layer info specific to RDMA ops*/
-void CmiSetRdmaRecvOpInfo(void *dest, void *buffer, void *src_ref, int size, int opIndex, void *rdmaInfo){
-  LrtsSetRdmaRecvOpInfo(dest, buffer, src_ref, size, opIndex, rdmaInfo);
-}
-
-/* Get the size occupied by the receiver specific machine layer specific to RDMA op*/
-int CmiGetRdmaOpRecvInfoSize(void){
-  return LrtsGetRdmaOpRecvInfoSize();
-}
-
-/* Get the size occupied by the receiver specific machine layer info generic to RDMA ops*/
-int CmiGetRdmaGenRecvInfoSize(void){
-  return LrtsGetRdmaGenRecvInfoSize();
-}
-
-/* Get the total size occupied by the receiver specific machine layer info*/
-int CmiGetRdmaRecvInfoSize(int numOps){
-  return LrtsGetRdmaRecvInfoSize(numOps);
-}
-
-/* Issue RDMA get calls on the pe using the message containing the metadata information*/
-void CmiIssueRgets(void *recv, int pe){
-  LrtsIssueRgets(recv, pe);
-}
-
-#endif
-
 #if CMK_ONESIDED_DIRECT_IMPL
 
-// Function Pointer to the individual Acknowledement handler function for the Direct API
-RdmaAckHandlerFn ncpyAckHandlerFn;
+// Function Pointer to the acknowledement handler function for the Direct API
+RdmaAckHandlerFn ncpyDirectAckHandlerFn;
 
 typedef struct _cmi_rdma_direct_ack {
   const void *srcAddr;
@@ -211,12 +111,12 @@ void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short
 }
 
 void CmiInvokeNcpyAck(void *ack) {
-  ncpyAckHandlerFn(ack);
+  ncpyDirectAckHandlerFn(ack);
 }
 
 /* Set the ack handler function used in the Direct API */
-void CmiSetRdmaNcpyAck(RdmaAckHandlerFn fn){
-  ncpyAckHandlerFn = fn;
+void CmiSetDirectNcpyAckHandler(RdmaAckHandlerFn fn){
+  ncpyDirectAckHandlerFn = fn;
 }
 
 /* Perform an RDMA Get operation into the local destination address from the remote source address*/
index 2164cc6ebdefd7e4d2a49d3fcba867daea91472c..29689c9d1ab56e41df483b7373fb2170955b5d38 100644 (file)
@@ -1855,17 +1855,6 @@ static inline void processRecvWC(struct ibv_wc *recvWC,const int toBuffer){
                        processRdmaRequest(rdmaPacket,nodeNo,0);
                }*/
        }
-       if(rdma && header->code == INFIRDMA_ACK){
-               struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)(buffer->buf+sizeof(struct infiPacketHeader)) ;
-#if CMK_ONESIDED_IMPL
-               if (rdmaPacket->type == INFI_ONESIDED)
-                       verbsOnesidedReceivedAck(rdmaPacket);
-               else
-#endif
-               {
-                       processRdmaAck(rdmaPacket);
-               }
-       }
        if(header->code == INFIRDMA_DIRECT_REG_AND_PUT){
                // Register the source buffer and perform PUT
                NcpyOperationInfo *ncpyOpInfo = (NcpyOperationInfo *)(buffer->buf+sizeof(struct infiPacketHeader));
@@ -1881,6 +1870,9 @@ static inline void processRecvWC(struct ibv_wc *recvWC,const int toBuffer){
                // Set the source as registered
                newNcpyOpInfo->isSrcRegistered = 1;
                
+               // Free the NcpyOperationInfo in a reverse API operation
+               newNcpyOpInfo->freeMe = CMK_FREE_NCPYOPINFO;
+               
                struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
                rdmaPacket->type = INFI_ONESIDED_DIRECT;
                rdmaPacket->localBuffer = newNcpyOpInfo;
@@ -2068,20 +2060,6 @@ static inline  void processRdmaWC(struct ibv_wc *rdmaWC,const int toBuffer){
                return;
        }*/
 //     CmiAssert(rdmaPacket->type == INFI_MESG);
-#if CMK_ONESIDED_IMPL
-       // Zerocopy Entry Method API
-       if (rdmaPacket->type == INFI_ONESIDED) {
-
-#if CMK_IBVERBS_TOKENS_FLOW
-               context->tokensLeft++;
-#endif
-
-               verbsOnesidedOpDone((CmiVerbsRdmaRecvOp_t *)rdmaPacket->localBuffer);
-               free(rdmaPacket);
-
-               return;
-       }
-#endif
 #if CMK_ONESIDED_DIRECT_IMPL
        // Zerocopy Direct API
        if (rdmaPacket->type == INFI_ONESIDED_DIRECT) {
index 3c312fc166c69031575cd077af80050ef9969bc2..259b43a2b4ccf538b2ca120b762be73aace2c15c 100644 (file)
@@ -3,29 +3,6 @@
  * Jaemin Choi
  */
 
-void verbsOnesidedAllOpsDone(char *msg) {
-  int sndlen = ((CmiMsgHeaderBasic *) msg)->size;
-
-  handleOneRecvedMsg(sndlen, msg);
-}
-
-/* function called on completion of the rdma operation */
-void verbsOnesidedOpDone(CmiVerbsRdmaRecvOp_t *recvOpInfo) {
-  CmiVerbsRdmaRecv_t *recvInfo = (CmiVerbsRdmaRecv_t *)(
-      ((char *)recvOpInfo) - recvOpInfo->opIndex * sizeof(CmiVerbsRdmaRecvOp_t)
-    - sizeof(CmiVerbsRdmaRecv_t));
-
-  if (ibv_dereg_mr(recvOpInfo->local_mr)) {
-    MACHSTATE(3, "ibv_dereg_mr() failed\n");
-  }
-
-  verbsOnesidedSendAck(recvInfo->peNum, recvOpInfo);
-  recvInfo->comOps++;
-  if (recvInfo->comOps == recvInfo->numOps) {
-    verbsOnesidedAllOpsDone((char *)recvInfo->msg);
-  }
-}
-
 void postRdma(
   uint64_t local_addr,
   uint32_t local_rkey,
@@ -74,74 +51,6 @@ void postRdma(
   }
 }
 
-/* function to perform RDMA read operation */
-void verbsOnesidedPostRdmaRead(int peNum, CmiVerbsRdmaRecvOp_t *recvOpInfo) {
-  struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)malloc(sizeof(struct infiRdmaPacket));
-  rdmaPacket->type = INFI_ONESIDED;
-  rdmaPacket->localBuffer = recvOpInfo;
-
-#if CMK_IBVERBS_TOKENS_FLOW
-  context->tokensLeft--;
-#if CMK_IBVERBS_STATS
-  if(context->tokensLeft < minTokensLeft)
-    minTokensLeft = context->tokensLeft;
-#endif
-#endif
-
-  struct ibv_mr *mr;
-  mr = ibv_reg_mr(context->pd, (void *)recvOpInfo->local_addr, recvOpInfo->size, IBV_ACCESS_LOCAL_WRITE |
-      IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
-  if (!mr) {
-    MACHSTATE(3, "ibv_reg_mr() failed\n");
-  }
-  recvOpInfo->local_mr = mr;
-
-  postRdma(recvOpInfo->local_addr,
-          mr->lkey,
-          recvOpInfo->remote_addr,
-          recvOpInfo->key,
-          recvOpInfo->size,
-          peNum,
-          (uint64_t)rdmaPacket,
-          IBV_WR_RDMA_READ);
-}
-
-void LrtsIssueRgets(void *recv, int pe) {
-  CmiVerbsRdmaRecv_t* recvInfo = (CmiVerbsRdmaRecv_t*)recv;
-  int peNum = recvInfo->peNum;
-  int i;
-
-  for (i = 0; i < recvInfo->numOps; i++)
-    verbsOnesidedPostRdmaRead(peNum, &recvInfo->rdmaOp[i]);
-}
-
-/* function to send the acknowledgement to the sender */
-void verbsOnesidedSendAck(int peNum, CmiVerbsRdmaRecvOp_t *recvOpInfo) {
-  struct infiRdmaPacket *rdmaPacket = (struct infiRdmaPacket *)CmiAlloc(sizeof(struct infiRdmaPacket));
-  rdmaPacket->fromNodeNo = CmiNodeOf(peNum);
-  rdmaPacket->type = INFI_ONESIDED;
-  rdmaPacket->keyPtr = recvOpInfo->remote_mr;
-  rdmaPacket->localBuffer = recvOpInfo->src_info;
-
-  EnqueueRdmaAck(rdmaPacket);
-  CmiFree(rdmaPacket);
-}
-
-/* function called on the sender on receiving acknowledgement
- * from the receiver to signal the completion of the rdma operation */
-void verbsOnesidedReceivedAck(struct infiRdmaPacket *rdmaPacket) {
-  struct ibv_mr *mr = rdmaPacket->keyPtr;
-  if (ibv_dereg_mr(mr)) {
-    MACHSTATE(3, "ibv_dereg_mr() failed\n");
-  }
-
-  CmiRdmaAck *ack = (CmiRdmaAck *)rdmaPacket->localBuffer;
-  ack->fnPtr(ack->token);
-
-  //free callback structure, CmiRdmaAck allocated in CmiSetRdmaAck
-  free(ack);
-}
-
 /* Support for Nocopy Direct API */
 
 // Set the machine specific information for a nocopy pointer
@@ -191,7 +100,19 @@ void LrtsIssueRget(NcpyOperationInfo *ncpyOpInfo) {
     packet->header.code = INFIRDMA_DIRECT_REG_AND_PUT;
     packet->ogm  = NULL;
 
-    struct ibv_mr *packetKey = METADATAFIELD(ncpyOpInfo)->key;
+    struct ibv_mr *packetKey;
+    if(ncpyOpInfo->opMode == CMK_DIRECT_API) {
+      packetKey = METADATAFIELD(ncpyOpInfo)->key;
+    } else if(ncpyOpInfo->opMode == CMK_EM_API) {
+      // Register the small message in order to send it to the other side
+      packetKey = ibv_reg_mr(context->pd, (void *)ncpyOpInfo, ncpyOpInfo->ncpyOpInfoSize, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
+      if (!packetKey) {
+        CmiAbort("Memory Registration Failed in LrtsIssueRget!\n");
+      }
+      // set opMode to for Reverse operation
+      setReverseModeForNcpyOpInfo(ncpyOpInfo);
+    }
+
     OtherNode node = &nodes[CmiNodeOf(ncpyOpInfo->srcPe)];
     EnqueuePacket(node, packet, ncpyOpInfo->ncpyOpInfoSize, packetKey);
 
index b1770aa4087e13dfd581a56bab8603924072eb72..7b0a799ced922a2db37fc3085a0c043e47401be6 100644 (file)
@@ -1,115 +1,6 @@
 #ifndef VERBS_MACHINE_ONESIDED_H
 #define VERBS_MACHINE_ONESIDED_H
 
-typedef struct _cmi_verbs_rdma_op {
-  uint64_t remote_addr;
-  int size;
-  struct ibv_mr *mr;
-  uint32_t key;
-} CmiVerbsRdmaOp_t;
-
-typedef struct _cmi_verbs_rdma{
-  int numOps;
-  int peNum;
-  CmiVerbsRdmaOp_t rdmaOp[0];
-} CmiVerbsRdma_t;
-
-typedef struct _cmi_verbs_rdma_recv_op {
-  struct ibv_mr *remote_mr;
-  struct ibv_mr *local_mr;
-  void *src_info;
-  uint64_t remote_addr;
-  uint64_t local_addr;
-  int size;
-  uint32_t key;
-  int opIndex;
-} CmiVerbsRdmaRecvOp_t;
-
-typedef struct _cmi_verbs_rdma_recv {
-  int numOps;
-  int comOps;
-  int peNum;
-  void *msg;
-  CmiVerbsRdmaRecvOp_t rdmaOp[0];
-} CmiVerbsRdmaRecv_t;
-
-void verbsOnesidedOpDone(CmiVerbsRdmaRecvOp_t *recvOpInfo);
-
-void verbsOnesidedSendAck(int peNum, CmiVerbsRdmaRecvOp_t *recvOpInfo);
-
-void verbsOnesidedReceivedAck(struct infiRdmaPacket *rdmaPacket);
-
-int LrtsGetRdmaOpInfoSize(){
-  return sizeof(CmiVerbsRdmaOp_t);
-}
-
-int LrtsGetRdmaGenInfoSize(){
-  return sizeof(CmiVerbsRdma_t);
-}
-
-int LrtsGetRdmaInfoSize(int numOps){
-  return sizeof(CmiVerbsRdma_t) + numOps * sizeof(CmiVerbsRdmaOp_t);
-}
-
-int LrtsGetRdmaOpRecvInfoSize(){
-  return sizeof(CmiVerbsRdmaRecvOp_t);
-}
-
-int LrtsGetRdmaGenRecvInfoSize(){
-  return sizeof(CmiVerbsRdmaRecv_t);
-}
-
-int LrtsGetRdmaRecvInfoSize(int numOps){
-  return sizeof(CmiVerbsRdmaRecv_t) + numOps * sizeof(CmiVerbsRdmaRecvOp_t);
-}
-
-void LrtsSetRdmaRecvInfo(void *rdmaRecv, int numOps, void *msg, void *rdmaSend, int msgSize){
-  CmiVerbsRdmaRecv_t *rdmaRecvInfo = (CmiVerbsRdmaRecv_t *)rdmaRecv;
-  CmiVerbsRdma_t *rdmaSendInfo = (CmiVerbsRdma_t *)rdmaSend;
-
-  rdmaRecvInfo->numOps = numOps;
-  rdmaRecvInfo->comOps = 0;
-  rdmaRecvInfo->peNum = rdmaSendInfo->peNum;
-  rdmaRecvInfo->msg = msg;
-}
-
-void LrtsSetRdmaRecvOpInfo(void *rdmaRecvOp, void *buffer, void *src_ref, int size, int opIndex, void *rdmaSend){
-  CmiVerbsRdmaRecvOp_t *rdmaRecvOpInfo = (CmiVerbsRdmaRecvOp_t *)rdmaRecvOp;
-  CmiVerbsRdma_t *rdmaSendInfo = (CmiVerbsRdma_t *)rdmaSend;
-
-  rdmaRecvOpInfo->remote_mr = rdmaSendInfo->rdmaOp[opIndex].mr;
-  rdmaRecvOpInfo->src_info = src_ref;
-  rdmaRecvOpInfo->remote_addr = rdmaSendInfo->rdmaOp[opIndex].remote_addr;
-  rdmaRecvOpInfo->local_addr = (uint64_t)buffer;
-  rdmaRecvOpInfo->size = size;
-  rdmaRecvOpInfo->key = rdmaSendInfo->rdmaOp[opIndex].key;
-  rdmaRecvOpInfo->opIndex = opIndex;
-}
-
-void LrtsSetRdmaInfo(void *dest, int destPE, int numOps){
-  CmiVerbsRdma_t *rdma = (CmiVerbsRdma_t*)dest;
-  rdma->numOps = numOps;
-  rdma->peNum = CmiMyPe();
-}
-
-void LrtsSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE){
-  struct ibv_mr *mr;
-
-  CmiVerbsRdmaOp_t *rdmaOp = (CmiVerbsRdmaOp_t *)dest;
-  rdmaOp->remote_addr = (uint64_t)ptr;
-  rdmaOp->size = size;
-
-  mr = ibv_reg_mr(context->pd, (void *)ptr, size, IBV_ACCESS_REMOTE_READ |
-      IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-  if (!mr) {
-    MACHSTATE(3, "ibv_reg_mr() failed\n");
-  }
-  rdmaOp->mr = mr;
-  rdmaOp->key = mr->rkey;
-}
-
-#endif /* VERBS_MACHINE_ONESIDED_H */
-
 typedef struct _cmi_verbs_rzv_rdma_pointer {
   struct ibv_mr *mr;
   uint32_t key;
@@ -192,3 +83,4 @@ void LrtsIssueRput(
 
 void registerDirectMemory(void *info, const void *addr, int size);
 
+#endif /* VERBS_MACHINE_ONESIDED_H */
index b00d4c82cfacf847e85d68833d92df4c1621b34c..fc8579ff268c77f527fad4296c6e5e637bff2094 100644 (file)
@@ -192,8 +192,6 @@ public:
 
 #include "ckcallback.h"
 
-#include "ckrdmawrapper.h"
-
 #include "ckrdma.h"
 
 /********************* Superclass of all Chares ******************/
index e0277b868aac749f54ddcd54a83d1d2b6d03d5a9..c56dbd72ac2474e87a2dfcb8783889b27e826da6 100644 (file)
@@ -1213,18 +1213,15 @@ void _processHandler(void *converseMsg,CkCoreState *ck)
 
 #if CMK_ONESIDED_IMPL
   if(env->isRdma()){
-    //Receiver copies the buffer from the sender when they share
-    //the address space
-    if(CmiNodeOf(env->getSrcPe())==CmiMyNode()){
-      envelope *prevEnv = env;
-      env = CkRdmaCopyMsg(prevEnv);
+    envelope *prevEnv = env;
+    env = CkRdmaIssueRgets(prevEnv);
+    if(env) {
+      // Within pe or logical node, env points to new message with data
+
+      // Free prevEnv
       CkFreeMsg(EnvToUsr(prevEnv));
-    }
-    //Receiver issues Rdma Get calls to fetch data over network
-    else{
-      CkRdmaIssueRgets(env);
-      //return because the new message with the rdma buffer is again passed
-      //to the process handler after the Rdma Get call completion
+    } else{
+      // async rdma call in place, asynchronous return and ack handling
       return;
     }
   }
@@ -1421,13 +1418,6 @@ void _skipCldEnqueue(int pe,envelope *env, int infoFn)
   }
 #endif
 
-#if CMK_ONESIDED_IMPL
-  if(env->isRdma()){
-    CkRdmaPrepareMsg(&env, pe);
-    //CkPrintf("[%d] _skipCldEnqueue metadata-msg: %d \n", CkMyPe(), env->getTotalsize());
-  }
-#endif
-
 #if CMK_FAULT_EVAC
   if(pe == CkMyPe() ){
     if(!CmiNodeAlive(CkMyPe())){
@@ -1529,13 +1519,6 @@ static void _noCldEnqueue(int pe, envelope *env)
   }
 #endif
 
-#if CMK_ONESIDED_IMPL
-  if(env->isRdma()){
-    CkRdmaPrepareMsg(&env, pe);
-    //CkPrintf("[%d] _noCldEnqueue metadata-msg: %d \n", CkMyPe(), env->getTotalsize());
-  }
-#endif
-
   CkPackMessage(&env);
   int len=env->getTotalsize();
   if (pe==CLD_BROADCAST) { CmiSyncBroadcastAndFree(len, (char *)env); }
@@ -1559,12 +1542,6 @@ void _noCldNodeEnqueue(int node, envelope *env)
   }
 #endif
 
-#if CMK_ONESIDED_IMPL
-  if(env->isRdma()){
-    CkRdmaPrepareMsg(&env, CmiNodeFirst(node));
-    //CkPrintf("[%d] noCldEnqueue metadata-msg: %d \n", CkMyPe(), env->getTotalsize());
-  }
-#endif
   CkPackMessage(&env);
   int len=env->getTotalsize();
   if (node==CLD_BROADCAST) { 
index aa22e4aafd366ef98b9999a175f6054a74c9a1d5..b2b85239c036fafa71daf86e875c352859ee9d24 100644 (file)
@@ -28,11 +28,6 @@ inline void _CldEnqueue(int pe, void *msg, int infofn) {
     CmiFree(msg);
     return;
   }
-#if CMK_ONESIDED_IMPL
-  if(((envelope *)msg)->isRdma()){
-    CkRdmaPrepareMsg((envelope **)&msg, pe);
-  }
-#endif
   CldEnqueue(pe, msg, infofn);
 }
 inline void _CldEnqueueMulti(int npes, int *pes, void *msg, int infofn) {
@@ -54,30 +49,15 @@ inline void _CldNodeEnqueue(int node, void *msg, int infofn) {
     CmiFree(msg);
     return;
   }
-#if CMK_ONESIDED_IMPL
-  if(((envelope *)msg)->isRdma()){
-    CkRdmaPrepareMsg((envelope **)&msg, CmiNodeFirst(node));
-  }
-#endif
   CldNodeEnqueue(node, msg, infofn);
 }
 #else
 
 inline void _CldEnqueue(int pe, void *msg, int infofn) {
-#if CMK_ONESIDED_IMPL
-  if(((envelope *)msg)->isRdma()){
-    CkRdmaPrepareMsg((envelope **)&msg, pe);
-  }
-#endif
   CldEnqueue(pe, msg, infofn);
 }
 
 inline void _CldNodeEnqueue(int node, void *msg, int infofn) {
-#if CMK_ONESIDED_IMPL
-  if(((envelope *)msg)->isRdma()){
-    CkRdmaPrepareMsg((envelope **)&msg, CmiNodeFirst(node));
-  }
-#endif
   CldNodeEnqueue(node, msg, infofn);
 }
 #define _CldEnqueueMulti  CldEnqueueMulti
index c5d3f9b6c8ab15947cd8aa662116c3a194237b89..2a5fc9469cfc27bd11e58778c3c712e9987797eb 100644 (file)
 /*readonly*/ extern CProxy_ckcallback_group _ckcallbackgroup;
 #endif
 
-#if CMK_ONESIDED_IMPL
-/* Sender Functions */
-
-/*
- * Add the machine specific information if metadata message
- * sent across network. Do nothing if message is sent to same PE/Node
- */
-void CkRdmaPrepareMsg(envelope **env, int pe){
-  if(CmiNodeOf(pe)!=CmiMyNode()){
-    envelope *prevEnv = *env;
-    *env = CkRdmaCreateMetadataMsg(prevEnv, pe);
-    CkFreeMsg(EnvToUsr(prevEnv));
-  }
-#if CMK_SMP && CMK_IMMEDIATE_MSG
-  //Reset the immediate bit if it is a within node/pe message
-  else
-    CmiResetImmediate(*env);
-#endif
-}
-
-/*
- * Add machine specific information to the msg. This includes information
- * information both common and specific to each rdma parameter
- * Metdata message format: <-env-><-msg-><-migen-><-mispec1-><-mispec2->..<-mispecn->
- * migen: machine info generic to rdma operation
- * mispec: machine info specific to rdma operation for n rdma operations
- */
-envelope* CkRdmaCreateMetadataMsg(envelope *env, int pe){
-
-  int numops = getRdmaNumOps(env);
-  int msgsize = env->getTotalsize();
-
-  //CmiGetRdmaInfoSize returns the size of machine specific information
-  // for numops RDMA operations
-  int mdsize = msgsize + CmiGetRdmaInfoSize(numops);
-
-  //pack before copying
-  CkPackMessage(&env);
-
-  //Allocate a new metadata message, set machine specific info
-  envelope *copyenv = (envelope *)CmiAlloc(mdsize);
-  memcpy(copyenv, env, msgsize);
-  copyenv->setTotalsize(mdsize);
-#if CMK_SMP && CMK_IMMEDIATE_MSG
-  CmiBecomeImmediate(copyenv);
-#endif
-
-  //Set the generic information
-  char *md = (char *)copyenv + msgsize;
-  CmiSetRdmaInfo(md, pe, numops);
-  md += CmiGetRdmaGenInfoSize();
-
-  //Set the rdma op specific information
-  CkUnpackMessage(&copyenv);
-  PUP::fromMem up((void *)(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf));
-  PUP::toMem p((void *)(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf));
-  up|numops;
-  p|numops;
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w; up|w;
-    CkRdmaWrapper *wack = new CkRdmaWrapper(w);
-    void *ack = CmiSetRdmaAck(CkHandleRdmaCookie, wack);
-    w.callback = (CkCallback *)ack;
-    p|w;
-    CmiSetRdmaOpInfo(md, w.ptr, w.cnt, w.callback, pe);
-    md += CmiGetRdmaOpInfoSize();
-  }
-  CkPackMessage(&copyenv);
-
-  //return the env with the machine specific information
-  return copyenv;
-}
-
-/*
- * Method called on sender when ack is received
- * Access the CkRdmaWrapper using the cookie passed and invoke callback
- */
-void CkHandleRdmaCookie(void *cookie){
-  CkRdmaWrapper *w = (CkRdmaWrapper *)cookie;
-  CkCallback *cb= w->callback;
-#if CMK_SMP
-  //call the callbackgroup on my node's first PE when callback requires to be called from comm thread
-  //this adds one more trip through the scheduler
-  _ckcallbackgroup[CmiNodeFirst(CmiMyNode())].call(*cb, sizeof(void *), (char*)&w->ptr);
-#else
-  //direct call to callback when calling from worker thread
-  cb->send(sizeof(void *), &w->ptr);
-#endif
-
-  delete cb;
-  delete (CkRdmaWrapper *)cookie;
-}
-
-
-/* Receiver Functions */
-
-/*
- * Method called when received message is on the same PE/Node
- * This involves a direct copy from the sender's buffer into the receiver's buffer
- * A new message is allocated with size = existing message size + sum of all rdma buffers.
- * The newly allocated message contains the existing marshalled message along with
- * space for the rdma buffers which are copied from the source buffers.
- * The buffer and the message are allocated contiguously to free the buffer
- * when the message gets free.
- */
-envelope* CkRdmaCopyMsg(envelope *env){
-  int numops, bufsize, msgsize;
-  bufsize = getRdmaBufSize(env);
-  msgsize = env->getTotalsize();
-
-  CkPackMessage(&env);
-  envelope *copyenv = (envelope *)CmiAlloc(CK_ALIGN(msgsize, 16) + bufsize);
-  memcpy(copyenv, env, msgsize);
-  copyenv->setTotalsize(CK_ALIGN(msgsize, 16) + bufsize);
-  copyenv->setRdma(false);
-
-  char* buf = (char *)copyenv + CK_ALIGN(msgsize, 16);
-  CkUnpackMessage(&copyenv);
-  PUP::toMem p((void *)(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf));
-  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf);
-  up|numops;
-  p|numops;
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w;
-    up|w;
-    //Copy the buffer from the source pointer to the message
-    memcpy(buf, w.ptr, w.cnt);
-
-    //Invoke callback as it is safe to rewrite into the source buffer
-    (w.callback)->send(sizeof(void *), &w.ptr);
-    delete w.callback;
-
-    //Update the CkRdmaWrapper pointer of the new message
-    w.ptr = buf;
-
-    //Update the pointer
-    buf += CK_ALIGN(w.cnt, 16);
-    p|w;
-  }
-  CkPackRdmaPtrs(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf);
-  CkPackMessage(&copyenv);
-  return copyenv;
-}
-
-/*
- * Extract rdma based information from the metadata message,
- * allocate buffers and issue RDMA get call
- */
-void CkRdmaIssueRgets(envelope *env){
-  /*
-   * Determine the buffer size('bufsize') and the message size('msgsize')
-   * from the metadata message. 'msgSize' is the metadata message size
-   * without the sender's machine specific information.
-   */
-  int numops, bufsize, msgsize;
-  bufsize = getRdmaBufSize(env);
-  numops = getRdmaNumOps(env);
-  msgsize = env->getTotalsize() - CmiGetRdmaInfoSize(numops);
-
-  /* Allocate the receiver's message, which contains the metadata message sent by the sender
-   * (without its machine specific info) of size 'msgSize', the entire receiver's buffer of
-   * size 'bufsize', and the receiver's machine specific info of size 'CmiGetRdmaRecvInfoSize(numops)'.
-   * The receiver's machine specific info is added to this message to avoid separately freeing it
-   * in the machine layer.
-   */
-  envelope *copyenv = (envelope *)CmiAlloc(CK_ALIGN(msgsize, 16) + bufsize + CmiGetRdmaRecvInfoSize(numops));
-
-  //Copy the metadata message(without the machine specific info) into the buffer
-  memcpy(copyenv, env, msgsize);
-
-#if CMK_SMP && CMK_IMMEDIATE_MSG
-  CmiResetImmediate(copyenv);
-#endif
-
-  //Receiver's machine specific info is at an offset, after the sender md and the receiver's buffer
-  char *recv_md = ((char *)copyenv) + CK_ALIGN(msgsize, 16) + bufsize;
-
-  /* Set the total size of the message excluding the receiver's machine specific info
-   * which is not required when the receiver's entry method executes
-   */
-  copyenv->setTotalsize(CK_ALIGN(msgsize, 16) + bufsize);
-
-  CkUnpackMessage(&copyenv);
-  CkUpdateRdmaPtrs(copyenv, msgsize, recv_md, ((char *)env) + msgsize);
-  CkPackRdmaPtrs(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf);
-  CkPackMessage(&copyenv);
-
-  // Set rdma to be false to prevent message handler on the receiver
-  // from intercepting it
-  copyenv->setRdma(false);
-
-  // Free the existing message
-  CkFreeMsg(EnvToUsr(env));
-
-  //Call the lower layer API for performing RDMA gets
-  CmiIssueRgets(recv_md, copyenv->getSrcPe());
-}
-
-
-/*
- * Method called to update machine specific information for receiver
- * using the metadata message given by the sender along with updating
- * pointers of the CkRdmawrappers
- * - assumes that the msg is unpacked
- */
-void CkUpdateRdmaPtrs(envelope *env, int msgsize, char *recv_md, char *src_md){
-  PUP::toMem p((void *)(((CkMarshallMsg *)EnvToUsr(env))->msgBuf));
-  PUP::fromMem up((void *)(((CkMarshallMsg *)EnvToUsr(env))->msgBuf));
-  int numops;
-  up|numops;
-  p|numops;
-
-  //Use the metadata info to set the machine info for receiver
-  //generic info for all RDMA operations
-  CmiSetRdmaRecvInfo(recv_md, numops, env, src_md, env->getTotalsize());
-  recv_md += CmiGetRdmaGenRecvInfoSize();
-  char *buf = ((char *)env) + CK_ALIGN(msgsize, 16);
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w;
-    up|w;
-    //Set RDMA operation specific info
-    CmiSetRdmaRecvOpInfo(recv_md, buf, w.callback, w.cnt, i, src_md);
-    recv_md += CmiGetRdmaOpRecvInfoSize();
-    w.ptr = buf;
-    buf += CK_ALIGN(w.cnt, 16);
-    p|w;
-  }
-}
-
-
-/*
- * Method called to pack rdma pointers inside CkRdmaWrappers in the message
- * Assumes that msg is unpacked
- */
-void CkPackRdmaPtrs(char *msgBuf){
-  PUP::toMem p((void *)msgBuf);
-  PUP::fromMem up((void *)msgBuf);
-  int numops;
-  up|numops;
-  p|numops;
-
-  //Pack Rdma pointers in CkRdmaWrappers
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w;
-    up|w;
-    w.ptr = (void *)((char *)w.ptr - (char *)msgBuf);
-    p|w;
-  }
-}
-
-
-/*
- * Method called to unpack rdma pointers inside CkRdmaWrappers in the message
- * Assumes that msg is unpacked
- */
-void CkUnpackRdmaPtrs(char *msgBuf){
-  PUP::toMem p((void *)msgBuf);
-  PUP::fromMem up((void *)msgBuf);
-  int numops;
-  up|numops;
-  p|numops;
-
-  //Unpack Rdma pointers in CkRdmaWrappers
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w;
-    up|w;
-    w.ptr = (void *)((char *)msgBuf + (size_t)w.ptr);
-    p|w;
-  }
-}
-
-
-//Determine the number of rdma ops from the message
-int getRdmaNumOps(envelope *env){
-  int numops;
-  CkUnpackMessage(&env);
-  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(env))->msgBuf);
-  up|numops;
-  CkPackMessage(&env);
-  return numops;
-}
-
-/*
- * Determine the total size of the buffers to be copied
- * This is to be allocated at the end of the message
- */
-int getRdmaBufSize(envelope *env){
-  /*
-   * Determine the number of rdma operations and the sum of all
-   * rdma buffer sizes by iterating over the ckrdmawrappers in the message
-   */
-  int numops, bufsize=0;
-  CkUnpackMessage(&env);
-  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(env))->msgBuf);
-  up|numops;
-  for(int i=0; i<numops; i++){
-    CkRdmaWrapper w; up|w;
-    bufsize += CK_ALIGN(w.cnt, 16);
-  }
-  CkPackMessage(&env);
-  return bufsize;
-}
-
-#endif
-/* End of CMK_ONESIDED_IMPL */
-
-/* Support for Direct Nocopy API */
-
-// Ack handler function which invokes the callback
-void CkRdmaDirectAckHandler(void *ack) {
-
-  // Process QD to mark completion of the outstanding RDMA operation
-  QdProcess(1);
-
-  NcpyOperationInfo *info = (NcpyOperationInfo *)(ack);
-
-  CkCallback *srcCb = (CkCallback *)(info->srcAck);
-  CkCallback *destCb = (CkCallback *)(info->destAck);
-
-  CkNcpyBuffer src, dest;
-
-  if(srcCb->requiresMsgConstruction()) {
-    // reconstruct the CkNcpyBuffer object for the source
-    src.ptr = info->srcPtr;
-    src.pe  = info->srcPe;
-    src.cnt = info->srcSize;
-    src.ref = info->srcRef;
-    src.mode = info->srcMode;
-    src.isRegistered = info->isSrcRegistered;
-    memcpy((char *)(&src.cb), srcCb, info->srcAckSize); // initialize cb
-    memcpy((char *)(src.layerInfo), info->srcLayerInfo, info->srcLayerSize); // initialize layerInfo
-  }
-
-  if(destCb->requiresMsgConstruction()) {
-    // reconstruct the CkNcpyBuffer object for the destination
-    dest.ptr = info->destPtr;
-    dest.pe  = info->destPe;
-    dest.cnt = info->destSize;
-    dest.ref = info->destRef;
-    dest.mode = info->destMode;
-    dest.isRegistered = info->isDestRegistered;
-    memcpy((char *)(&dest.cb), destCb, info->destAckSize); // initialize cb
-    memcpy((char *)(dest.layerInfo), info->destLayerInfo, info->destLayerSize); // initialize layerInfo
-  }
-
-  if(info->ackMode == 0 || info->ackMode == 1) {
-
-#if CMK_SMP
-    //call the callbackgroup on my node's first PE when callback requires to be called from comm thread
-    //this adds one more trip through the scheduler
-    _ckcallbackgroup[CmiNodeFirst(CmiMyNode())].call(*srcCb, sizeof(CkNcpyBuffer), (const char *)(&src));
-#else
-    //Invoke the destination callback
-    srcCb->send(sizeof(CkNcpyBuffer), &src);
-#endif
-  }
-
-  if(info->ackMode == 0 || info->ackMode == 2) {
-
-#if CMK_SMP
-    //call the callbackgroup on my node's first PE when callback requires to be called from comm thread
-    //this adds one more trip through the scheduler
-    _ckcallbackgroup[CmiNodeFirst(CmiMyNode())].call(*destCb, sizeof(CkNcpyBuffer), (const char *)(&dest));
-#else
-    //Invoke the destination callback
-    destCb->send(sizeof(CkNcpyBuffer), &dest);
-#endif
-  }
-
-  if(info->freeMe)
-    CmiFree(info);
-}
-
-// Returns CkNcpyMode::MEMCPY if both the PEs are the same and memcpy can be used
-// Returns CkNcpyMode::CMA if both the PEs are in the same physical node and CMA can be used
-// Returns CkNcpyMode::RDMA if RDMA needs to be used
-CkNcpyMode findTransferMode(int srcPe, int destPe) {
-  if(CmiNodeOf(srcPe)==CmiNodeOf(destPe))
-    return CkNcpyMode::MEMCPY;
-#if CMK_USE_CMA
-  else if(CmiDoesCMAWork() && CmiPeOnSamePhysicalNode(srcPe, destPe))
-    return CkNcpyMode::CMA;
-#endif
-  else
-    return CkNcpyMode::RDMA;
-}
+/*********************************** Zerocopy Direct API **********************************/
 
 // Get Methods
 void CkNcpyBuffer::memcpyGet(CkNcpyBuffer &source) {
@@ -408,12 +23,12 @@ void CkNcpyBuffer::memcpyGet(CkNcpyBuffer &source) {
 #if CMK_USE_CMA
 void CkNcpyBuffer::cmaGet(CkNcpyBuffer &source) {
   CmiIssueRgetUsingCMA(source.ptr,
-                       source.layerInfo,
-                       source.pe,
-                       ptr,
-                       layerInfo,
-                       pe,
-                       cnt);
+         source.layerInfo,
+         source.pe,
+         ptr,
+         layerInfo,
+         pe,
+         cnt);
 }
 #endif
 
@@ -658,3 +273,411 @@ CkNcpyStatus CkNcpyBuffer::put(CkNcpyBuffer &destination){
     CkAbort("Invalid CkNcpyMode");
   }
 }
+
+// reconstruct the CkNcpyBuffer object for the source
+void constructSourceBufferObject(NcpyOperationInfo *info, CkNcpyBuffer &src) {
+  src.ptr = info->srcPtr;
+  src.pe  = info->srcPe;
+  src.cnt = info->srcSize;
+  src.ref = info->srcRef;
+  src.mode = info->srcMode;
+  src.isRegistered = info->isSrcRegistered;
+  memcpy((char *)(&src.cb), info->srcAck, info->srcAckSize); // initialize cb
+  memcpy((char *)(src.layerInfo), info->srcLayerInfo, info->srcLayerSize); // initialize layerInfo
+}
+
+// reconstruct the CkNcpyBuffer object for the destination
+void constructDestinationBufferObject(NcpyOperationInfo *info, CkNcpyBuffer &dest) {
+  dest.ptr = info->destPtr;
+  dest.pe  = info->destPe;
+  dest.cnt = info->destSize;
+  dest.ref = info->destRef;
+  dest.mode = info->destMode;
+  dest.isRegistered = info->isDestRegistered;
+  memcpy((char *)(&dest.cb), info->destAck, info->destAckSize); // initialize cb
+  memcpy((char *)(dest.layerInfo), info->destLayerInfo, info->destLayerSize); // initialize layerInfo
+}
+
+void invokeSourceCallback(NcpyOperationInfo *info) {
+  CkCallback *srcCb = (CkCallback *)(info->srcAck);
+  if(srcCb->requiresMsgConstruction()) {
+    if(info->ackMode == CMK_SRC_DEST_ACK || info->ackMode == CMK_SRC_ACK) {
+      CkNcpyBuffer src;
+      constructSourceBufferObject(info, src);
+      //Invoke the sender's callback
+      invokeCallback(info->srcAck, info->srcPe, src);
+    }
+  }
+}
+
+void invokeDestinationCallback(NcpyOperationInfo *info) {
+  CkCallback *destCb = (CkCallback *)(info->destAck);
+  if(destCb->requiresMsgConstruction()) {
+    if(info->ackMode == CMK_SRC_DEST_ACK || info->ackMode == CMK_DEST_ACK) {
+      CkNcpyBuffer dest;
+      constructDestinationBufferObject(info, dest);
+      //Invoke the receiver's callback
+      invokeCallback(info->destAck, info->destPe, dest);
+    }
+  }
+}
+
+void handleDirectApiCompletion(NcpyOperationInfo *info) {
+  invokeSourceCallback(info);
+  invokeDestinationCallback(info);
+
+  if(info->freeMe == CMK_FREE_NCPYOPINFO)
+    CmiFree(info);
+}
+
+// Ack handler function which invokes the callback
+void CkRdmaDirectAckHandler(void *ack) {
+
+  // Process QD to mark completion of the outstanding RDMA operation
+  QdProcess(1);
+
+  NcpyOperationInfo *info = (NcpyOperationInfo *)(ack);
+
+  CkCallback *srcCb = (CkCallback *)(info->srcAck);
+  CkCallback *destCb = (CkCallback *)(info->destAck);
+
+  switch(info->opMode) {
+    case CMK_DIRECT_API    : handleDirectApiCompletion(info); // Ncpy Direct API
+                             break;
+#if CMK_ONESIDED_IMPL
+    case CMK_EM_API        : handleEntryMethodApiCompletion(info); // Ncpy EM API invoked through a GET
+                             break;
+
+    case CMK_EM_API_REVERSE: handleReverseEntryMethodApiCompletion(info); // Ncpy EM API invoked through a PUT
+                             break;
+#endif
+    default                : CmiAbort("Unknown opMode");
+                             break;
+  }
+}
+
+// Helper methods
+void invokeCallback(void *cb, int pe, CkNcpyBuffer &buff) {
+
+#if CMK_SMP
+    //call to callbackgroup to call the callback when calling from comm thread
+    //this adds one more trip through the scheduler
+    _ckcallbackgroup[pe].call(*(CkCallback *)(cb), sizeof(CkNcpyBuffer), (const char *)(&buff));
+#else
+    //Invoke the destination callback
+    ((CkCallback *)(cb))->send(sizeof(CkNcpyBuffer), &buff);
+#endif
+}
+
+// Returns CkNcpyMode::MEMCPY if both the PEs are the same and memcpy can be used
+// Returns CkNcpyMode::CMA if both the PEs are in the same physical node and CMA can be used
+// Returns CkNcpyMode::RDMA if RDMA needs to be used
+CkNcpyMode findTransferMode(int srcPe, int destPe) {
+  if(CmiNodeOf(srcPe)==CmiNodeOf(destPe))
+    return CkNcpyMode::MEMCPY;
+#if CMK_USE_CMA
+  else if(CmiDoesCMAWork() && CmiPeOnSamePhysicalNode(srcPe, destPe))
+    return CkNcpyMode::CMA;
+#endif
+  else
+    return CkNcpyMode::RDMA;
+}
+
+
+
+
+
+
+/*********************************** Zerocopy Entry Method API ****************************/
+#if CMK_ONESIDED_IMPL
+/*
+ * Extract ncpy buffer information from the metadata message, allocate buffers
+ * and issue ncpy calls (either memcpy or cma read or rdma get). Main method called on
+ * the destination to perform zerocopy operations as a part of the Zerocopy Entry Method
+ * API
+ */
+envelope* CkRdmaIssueRgets(envelope *env){
+  // Iterate over the ncpy buffer and either perform the operations
+  int numops, bufsize, msgsize;
+  bufsize = getRdmaBufSize(env);
+  numops = getRdmaNumOps(env);
+  msgsize = env->getTotalsize();
+
+  CkNcpyMode ncpyMode = findTransferMode(env->getSrcPe(), CkMyPe());
+
+  int totalMsgSize = CK_ALIGN(msgsize, 16) + bufsize;
+  char *ref;
+  int layerInfoSize, ncpyObjSize, extraSize;
+
+  if(ncpyMode == CkNcpyMode::RDMA) {
+
+    layerInfoSize = CMK_COMMON_NOCOPY_DIRECT_BYTES + CMK_NOCOPY_DIRECT_BYTES;
+
+    ncpyObjSize = getNcpyOpInfoTotalSize(
+                  layerInfoSize,
+                  sizeof(CkCallback),
+                  layerInfoSize,
+                  0);
+
+    extraSize = ncpyObjSize - sizeof(NcpyOperationInfo);
+
+    totalMsgSize += sizeof(NcpyEmInfo) + numops*(sizeof(NcpyEmBufferInfo) + extraSize);
+  }
+
+  // Allocate the new message which stores the receiver buffers
+  envelope *copyenv = (envelope *)CmiAlloc(totalMsgSize);
+
+  //Copy the metadata message(without the machine specific info) into the buffer
+  memcpy(copyenv, env, msgsize);
+
+  /* Set the total size of the message excluding the receiver's machine specific info
+   * which is not required when the receiver's entry method executes
+   */
+  copyenv->setTotalsize(totalMsgSize);
+
+  // Set rdma flag to be false to prevent message handler on the receiver
+  // from intercepting it
+  copyenv->setRdma(false);
+
+  if(ncpyMode == CkNcpyMode::RDMA) {
+    ref = (char *)copyenv + CK_ALIGN(msgsize, 16) + bufsize;
+
+    NcpyEmInfo *ncpyEmInfo = (NcpyEmInfo *)ref;
+    ncpyEmInfo->numOps = numops;
+    ncpyEmInfo->counter = 0;
+    ncpyEmInfo->msg = copyenv;
+  }
+
+  char *buf = (char *)copyenv + CK_ALIGN(msgsize, 16);
+
+  CkUnpackMessage(&copyenv);
+  PUP::toMem p((void *)(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf));
+  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf);
+  up|numops;
+  p|numops;
+
+  for(int i=0; i<numops; i++){
+    // source buffer
+    CkNcpyBuffer source;
+    up|source;
+
+    // destination buffer
+    CkNcpyBuffer dest((const void *)buf, CK_BUFFER_UNREG);
+    dest.cnt = source.cnt;
+
+    // Set the common layerInfo for the destination
+    CmiSetRdmaCommonInfo(dest.layerInfo, dest.ptr, dest.cnt);
+
+    if(ncpyMode == CkNcpyMode::MEMCPY) {
+
+      dest.memcpyGet(source);
+
+      // Invoke source callback
+      source.cb.send(sizeof(CkNcpyBuffer), &source);
+
+#if CMK_USE_CMA
+    } else if(ncpyMode == CkNcpyMode::CMA) {
+
+      dest.cmaGet(source);
+
+      // Invoke source callback
+      source.cb.send(sizeof(CkNcpyBuffer), &source);
+
+#endif
+    } else if(ncpyMode == CkNcpyMode::RDMA) {
+
+      if(dest.mode == CK_BUFFER_UNREG) {
+        // register it because it is required for RGET
+        CmiSetRdmaBufferInfo(dest.layerInfo + CmiGetRdmaCommonInfoSize(), dest.ptr, dest.cnt, dest.mode);
+
+        dest.isRegistered = true;
+      }
+
+      NcpyEmBufferInfo *ncpyEmBufferInfo = (NcpyEmBufferInfo *)(ref + sizeof(NcpyEmInfo) + i *(sizeof(NcpyEmBufferInfo) + extraSize));
+      ncpyEmBufferInfo->index = i;
+
+      NcpyOperationInfo *ncpyOpInfo = &(ncpyEmBufferInfo->ncpyOpInfo);
+
+      setNcpyOpInfo(source.ptr,
+                    (char *)(source.layerInfo),
+                    layerInfoSize,
+                    (char *)(&source.cb),
+                    sizeof(CkCallback),
+                    source.cnt,
+                    source.mode,
+                    source.isRegistered,
+                    source.pe,
+                    source.ref,
+                    dest.ptr,
+                    (char *)(dest.layerInfo),
+                    layerInfoSize,
+                    NULL,
+                    0,
+                    dest.cnt,
+                    dest.mode,
+                    dest.isRegistered,
+                    dest.pe,
+                    (char *)(ncpyEmBufferInfo), // destRef
+                    ncpyOpInfo);
+
+      // set opMode for entry method API
+      ncpyOpInfo->opMode = CMK_EM_API;
+      ncpyOpInfo->freeMe = CMK_DONT_FREE_NCPYOPINFO; // Since ncpyOpInfo is a part of the charm message, don't explicitly free it
+                                                     // It'll be freed when the message is freed by the RTS after the execution of the entry method
+      ncpyOpInfo->refPtr = ncpyEmBufferInfo;
+
+      // Do no launch Rgets here as they could potentially cause a race condition in the SMP mode
+      // The race condition is caused when an RGET completes and invokes the CkRdmaDirectAckHandler
+      // on the comm. thread as the message is being inside this for loop on the worker thread
+
+    } else {
+      CkAbort("Invalid Mode");
+    }
+
+    //Update the CkRdmaWrapper pointer of the new message
+    source.ptr = buf;
+
+    //Update the pointer
+    buf += CK_ALIGN(source.cnt, 16);
+    p|source;
+  }
+
+  CkPackRdmaPtrs(((CkMarshallMsg *)EnvToUsr(copyenv))->msgBuf);
+
+  if(ncpyMode == CkNcpyMode::MEMCPY || ncpyMode == CkNcpyMode::CMA ) {
+    // All operations have completed
+    CkPackMessage(&copyenv);
+    return copyenv; // to indicate the completion of the gets
+    // copyenv represents the new message which consists of the destination buffers
+
+  } else{
+
+    // Launch rgets
+    for(int i=0; i<numops; i++){
+      NcpyEmBufferInfo *ncpyEmBufferInfo = (NcpyEmBufferInfo *)(ref + sizeof(NcpyEmInfo) + i *(sizeof(NcpyEmBufferInfo) + extraSize));
+      NcpyOperationInfo *ncpyOpInfo = &(ncpyEmBufferInfo->ncpyOpInfo);
+      CmiIssueRget(ncpyOpInfo);
+    }
+    return NULL; // to indicate an async operation which may not be complete
+  }
+}
+
+// Method called to unpack rdma pointers
+void CkPackRdmaPtrs(char *msgBuf){
+  PUP::toMem p((void *)msgBuf);
+  PUP::fromMem up((void *)msgBuf);
+  int numops;
+  up|numops;
+  p|numops;
+
+  // Pack ncpy pointers in CkNcpyBuffer
+  for(int i=0; i<numops; i++){
+    CkNcpyBuffer w;
+    up|w;
+    w.ptr = (void *)((char *)w.ptr - (char *)msgBuf);
+    p|w;
+  }
+}
+
+// Method called to unpack rdma pointers
+void CkUnpackRdmaPtrs(char *msgBuf){
+  PUP::toMem p((void *)msgBuf);
+  PUP::fromMem up((void *)msgBuf);
+  int numops;
+  up|numops;
+  p|numops;
+
+  // Unpack ncpy pointers in CkNcpyBuffer
+  for(int i=0; i<numops; i++){
+    CkNcpyBuffer w;
+    up|w;
+    w.ptr = (void *)((char *)msgBuf + (size_t)w.ptr);
+    p|w;
+  }
+}
+
+//Determine the number of ncpy ops from the message
+int getRdmaNumOps(envelope *env){
+  int numops;
+  CkUnpackMessage(&env);
+  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(env))->msgBuf);
+  up|numops;
+  CkPackMessage(&env);
+  return numops;
+}
+
+// Get the sum of ncpy buffer sizes using the metadata message
+int getRdmaBufSize(envelope *env){
+  /*
+   * Determine the number of ncpy operations and the sum of all
+   * ncpy buffer sizes by iterating over the CkNcpyBuffers in the message
+   */
+  int numops, bufsize=0;
+  CkUnpackMessage(&env);
+  PUP::fromMem up((void *)((CkMarshallMsg *)EnvToUsr(env))->msgBuf);
+  up|numops;
+  for(int i=0; i<numops; i++){
+    CkNcpyBuffer w; up|w;
+    bufsize += CK_ALIGN(w.cnt, 16);
+  }
+  CkPackMessage(&env);
+  return bufsize;
+}
+
+void handleEntryMethodApiCompletion(NcpyOperationInfo *info) {
+  invokeSourceCallback(info);
+  if(info->ackMode == CMK_SRC_DEST_ACK || info->ackMode == CMK_DEST_ACK) {
+    // Invoke the ackhandler function to update the counter
+    CkRdmaEMAckHandler(info->destPe, info->refPtr);
+  }
+}
+
+void handleReverseEntryMethodApiCompletion(NcpyOperationInfo *info) {
+  invokeSourceCallback(info);
+  if(info->ackMode == CMK_SRC_DEST_ACK || info->ackMode == CMK_DEST_ACK) {
+    // Send a message to the receiver to invoke the ackhandler function to update the counter
+    CmiInvokeRemoteAckHandler(info->destPe, info->refPtr);
+  }
+  if(info->freeMe == CMK_FREE_NCPYOPINFO)
+    CmiFree(info);
+}
+
+// Ack handler function called when a Zerocopy Entry Method buffer completes
+void CkRdmaEMAckHandler(int destPe, void *ack) {
+
+  NcpyEmBufferInfo *emBuffInfo = (NcpyEmBufferInfo *)(ack);
+
+  char *ref = (char *)(emBuffInfo);
+
+  int layerInfoSize = CMK_COMMON_NOCOPY_DIRECT_BYTES + CMK_NOCOPY_DIRECT_BYTES;
+  int ncpyObjSize = getNcpyOpInfoTotalSize(
+                    layerInfoSize,
+                    sizeof(CkCallback),
+                    layerInfoSize,
+                    0);
+
+
+  NcpyEmInfo *ncpyEmInfo = (NcpyEmInfo *)(ref - (emBuffInfo->index) * (sizeof(NcpyEmBufferInfo) + ncpyObjSize - sizeof(NcpyOperationInfo)) - sizeof(NcpyEmInfo));
+  ncpyEmInfo->counter++; // A zerocopy get completed, update the counter
+
+#if CMK_REG_REQUIRED
+  NcpyOperationInfo *ncpyOpInfo = &(emBuffInfo->ncpyOpInfo);
+  // De-register the destination buffer
+  CmiDeregisterMem(ncpyOpInfo->destPtr, ncpyOpInfo->destLayerInfo + CmiGetRdmaCommonInfoSize(), ncpyOpInfo->destPe, ncpyOpInfo->destMode);
+#endif
+
+  // Check if all rdma operations are complete
+  if(ncpyEmInfo->counter == ncpyEmInfo->numOps) {
+
+    // invoke the charm message handler to enqueue the messsage
+#if CMK_SMP && !CMK_ENABLE_ASYNC_PROGRES
+    // invoked from the comm thread, so send message to the worker thread
+    CmiPushPE(CmiRankOf(destPe), ncpyEmInfo->msg);
+#else
+    // invoked from the worker thread, process message
+    CmiHandleMessage(ncpyEmInfo->msg);
+#endif
+  }
+}
+#endif
+/* End of CMK_ONESIDED_IMPL */
index 5accbf6eaee51b66244646eb053d0e375d3110cf..7c5ba362064af10cf4acd0a57e27605c27542008 100644 (file)
@@ -7,80 +7,13 @@
 
 #include "envelope.h"
 
+/*********************************** Zerocopy Direct API **********************************/
+
 #define CK_BUFFER_REG     CMK_BUFFER_REG
 #define CK_BUFFER_UNREG   CMK_BUFFER_UNREG
 #define CK_BUFFER_PREREG  CMK_BUFFER_PREREG
 #define CK_BUFFER_NOREG   CMK_BUFFER_NOREG
 
-/* CK_MSG_RDMA is passed in as entry method opts in the generated code for an entry
- * method containing RDMA parameters. In the SMP mode with IMMEDIATE message support,
- * it is used to mark the entry method invocation as IMMEDIATE to have the comm thread
- * handle the metadata message. In all other cases (Non-SMP mode, No comm thread support),
- * its value is used as 0.
- */
-#if CMK_ONESIDED_IMPL && CMK_SMP && CK_MSG_IMMEDIATE
-#define CK_MSG_RDMA CK_MSG_IMMEDIATE
-#else
-#define CK_MSG_RDMA 0
-#endif
-
-#if CMK_ONESIDED_IMPL
-
-/* Sender Functions */
-
-//Prepare metadata message with the relevant machine specific info
-void CkRdmaPrepareMsg(envelope **env, int pe);
-
-//Create a new message with machine specific information
-envelope* CkRdmaCreateMetadataMsg(envelope *env, int pe);
-
-//Handle ack received on the sender by invoking callback
-void CkHandleRdmaCookie(void *cookie);
-
-
-
-/* Receiver Functions */
-
-//Copy the message using pointers when it's on the same PE/node
-envelope* CkRdmaCopyMsg(envelope *env);
-
-/*
- * Extract rdma based information from the metadata message,
- * allocate buffers and issue RDMA get call
- */
-void CkRdmaIssueRgets(envelope *env);
-
-/*
- * Method called to update machine specific information and pointers
- * inside Ckrdmawrappers
- */
-void CkUpdateRdmaPtrs(envelope *msg, int msgsize, char *recv_md, char *src_md);
-
-/*
- * Method called to pack rdma pointers
- * inside Ckrdmawrappers
- */
-void CkPackRdmaPtrs(char *msgBuf);
-
-/*
- * Method called to unpack rdma pointers
- * inside Ckrdmawrappers
- */
-void CkUnpackRdmaPtrs(char *msgBuf);
-
-//Get the number of rdma ops using the metadata message
-int getRdmaNumOps(envelope *env);
-
-//Get the sum of rdma buffer sizes using the metadata message
-int getRdmaBufSize(envelope *env);
-
-
-#endif /* End of CMK_ONESIDED_IMPL */
-
-
-/* Support for Nocopy Direct API */
-
-/* Use 0 sized headers for generic Direct API implementation */
 #ifndef CMK_NOCOPY_DIRECT_BYTES
 
 #if defined(_WIN32)
@@ -103,15 +36,19 @@ int getRdmaBufSize(envelope *env);
 #define CkRdmaAlloc CmiRdmaAlloc
 #define CkRdmaFree  CmiRdmaFree
 
-// Ack handler function which invokes the callbacks on the source and destination PEs
-void CkRdmaAckHandler(void *cookie);
-void CkRdmaDirectAckHandler(void *ack);
-
+// Represents the mode of the zerocopy transfer
+// CkNcpyMode::MEMCPY indicates that the PEs are on the logical node and memcpy can be used
+// CkNcpyMode::CMA indicates that the PEs are on the same physical node and CMA can be used
+// CkNcpyMode::RDMA indicates that the neither MEMCPY or CMA can be used and REMOTE Direct Memory Access needs to be used
 enum class CkNcpyMode : char { MEMCPY, CMA, RDMA };
 
+// Represents the completion status of the zerocopy transfer (used as a return value for CkNcpyBuffer::get & CkNcpyBuffer:::put)
+// CMA and MEMCPY transfers complete instantly and return CkNcpyStatus::complete
+// RDMA transfers use a remote asynchronous call and hence return CkNcpyStatus::incomplete
 enum class CkNcpyStatus : char { incomplete, complete };
 
-// Class to represent an RDMA buffer
+// Class to represent an Zerocopy buffer
+// CkSendBuffer(....) passed by the user internally translates to a CkNcpyBuffer
 class CkNcpyBuffer{
 
   private:
@@ -150,6 +87,22 @@ class CkNcpyBuffer{
 
   CkNcpyBuffer() : isRegistered(false), ptr(NULL), cnt(0), pe(-1), mode(CK_BUFFER_REG), ref(NULL) {}
 
+  explicit CkNcpyBuffer(const void *address, unsigned short int mode_=CK_BUFFER_REG) {
+    ptr = address;
+    pe = CkMyPe();
+    cb = CkCallback(CkCallback::ignore);
+    mode = mode_;
+    isRegistered = false;
+  }
+
+  CkNcpyBuffer(const void *address, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_REG) {
+    ptr = address;
+    pe = CkMyPe();
+    cb = cb_;
+    mode = mode_;
+    isRegistered = false;
+  }
+
   CkNcpyBuffer(const void *ptr_, size_t cnt_, CkCallback &cb_, unsigned short int mode_=CK_BUFFER_REG) {
     init(ptr_, cnt_, cb_, mode_);
   }
@@ -160,8 +113,8 @@ class CkNcpyBuffer{
     cb   = cb_;
     pe   = CkMyPe();
     mode = mode_;
-    isRegistered = false;
 
+    isRegistered = false;
     // Register memory everytime new values are initialized
     registerMem();
   }
@@ -180,9 +133,6 @@ class CkNcpyBuffer{
     // Check that this object is local when registerMem is called
     CkAssert(CkNodeOf(pe) == CkMyNode());
 
-    if(isRegistered == true)
-      return;
-
     // Set machine layer information when mode is not CK_BUFFER_NOREG
     if(mode != CK_BUFFER_NOREG) {
 
@@ -244,6 +194,89 @@ class CkNcpyBuffer{
   }
 
   friend void CkRdmaDirectAckHandler(void *ack);
+
+  friend void constructSourceBufferObject(NcpyOperationInfo *info, CkNcpyBuffer &src);
+  friend void constructDestinationBufferObject(NcpyOperationInfo *info, CkNcpyBuffer &dest);
+
+  friend envelope* CkRdmaIssueRgets(envelope *env);
 };
 
+// Ack handler for the Zerocopy Direct API
+// Invoked on the completion of any RDMA operation calling using the Direct API
+void CkRdmaDirectAckHandler(void *ack);
+
+// Method to invoke a callback on a particular pe with a CkNcpyBuffer being passed
+// as a part of a CkDataMsg. This method is used to invoke callbacks on specific pes
+// after the completion of the Zerocopy Direct API operation
+void invokeCallback(void *cb, int pe, CkNcpyBuffer &buff);
+
+// Returns CkNcpyMode::MEMCPY if both the PEs are the same and memcpy can be used
+// Returns CkNcpyMode::CMA if both the PEs are in the same physical node and CMA can be used
+// Returns CkNcpyMode::RDMA if RDMA needs to be used
+CkNcpyMode findTransferMode(int srcPe, int destPe);
+
+
+void invokeSourceCallback(NcpyOperationInfo *info);
+
+void invokeDestinationCallback(NcpyOperationInfo *info);
+
+
+
+
+/*********************************** Zerocopy Entry Method API ****************************/
+#define CkSendBuffer(...) CkNcpyBuffer(__VA_ARGS__)
+
+#if CMK_ONESIDED_IMPL
+
+// NOTE: Inside CkRdmaIssueRgets, a large message allocation is made consisting of space
+// for the destination or receiver buffers and some additional information required for processing
+// and acknowledgment handling. The space for additional information is typically equal to
+// sizeof(NcpyEmInfo) + numops * sizeof(NcpyEmBufferInfo)
+
+// This structure is used to store zerocopy information associated with an entry method
+// invocation which uses the RDMA mode of transfer in Zerocopy Entry Method API.
+// A variable of the structure stores the information in order to access it after the
+// completion of the Rget operation (which is an asynchronous call) in order to invoke
+// the entry method
+struct NcpyEmInfo{
+  int numOps; // number of zerocopy operations i.e number of buffers sent using CkSendBuffer
+  int counter; // used for tracking the number of completed RDMA operations
+  void *msg; // pointer to the Charm++ message which will be enqueued after completion of all Rgets
+};
+
+// This structure is used to store the buffer information specific to each buffer being sent
+// using the Zerocopy Entry Method API. A variable of the structure stores the information associated
+// with each buffer
+struct NcpyEmBufferInfo{
+  int index;  // Represents the index of the buffer information (from 0,1... numops - 1)
+  NcpyOperationInfo ncpyOpInfo; // Stores all the information required for the zerocopy operation
+};
+
+/*
+ * Extract ncpy buffer information from the metadata message,
+ * allocate buffers and issue ncpy calls (either memcpy or cma read or rdma get)
+ */
+envelope* CkRdmaIssueRgets(envelope *env);
+
+void handleEntryMethodApiCompletion(NcpyOperationInfo *info);
+
+void handleReverseEntryMethodApiCompletion(NcpyOperationInfo *info);
+
+// Method called to pack rdma pointers
+void CkPackRdmaPtrs(char *msgBuf);
+
+// Method called to pack rdma pointers
+void CkUnpackRdmaPtrs(char *msgBuf);
+
+// Get the number of ncpy ops using the metadata message
+int getRdmaNumOps(envelope *env);
+
+// Get the sum of ncpy buffer sizes using the metadata message
+int getRdmaBufSize(envelope *env);
+
+// Ack handler function for the nocopy EM API
+void CkRdmaEMAckHandler(int destPe, void *ack);
+
+#endif /* End of CMK_ONESIDED_IMPL */
+
 #endif
diff --git a/src/ck-core/ckrdmawrapper.h b/src/ck-core/ckrdmawrapper.h
deleted file mode 100644 (file)
index d7ee952..0000000
+++ /dev/null
@@ -1,38 +0,0 @@
-/*This class is a wrapper class for binding
- * the data specific to one zero copy operation*/
-
-#ifndef __CKRDMAWRAPPER_H
-#define __CKRDMAWRAPPER_H
-
-#include "ckcallback.h"
-
-#define CkSendBuffer(...) CkRdmaWrapper(__VA_ARGS__)
-
-class CkRdmaWrapper{
-  public:
-  const void* ptr;
-  CkCallback *callback;
-
-  //this field is used for handling rdma acks called by comm thread
-  //the comm thread calls the callbackgroup to call the callback on the
-  //appropriate pe.
-  int srcPe;
-  size_t cnt;
-
-  void pup(PUP::er &p){
-    pup_bytes(&p, this, sizeof(CkRdmaWrapper));
-  }
-
-  CkRdmaWrapper() : ptr(NULL), callback(NULL) {
-    srcPe = -1;
-  }
-  explicit CkRdmaWrapper(const void *address) : ptr(address){
-    srcPe = CkMyPe();
-    callback = new CkCallback(CkCallback::ignore);
-  }
-  CkRdmaWrapper(const void *address, CkCallback cb) : ptr(address) {
-    srcPe = CkMyPe();
-    callback = new CkCallback(cb);
-  }
-};
-#endif
index 3c612f18a5b77c333cf6f2f24d21bd68b26c0635..8a2ce4ddc488e363556eef08cafe7543130e8c5a 100644 (file)
@@ -1286,9 +1286,12 @@ void _initCharm(int unused_argc, char **argv)
     CkpvAccess(envelopeEventID) = 0;
        CkMessageWatcherInit(argv,CkpvAccess(_coreState));
        
-       // Set the ack handler function used for the direct nocopy api
-       CmiSetRdmaNcpyAck(CkRdmaDirectAckHandler);
+       // Set the ack handler function used for the direct nocopy api and the entry method nocopy api
+       CmiSetDirectNcpyAckHandler(CkRdmaDirectAckHandler);
 
+#if CMK_ONESIDED_IMPL
+       CmiSetEMNcpyAckHandler(CkRdmaEMAckHandler);
+#endif
        /**
          The rank-0 processor of each node calls the 
          translator-generated "_register" routines. 
index 5a40de4cb64bf21299b4d25925e5adab3d0ba0f0..ef1bb4fc0d50aaa46ceed4843254cd61a430825d 100644 (file)
@@ -17,7 +17,7 @@ int CmiGetRdmaCommonInfoSize() {
 /* Support for generic implementation */
 
 // Function Pointer to Acknowledement handler function for the Direct API
-RdmaAckCallerFn ncpyAckHandlerFn;
+RdmaDirectAckCallerFn ncpyDirectAckHandlerFn;
 
 // An Rget initiator PE sends this message to the target PE that will be the source of the data
 typedef struct _converseRdmaMsg {
@@ -34,7 +34,7 @@ static void getRequestHandler(ConverseRdmaMsg *getReqMsg){
 
   resetNcpyOpInfoPointers(ncpyOpInfo);
 
-  ncpyOpInfo->freeMe = 0;
+  ncpyOpInfo->freeMe = CMK_DONT_FREE_NCPYOPINFO;
 
   // Get is implemented internally using a call to Put
   CmiIssueRput(ncpyOpInfo);
@@ -53,10 +53,9 @@ static void putDataHandler(ConverseRdmaMsg *payloadMsg) {
          ncpyOpInfo->srcSize);
 
   // Invoke the destination ack
-  ncpyOpInfo->ackMode = 2;
-  ncpyOpInfo->freeMe  = 0;
-  ncpyAckHandlerFn(ncpyOpInfo);
-  //ncpyAckHandlerFn(destAck, recvMsg->pe, recvMsg->destAddr);
+  ncpyOpInfo->ackMode = CMK_DEST_ACK; // Only invoke the destination ack
+  ncpyOpInfo->freeMe  = CMK_DONT_FREE_NCPYOPINFO;
+  ncpyDirectAckHandlerFn(ncpyOpInfo);
 }
 
 // Rget/Rput operations are implemented as normal converse messages
@@ -66,8 +65,8 @@ void CmiOnesidedDirectInit(void) {
   put_data_handler_idx = CmiRegisterHandler((CmiHandler)putDataHandler);
 }
 
-void CmiSetRdmaNcpyAck(RdmaAckCallerFn fn) {
-  ncpyAckHandlerFn = fn;
+void CmiSetDirectNcpyAckHandler(RdmaDirectAckCallerFn fn) {
+  ncpyDirectAckHandlerFn = fn;
 }
 
 void CmiIssueRget(NcpyOperationInfo *ncpyOpInfo) {
@@ -108,9 +107,9 @@ void CmiIssueRput(NcpyOperationInfo *ncpyOpInfo) {
          size);
 
   // Invoke the source ack
-  ncpyOpInfo->ackMode = 1;
+  ncpyOpInfo->ackMode = CMK_SRC_ACK; // only invoke the source ack
 
-  ncpyAckHandlerFn(ncpyOpInfo);
+  ncpyDirectAckHandlerFn(ncpyOpInfo);
 
   CmiSetHandler(payloadMsg, put_data_handler_idx);
   CmiSyncSendAndFree(ncpyOpInfo->destPe,
@@ -121,4 +120,44 @@ void CmiIssueRput(NcpyOperationInfo *ncpyOpInfo) {
 void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode) {}
 
 void CmiDeregisterMem(const void *ptr, void *info, int pe, unsigned short int mode) {}
+
+#else
+
+// Support for sending an ack message for the Entry Method API
+
+RdmaEMAckCallerFn ncpyEMAckHandlerFn;
+
+static int invoke_entry_method_ack_handler_idx;
+
+// Ack Message is typically used in case of reverse operation (when a reverse put is used instead of a get)
+typedef struct _ackEntryMethodMsg{
+  char cmicore[CmiMsgHeaderSizeBytes];
+  void *ref;
+} ackEntryMethodMsg;
+
+// Handler invoked on receiving a ackEntryMethodMsg
+// This handler invokes the ncpyEMAckHandler on the receiver side
+static void ackEntryMethodHandler(ackEntryMethodMsg *msg) {
+  // Invoke the charm handler
+  ncpyEMAckHandlerFn(CmiMyPe(), msg->ref);
+}
+
+// Method to create a ackEntryMethodMsg and send it
+void CmiInvokeRemoteAckHandler(int pe, void *ref) {
+  ackEntryMethodMsg *msg = (ackEntryMethodMsg *)CmiAlloc(sizeof(ackEntryMethodMsg));
+  msg->ref = ref;
+
+  CmiSetHandler(msg, invoke_entry_method_ack_handler_idx);
+  CmiSyncSendAndFree(pe, sizeof(ackEntryMethodMsg), msg);
+}
+
+// Register converse handler for invoking ack on reverse operation
+void CmiOnesidedDirectInit(void) {
+  invoke_entry_method_ack_handler_idx = CmiRegisterHandler((CmiHandler)ackEntryMethodHandler);
+}
+
+void CmiSetEMNcpyAckHandler(RdmaEMAckCallerFn fn) {
+  // set the EM Ack caller function
+  ncpyEMAckHandlerFn = fn;
+}
 #endif
index c3b6ee65927cc5691bb28ab2a58a7a3bae0ff7c8..5316a368e5f736e533dcb614824e55a3da07c3e0 100644 (file)
@@ -3,30 +3,20 @@
 
 #include "cmirdmautils.h"
 
-typedef void (*RdmaSingleAckCallerFn)(void *cbPtr, int pe, const void *ptr);
-typedef void (*RdmaAckCallerFn)(void *token);
-
-void *CmiSetRdmaAck(RdmaAckCallerFn fn, void *token);
-void CmiSetRdmaInfo(void *dest, int destPE, int numOps);
-void CmiSetRdmaOpInfo(void *dest, const void *ptr, int size, void *ack, int destPE);
-int CmiGetRdmaOpInfoSize(void);
-int CmiGetRdmaGenInfoSize(void);
-
-int CmiGetRdmaInfoSize(int numOps);
-void CmiSetRdmaRecvInfo(void *dest, int numOps, void *msg, void *rdmaInfo, int msgSize);
-void CmiSetRdmaRecvOpInfo(void *dest, void *buffer, void *src_ref, int size, int opIndex, void *rdmaInfo);
-int CmiGetRdmaOpRecvInfoSize(void);
-int CmiGetRdmaGenRecvInfoSize(void);
-int CmiGetRdmaRecvInfoSize(int numOps);
-
-void CmiIssueRgets(void *recv, int pe);
+typedef void (*RdmaEMAckCallerFn)(int destPe, void *token);
+typedef void (*RdmaDirectAckCallerFn)(void *token);
 
 /* Support for Direct API */
 void CmiSetRdmaCommonInfo(void *info, const void *ptr, int size);
 int CmiGetRdmaCommonInfoSize(void);
 
 void CmiSetRdmaBufferInfo(void *info, const void *ptr, int size, unsigned short int mode);
-void CmiSetRdmaNcpyAck(RdmaAckCallerFn fn);
+
+// Function to set the ack handler for the Direct API
+void CmiSetDirectNcpyAckHandler(RdmaDirectAckCallerFn fn);
+
+// Function to set the ack handler for the Entry Method API
+void CmiSetEMNcpyAckHandler(RdmaEMAckCallerFn fn);
 
 /* CmiIssueRget initiates an RDMA read operation, transferring 'size' bytes of data from the address space of 'srcPe' to local address, 'destAddr'.
  * When the runtime invokes srcAck on the source (target), it indicates safety to overwrite or free the srcAddr buffer.
@@ -70,9 +60,10 @@ void *CmiRdmaAlloc(int size);
 
 int CmiDoesCMAWork(void);
 
-#if !CMK_ONESIDED_DIRECT_IMPL
-// Function declaration used for the generic implementation of the Nocopy Direct API
+// Method used to send an ack after completion of a reverse rdma operation
+void CmiInvokeRemoteAckHandler(int pe, void *ref);
+
+// Function declaration for onesided initialization
 void CmiOnesidedDirectInit(void);
-#endif
 
 #endif
index 0db9e29d276a8e3dfbee6d3936362b0ee3b293da..da3500c8b613d686ed8f3e2e809100b16ccab9da 100644 (file)
@@ -3860,10 +3860,8 @@ void ConverseCommonInit(char **argv)
 
   CmiPersistentInit();
   CmiIsomallocInit(argv);
-#if !CMK_ONESIDED_DIRECT_IMPL
   // Initialize converse handlers for supporting generic Direct Nocopy API
   CmiOnesidedDirectInit();
-#endif
   CmiDeliversInit();
   CsdInit(argv);
 #if CMK_CCS_AVAILABLE
index 86908cd840490476a7a8e2af767d5db29b6fd25b..3faff4a5089bb869328c3e90c59613483803fc36 100644 (file)
@@ -12,17 +12,16 @@ AdaptiveLB.o: AdaptiveLB.C AdaptiveLB.h CentralLB.h BaseLB.h LBDatabase.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h AdaptiveLB.decl.h ckgraph.h \
- AdaptiveLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ AdaptiveLB.decl.h ckgraph.h AdaptiveLB.def.h
 
 BaseLB.o: BaseLB.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -34,18 +33,18 @@ BaseLB.o: BaseLB.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h BaseLB.h LBSimulation.h charm++.h \
- CentralLB.h CentralLB.decl.h CentralLBMsg.h manager.h BaseLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
BaseLB.h LBSimulation.h charm++.h CentralLB.h CentralLB.decl.h \
+ CentralLBMsg.h manager.h BaseLB.def.h
 
 BlockLB.o: BlockLB.C BlockLB.decl.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -58,18 +57,18 @@ BlockLB.o: BlockLB.C BlockLB.decl.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
- BlockLB.h CentralLB.h BaseLB.h manager.h BlockLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h BlockLB.h CentralLB.h BaseLB.h manager.h \
+ BlockLB.def.h
 
 BlueGene.o: BlueGene.C BlueGene.h BlueGene.decl.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -82,17 +81,17 @@ BlueGene.o: BlueGene.C BlueGene.h BlueGene.decl.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h BlueGene.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ BlueGene.def.h
 
 CentralLB.o: CentralLB.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -104,22 +103,21 @@ CentralLB.o: CentralLB.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h ckcallback.h envelope.h \
- init.h charm++.h register.h stats.h ckfutures.h ../include/TopoManager.h \
- ../include/topomanager_config.h ../include/converse.h CentralLB.h \
- BaseLB.h LBDatabase.h CentralLB.decl.h sdag.h BaseLB.decl.h \
- LBDatabase.decl.h CentralLBMsg.h pup_stl.h manager.h LBDBManager.h \
- LBSimulation.h CentralLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h ckcallback.h envelope.h init.h charm++.h register.h stats.h \
+ ckfutures.h ../include/TopoManager.h ../include/topomanager_config.h \
+ ../include/converse.h CentralLB.h BaseLB.h LBDatabase.h CentralLB.decl.h \
+ sdag.h BaseLB.decl.h LBDatabase.decl.h CentralLBMsg.h pup_stl.h \
+ manager.h LBDBManager.h LBSimulation.h CentralLB.def.h
 
 CentralPredictor.o: CentralPredictor.C charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -131,12 +129,12 @@ CentralPredictor.o: CentralPredictor.C charm++.h charm.h converse.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -158,17 +156,16 @@ ComboCentLB.o: ComboCentLB.C ComboCentLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h ComboCentLB.decl.h \
- ComboCentLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ ComboCentLB.decl.h ComboCentLB.def.h
 
 CommAwareRefineLB.o: CommAwareRefineLB.C CommAwareRefineLB.h CentralLB.h \
  BaseLB.h LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -183,17 +180,16 @@ CommAwareRefineLB.o: CommAwareRefineLB.C CommAwareRefineLB.h CentralLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h CommAwareRefineLB.decl.h \
- ckgraph.h CommAwareRefineLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ CommAwareRefineLB.decl.h ckgraph.h CommAwareRefineLB.def.h
 
 CommLB.o: CommLB.C CommLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -207,18 +203,17 @@ CommLB.o: CommLB.C CommLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h CommLB.decl.h charm++.h \
- envelope.h sdag.h CentralLB.decl.h CommLBHeap.h GreedyCommLB.h \
- CentralLB.h GreedyCommLB.decl.h CommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ CommLB.decl.h charm++.h envelope.h sdag.h CentralLB.decl.h CommLBHeap.h \
+ GreedyCommLB.h CentralLB.h GreedyCommLB.decl.h CommLB.def.h
 
 CommLBHeap.o: CommLBHeap.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -230,17 +225,17 @@ CommLBHeap.o: CommLBHeap.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CommLBHeap.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ CommLBHeap.h
 
 CommonLBs.o: CommonLBs.C LBDatabase.h lbdb.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -253,24 +248,23 @@ CommonLBs.o: CommonLBs.C LBDatabase.h lbdb.h converse.h conv-header.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
- ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
- cklocation.h LBDatabase.h MetaBalancer.h RandomForestModel.h \
- MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
- CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
- CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
- CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
- pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
- NullLB.decl.h BaseLB.decl.h CommonLBs.decl.h charm++.h envelope.h sdag.h \
- DummyLB.decl.h CentralLB.decl.h BaseLB.decl.h LBDatabase.decl.h \
- CentralLBMsg.h GreedyLB.decl.h GreedyRefineLB.decl.h CommLB.decl.h \
- RandCentLB.decl.h RefineLB.decl.h RefineCommLB.decl.h RotateLB.decl.h \
- DistributedLB.decl.h DistBaseLB.decl.h HierarchicalLB.decl.h \
- HybridLB.decl.h HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h \
- ComboCentLB.decl.h RefineSwapLB.decl.h NeighborLB.decl.h \
- NborBaseLB.decl.h OrbLB.decl.h BlockLB.decl.h GreedyCommLB.decl.h \
- NodeLevelLB.decl.h CommonLBs.def.h
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
+ CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
+ MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
+ ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
+ waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
+ ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
+ ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
+ CommonLBs.decl.h charm++.h envelope.h sdag.h DummyLB.decl.h \
+ CentralLB.decl.h BaseLB.decl.h LBDatabase.decl.h CentralLBMsg.h \
+ GreedyLB.decl.h GreedyRefineLB.decl.h CommLB.decl.h RandCentLB.decl.h \
+ RefineLB.decl.h RefineCommLB.decl.h RotateLB.decl.h DistributedLB.decl.h \
+ DistBaseLB.decl.h HierarchicalLB.decl.h HybridLB.decl.h \
+ HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h ComboCentLB.decl.h \
+ RefineSwapLB.decl.h NeighborLB.decl.h NborBaseLB.decl.h OrbLB.decl.h \
+ BlockLB.decl.h GreedyCommLB.decl.h NodeLevelLB.decl.h CommonLBs.def.h
 
 DistBaseLB.o: DistBaseLB.C BaseLB.h LBDatabase.h lbdb.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -284,16 +278,16 @@ DistBaseLB.o: DistBaseLB.C BaseLB.h LBDatabase.h lbdb.h converse.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- DistBaseLB.h DistBaseLB.decl.h DistBaseLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h DistBaseLB.h DistBaseLB.decl.h \
+ DistBaseLB.def.h
 
 DistributedLB.o: DistributedLB.C DistributedLB.h DistBaseLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -308,17 +302,16 @@ DistributedLB.o: DistributedLB.C DistributedLB.h DistBaseLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- DistBaseLB.decl.h DistributedLB.decl.h ckheap.h elements.h ckset.h \
- DistributedLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h DistBaseLB.decl.h DistributedLB.decl.h \
+ ckheap.h elements.h ckset.h DistributedLB.def.h
 
 DummyLB.o: DummyLB.C DummyLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -332,16 +325,16 @@ DummyLB.o: DummyLB.C DummyLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h DummyLB.decl.h DummyLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ DummyLB.decl.h DummyLB.def.h
 
 EveryLB.o: EveryLB.C LBDatabase.h lbdb.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -354,29 +347,28 @@ EveryLB.o: EveryLB.C LBDatabase.h lbdb.h converse.h conv-header.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
- ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
- cklocation.h LBDatabase.h MetaBalancer.h RandomForestModel.h \
- MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
- CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
- CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
- CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
- pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
- NullLB.decl.h BaseLB.decl.h EveryLB.decl.h charm++.h envelope.h sdag.h \
- DummyLB.decl.h CentralLB.decl.h BaseLB.decl.h LBDatabase.decl.h \
- CentralLBMsg.h GreedyLB.decl.h GreedyRefineLB.decl.h CommLB.decl.h \
- RandCentLB.decl.h RefineLB.decl.h RefineCommLB.decl.h RotateLB.decl.h \
- DistributedLB.decl.h DistBaseLB.decl.h HierarchicalLB.decl.h \
- HybridLB.decl.h HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h \
- ComboCentLB.decl.h RefineSwapLB.decl.h NeighborLB.decl.h \
- NborBaseLB.decl.h OrbLB.decl.h BlockLB.decl.h GreedyCommLB.decl.h \
- NodeLevelLB.decl.h GraphPartLB.decl.h GraphBFTLB.decl.h \
- GridCommLB.decl.h GridCommRefineLB.decl.h HbmLB.decl.h RefineKLB.decl.h \
- TempAwareCommLB.decl.h TreeMatchLB.decl.h GreedyAgentLB.decl.h \
- NeighborCommLB.decl.h PhasebyArrayLB.decl.h RecBipartLB.decl.h \
- CommAwareRefineLB.decl.h AdaptiveLB.decl.h MetisLB.decl.h \
- GridMetisLB.decl.h EveryLB.def.h
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
+ CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
+ MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
+ ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
+ waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
+ ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
+ ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
+ EveryLB.decl.h charm++.h envelope.h sdag.h DummyLB.decl.h \
+ CentralLB.decl.h BaseLB.decl.h LBDatabase.decl.h CentralLBMsg.h \
+ GreedyLB.decl.h GreedyRefineLB.decl.h CommLB.decl.h RandCentLB.decl.h \
+ RefineLB.decl.h RefineCommLB.decl.h RotateLB.decl.h DistributedLB.decl.h \
+ DistBaseLB.decl.h HierarchicalLB.decl.h HybridLB.decl.h \
+ HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h ComboCentLB.decl.h \
+ RefineSwapLB.decl.h NeighborLB.decl.h NborBaseLB.decl.h OrbLB.decl.h \
+ BlockLB.decl.h GreedyCommLB.decl.h NodeLevelLB.decl.h GraphPartLB.decl.h \
+ GraphBFTLB.decl.h GridCommLB.decl.h GridCommRefineLB.decl.h HbmLB.decl.h \
+ RefineKLB.decl.h TempAwareCommLB.decl.h TreeMatchLB.decl.h \
+ GreedyAgentLB.decl.h NeighborCommLB.decl.h PhasebyArrayLB.decl.h \
+ RecBipartLB.decl.h CommAwareRefineLB.decl.h AdaptiveLB.decl.h \
+ MetisLB.decl.h GridMetisLB.decl.h EveryLB.def.h
 
 GraphBFTLB.o: GraphBFTLB.C GraphBFTLB.h CentralLB.h BaseLB.h LBDatabase.h \
  lbdb.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -391,17 +383,16 @@ GraphBFTLB.o: GraphBFTLB.C GraphBFTLB.h CentralLB.h BaseLB.h LBDatabase.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h GraphBFTLB.decl.h ckgraph.h \
- GraphBFTLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ GraphBFTLB.decl.h ckgraph.h GraphBFTLB.def.h
 
 GraphPartLB.o: GraphPartLB.C GraphPartLB.h CentralLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -416,17 +407,16 @@ GraphPartLB.o: GraphPartLB.C GraphPartLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h GraphPartLB.decl.h ckgraph.h \
- GraphPartLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ GraphPartLB.decl.h ckgraph.h GraphPartLB.def.h
 
 GreedyAgentLB.o: GreedyAgentLB.C LBSimulation.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -438,12 +428,12 @@ GreedyAgentLB.o: GreedyAgentLB.C LBSimulation.h charm++.h charm.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -466,17 +456,16 @@ GreedyCommLB.o: GreedyCommLB.C elements.h converse.h conv-header.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h GreedyCommLB.decl.h \
- CommLBHeap.h GreedyCommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ GreedyCommLB.decl.h CommLBHeap.h GreedyCommLB.def.h
 
 GreedyLB.o: GreedyLB.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -488,19 +477,18 @@ GreedyLB.o: GreedyLB.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ckgraph.h BaseLB.h GreedyLB.h \
- CentralLB.h CentralLB.decl.h CentralLBMsg.h manager.h GreedyLB.decl.h \
- GreedyLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ ckgraph.h BaseLB.h GreedyLB.h CentralLB.h CentralLB.decl.h \
+ CentralLBMsg.h manager.h GreedyLB.decl.h GreedyLB.def.h
 
 GreedyRefineLB.o: GreedyRefineLB.C charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -513,19 +501,18 @@ GreedyRefineLB.o: GreedyRefineLB.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ckgraph.h BaseLB.h GreedyRefineLB.h \
- CentralLB.h CentralLB.decl.h CentralLBMsg.h manager.h \
- GreedyRefineLB.decl.h GreedyRefineLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ ckgraph.h BaseLB.h GreedyRefineLB.h CentralLB.h CentralLB.decl.h \
+ CentralLBMsg.h manager.h GreedyRefineLB.decl.h GreedyRefineLB.def.h
 
 GridCommLB.o: GridCommLB.C GridCommLB.decl.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -538,18 +525,18 @@ GridCommLB.o: GridCommLB.C GridCommLB.decl.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
GridCommLB.h CentralLB.h BaseLB.h manager.h GridCommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h GridCommLB.h CentralLB.h BaseLB.h \
+ manager.h GridCommLB.def.h
 
 GridCommRefineLB.o: GridCommRefineLB.C GridCommRefineLB.decl.h charm++.h \
  charm.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -562,18 +549,18 @@ GridCommRefineLB.o: GridCommRefineLB.C GridCommRefineLB.decl.h charm++.h \
  simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
GridCommRefineLB.h CentralLB.h BaseLB.h manager.h GridCommRefineLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h GridCommRefineLB.h CentralLB.h BaseLB.h \
+ manager.h GridCommRefineLB.def.h
 
 GridHybridLB.o: GridHybridLB.C GridHybridLB.decl.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -586,18 +573,18 @@ GridHybridLB.o: GridHybridLB.C GridHybridLB.decl.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
GridHybridLB.h CentralLB.h BaseLB.h manager.h GridHybridLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h GridHybridLB.h CentralLB.h BaseLB.h \
+ manager.h GridHybridLB.def.h
 
 GridHybridSeedLB.o: GridHybridSeedLB.C GridHybridSeedLB.decl.h charm++.h \
  charm.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -610,18 +597,18 @@ GridHybridSeedLB.o: GridHybridSeedLB.C GridHybridSeedLB.decl.h charm++.h \
  simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
GridHybridSeedLB.h CentralLB.h BaseLB.h manager.h GridHybridSeedLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h GridHybridSeedLB.h CentralLB.h BaseLB.h \
+ manager.h GridHybridSeedLB.def.h
 
 GridMetisLB.o: GridMetisLB.C GridMetisLB.decl.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -634,18 +621,18 @@ GridMetisLB.o: GridMetisLB.C GridMetisLB.decl.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
GridMetisLB.h CentralLB.h BaseLB.h manager.h GridMetisLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h GridMetisLB.h CentralLB.h BaseLB.h \
+ manager.h GridMetisLB.def.h
 
 HbmLB.o: HbmLB.C HbmLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -659,20 +646,19 @@ HbmLB.o: HbmLB.C HbmLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h HbmLB.decl.h NeighborLBMsg.h \
- HybridLBMsg.h topology.h GreedyLB.h GreedyLB.decl.h GreedyCommLB.h \
- GreedyCommLB.decl.h CommLBHeap.h RefineCommLB.h RefinerComm.h Refiner.h \
- elements.h ckset.h ckheap.h RefineLB.h RefineLB.decl.h \
- RefineCommLB.decl.h HbmLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ HbmLB.decl.h NeighborLBMsg.h HybridLBMsg.h topology.h GreedyLB.h \
+ GreedyLB.decl.h GreedyCommLB.h GreedyCommLB.decl.h CommLBHeap.h \
+ RefineCommLB.h RefinerComm.h Refiner.h elements.h ckset.h ckheap.h \
+ RefineLB.h RefineLB.decl.h RefineCommLB.decl.h HbmLB.def.h
 
 HierarchicalLB.o: HierarchicalLB.C HierarchicalLB.h HierarchicalLBTypes.h \
  DistBaseLB.h BaseLB.h LBDatabase.h lbdb.h converse.h conv-header.h \
@@ -687,17 +673,16 @@ HierarchicalLB.o: HierarchicalLB.C HierarchicalLB.h HierarchicalLBTypes.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- DistBaseLB.decl.h HierarchicalLB.decl.h ckheap.h elements.h ckset.h \
- HierarchicalLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h DistBaseLB.decl.h HierarchicalLB.decl.h \
+ ckheap.h elements.h ckset.h HierarchicalLB.def.h
 
 HybridBaseLB.o: HybridBaseLB.C HybridBaseLB.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -710,22 +695,21 @@ HybridBaseLB.o: HybridBaseLB.C HybridBaseLB.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h BaseLB.h CentralLB.h CentralLB.decl.h \
- CentralLBMsg.h manager.h HybridBaseLB.decl.h NeighborLBMsg.h \
- HybridLBMsg.h topology.h GreedyLB.h GreedyLB.decl.h GreedyCommLB.h \
- GreedyCommLB.decl.h CommLBHeap.h RefineCommLB.h RefinerComm.h Refiner.h \
- elements.h ckset.h ckheap.h RefineLB.h RefineLB.decl.h \
- RefineCommLB.decl.h HybridBaseLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ BaseLB.h CentralLB.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h topology.h GreedyLB.h \
+ GreedyLB.decl.h GreedyCommLB.h GreedyCommLB.decl.h CommLBHeap.h \
+ RefineCommLB.h RefinerComm.h Refiner.h elements.h ckset.h ckheap.h \
+ RefineLB.h RefineLB.decl.h RefineCommLB.decl.h HybridBaseLB.def.h
 
 HybridLB.o: HybridLB.C HybridLB.h CentralLB.h BaseLB.h LBDatabase.h \
  lbdb.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -740,18 +724,17 @@ HybridLB.o: HybridLB.C HybridLB.h CentralLB.h BaseLB.h LBDatabase.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h HybridBaseLB.h \
- HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h topology.h \
- HybridLB.decl.h GreedyLB.h GreedyLB.decl.h GreedyRefineLB.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ HybridBaseLB.h HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h \
+ topology.h HybridLB.decl.h GreedyLB.h GreedyLB.decl.h GreedyRefineLB.h \
  GreedyRefineLB.decl.h GreedyCommLB.h GreedyCommLB.decl.h CommLBHeap.h \
  RefineCommLB.h RefinerComm.h Refiner.h elements.h ckset.h ckheap.h \
  RefineLB.h RefineLB.decl.h RefineCommLB.decl.h MetisLB.h MetisLB.decl.h \
@@ -769,16 +752,16 @@ LBAgent.o: LBAgent.C LBAgent.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h topology.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ topology.h
 
 LBComm.o: LBComm.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -799,17 +782,17 @@ LBDBManager.o: LBDBManager.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h LBDBManager.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ LBDBManager.h
 
 LBDatabase.o: LBDatabase.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -823,19 +806,18 @@ LBDatabase.o: LBDatabase.C converse.h conv-header.h conv-config.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- LBSimulation.h charm++.h CentralLB.h BaseLB.h CentralLB.decl.h \
- CentralLBMsg.h manager.h topology.h DistributedLB.h DistBaseLB.h \
- DistBaseLB.decl.h DistributedLB.decl.h ckheap.h NullLB.h BaseLB.h \
- LBDatabase.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h LBSimulation.h charm++.h CentralLB.h \
+ BaseLB.h CentralLB.decl.h CentralLBMsg.h manager.h topology.h \
+ DistributedLB.h DistBaseLB.h DistBaseLB.decl.h DistributedLB.decl.h \
+ ckheap.h NullLB.h BaseLB.h LBDatabase.def.h
 
 LBMachineUtil.o: LBMachineUtil.C LBDatabase.h lbdb.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -849,15 +831,15 @@ LBMachineUtil.o: LBMachineUtil.C LBDatabase.h lbdb.h converse.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h
 
 LBObj.o: LBObj.C charm++.h charm.h converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -869,17 +851,17 @@ LBObj.o: LBObj.C charm++.h charm.h converse.h conv-header.h conv-config.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h LBObj.h LBOM.h LBDBManager.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ LBObj.h LBOM.h LBDBManager.h
 
 LBProfit.o: LBProfit.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -893,16 +875,15 @@ LBProfit.o: LBProfit.C converse.h conv-header.h conv-config.h \
  debug-conv++.h register.h simd.h ckmessage.h pup.h CkMarshall.decl.h \
  envelope.h charm.h middle.h cklists.h objid.h charm.h converse.h pup.h \
  sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
- ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
- cklocation.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
- CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
- cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
- waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
- CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
- pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
- NullLB.decl.h BaseLB.decl.h
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
+ CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
+ ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
+ waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
+ ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
+ ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h
 
 LBSimulation.o: LBSimulation.C LBSimulation.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -914,12 +895,12 @@ LBSimulation.o: LBSimulation.C LBSimulation.h charm++.h charm.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -940,16 +921,15 @@ LButil.o: LButil.C elements.h converse.h conv-header.h conv-config.h \
  debug-conv++.h register.h simd.h ckmessage.h pup.h CkMarshall.decl.h \
  envelope.h charm.h middle.h cklists.h objid.h charm.h converse.h pup.h \
  sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
- ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
- cklocation.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
- CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
- cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
- waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
- CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
- pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
- NullLB.decl.h BaseLB.decl.h
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
+ CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
+ ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
+ waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
+ ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
+ ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h
 
 MetaBalancer.o: MetaBalancer.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -963,12 +943,11 @@ MetaBalancer.o: MetaBalancer.C converse.h conv-header.h conv-config.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h \
- CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
- cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
- waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
  CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
  pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
  NullLB.decl.h BaseLB.decl.h RandomForestModel.h MetaBalancer.decl.h \
@@ -986,17 +965,16 @@ MetisLB.o: MetisLB.C MetisLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h MetisLB.decl.h ckgraph.h \
- MetisLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ MetisLB.decl.h ckgraph.h MetisLB.def.h
 
 NborBaseLB.o: NborBaseLB.C BaseLB.h LBDatabase.h lbdb.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1010,17 +988,16 @@ NborBaseLB.o: NborBaseLB.C BaseLB.h LBDatabase.h lbdb.h converse.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- NborBaseLB.h NborBaseLB.decl.h NeighborLBMsg.h topology.h \
- NborBaseLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h NborBaseLB.h NborBaseLB.decl.h \
+ NeighborLBMsg.h topology.h NborBaseLB.def.h
 
 NeighborCommLB.o: NeighborCommLB.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1035,17 +1012,16 @@ NeighborCommLB.o: NeighborCommLB.C elements.h converse.h conv-header.h \
  ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- NborBaseLB.decl.h NeighborLBMsg.h topology.h NeighborCommLB.decl.h \
- NeighborCommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h NborBaseLB.decl.h NeighborLBMsg.h topology.h \
+ NeighborCommLB.decl.h NeighborCommLB.def.h
 
 NeighborLB.o: NeighborLB.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1060,17 +1036,16 @@ NeighborLB.o: NeighborLB.C elements.h converse.h conv-header.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- NborBaseLB.decl.h NeighborLBMsg.h topology.h NeighborLB.decl.h \
- NeighborLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h NborBaseLB.decl.h NeighborLBMsg.h topology.h \
+ NeighborLB.decl.h NeighborLB.def.h
 
 NodeLevelLB.o: NodeLevelLB.C NodeLevelLB.h CentralLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1085,18 +1060,17 @@ NodeLevelLB.o: NodeLevelLB.C NodeLevelLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h HybridBaseLB.h \
- HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h topology.h \
- NodeLevelLB.decl.h NodeLevelLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ HybridBaseLB.h HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h \
+ topology.h NodeLevelLB.decl.h NodeLevelLB.def.h
 
 NullLB.o: NullLB.C NullLB.h BaseLB.h LBDatabase.h lbdb.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1110,18 +1084,18 @@ NullLB.o: NullLB.C NullLB.h BaseLB.h LBDatabase.h lbdb.h converse.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
NullLB.decl.h ck.h qd.h ckcallback.h envelope.h init.h charm++.h \
register.h stats.h ckfutures.h ../include/TopoManager.h \
- ../include/topomanager_config.h ../include/converse.h NullLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h NullLB.decl.h ck.h qd.h ckcallback.h \
envelope.h init.h charm++.h register.h stats.h ckfutures.h \
../include/TopoManager.h ../include/topomanager_config.h \
+ ../include/converse.h NullLB.def.h
 
 OrbLB.o: OrbLB.C OrbLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1135,16 +1109,16 @@ OrbLB.o: OrbLB.C OrbLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h OrbLB.decl.h OrbLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ OrbLB.decl.h OrbLB.def.h
 
 PhasebyArrayLB.o: PhasebyArrayLB.C PhasebyArrayLB.h CentralLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1159,17 +1133,16 @@ PhasebyArrayLB.o: PhasebyArrayLB.C PhasebyArrayLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h PhasebyArrayLB.decl.h \
- PhasebyArrayLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ PhasebyArrayLB.decl.h PhasebyArrayLB.def.h
 
 RandCentLB.o: RandCentLB.C RandCentLB.h CentralLB.h BaseLB.h LBDatabase.h \
  lbdb.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -1184,17 +1157,16 @@ RandCentLB.o: RandCentLB.C RandCentLB.h CentralLB.h BaseLB.h LBDatabase.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h RandCentLB.decl.h \
- RandCentLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ RandCentLB.decl.h RandCentLB.def.h
 
 RecBipartLB.o: RecBipartLB.C RecBipartLB.h CentralLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1209,17 +1181,16 @@ RecBipartLB.o: RecBipartLB.C RecBipartLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h RecBipartLB.decl.h ckgraph.h \
- RecBipartLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ RecBipartLB.decl.h ckgraph.h RecBipartLB.def.h
 
 RefineCommLB.o: RefineCommLB.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1234,17 +1205,17 @@ RefineCommLB.o: RefineCommLB.C elements.h converse.h conv-header.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h RefinerComm.h Refiner.h \
- RefineLB.h RefineLB.decl.h RefineCommLB.decl.h RefineCommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
RefinerComm.h Refiner.h RefineLB.h RefineLB.decl.h RefineCommLB.decl.h \
+ RefineCommLB.def.h
 
 RefineKLB.o: RefineKLB.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1259,17 +1230,16 @@ RefineKLB.o: RefineKLB.C elements.h converse.h conv-header.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h RefinerApprox.h Refiner.h \
- RefineKLB.decl.h RefineKLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ RefinerApprox.h Refiner.h RefineKLB.decl.h RefineKLB.def.h
 
 RefineLB.o: RefineLB.C elements.h converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -1284,17 +1254,16 @@ RefineLB.o: RefineLB.C elements.h converse.h conv-header.h conv-config.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h Refiner.h RefineLB.decl.h \
- RefineLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ Refiner.h RefineLB.decl.h RefineLB.def.h
 
 RefineSwapLB.o: RefineSwapLB.C RefineSwapLB.h CentralLB.h BaseLB.h \
  LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1309,17 +1278,16 @@ RefineSwapLB.o: RefineSwapLB.C RefineSwapLB.h CentralLB.h BaseLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h RefineSwapLB.decl.h ckgraph.h \
- RefineSwapLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ RefineSwapLB.decl.h ckgraph.h RefineSwapLB.def.h
 
 RefineTopoLB.o: RefineTopoLB.C RefineTopoLB.decl.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1332,19 +1300,18 @@ RefineTopoLB.o: RefineTopoLB.C RefineTopoLB.decl.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h TopoLB.decl.h CentralLB.decl.h \
- CentralLBMsg.h RefineTopoLB.h CentralLB.h BaseLB.h manager.h TopoLB.h \
- topology.h RefineTopoLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ TopoLB.decl.h CentralLB.decl.h CentralLBMsg.h RefineTopoLB.h CentralLB.h \
+ BaseLB.h manager.h TopoLB.h topology.h RefineTopoLB.def.h
 
 Refiner.o: Refiner.C Refiner.h elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1359,16 +1326,15 @@ Refiner.o: Refiner.C Refiner.h elements.h converse.h conv-header.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h
 
 RefinerApprox.o: RefinerApprox.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1383,16 +1349,16 @@ RefinerApprox.o: RefinerApprox.C elements.h converse.h conv-header.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h Refiner.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ Refiner.h
 
 RefinerComm.o: RefinerComm.C elements.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1402,21 +1368,21 @@ RefinerComm.o: RefinerComm.C elements.h converse.h conv-header.h \
  charm.h pup.h middle.h middle-conv.h ckset.h cklists.h pup_stl.h \
  conv-config.h ckheap.h RefinerComm.h CentralLB.h BaseLB.h LBDatabase.h \
  LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h charm++.h ckbitvector.h ckstream.h init.h charm-api.h \
- ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
- pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
- charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
CentralLB.decl.h CentralLBMsg.h manager.h Refiner.h
+ LBDatabase.decl.h charm++.h ckbitvector.h ckstream.h init.h charm-api.h \
+ ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
+ pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
+ charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
+ ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ Refiner.h
 
 RefinerTemp.o: RefinerTemp.C RefinerTemp.h elements.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1431,16 +1397,15 @@ RefinerTemp.o: RefinerTemp.C RefinerTemp.h elements.h converse.h \
  ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h
 
 RotateLB.o: RotateLB.C RotateLB.decl.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1453,18 +1418,18 @@ RotateLB.o: RotateLB.C RotateLB.decl.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
RotateLB.h CentralLB.h BaseLB.h manager.h RotateLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h RotateLB.h CentralLB.h BaseLB.h \
+ manager.h RotateLB.def.h
 
 ScotchLB.o: ScotchLB.C ScotchLB.h CentralLB.h BaseLB.h LBDatabase.h \
  lbdb.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -1479,17 +1444,16 @@ ScotchLB.o: ScotchLB.C ScotchLB.h CentralLB.h BaseLB.h LBDatabase.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h ScotchLB.decl.h ckgraph.h \
- ScotchLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ ScotchLB.decl.h ckgraph.h ScotchLB.def.h
 
 TeamLB.o: TeamLB.C TeamLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1503,17 +1467,16 @@ TeamLB.o: TeamLB.C TeamLB.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h TeamLB.decl.h ckgraph.h \
- TeamLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ TeamLB.decl.h ckgraph.h TeamLB.def.h
 
 TempAwareCommLB.o: TempAwareCommLB.C TempAwareCommLB.h CentralLB.h \
  BaseLB.h LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1528,17 +1491,16 @@ TempAwareCommLB.o: TempAwareCommLB.C TempAwareCommLB.h CentralLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h TempAwareCommLB.decl.h \
- ckgraphTemp.h ckgraph.h TempAwareCommLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ TempAwareCommLB.decl.h ckgraphTemp.h ckgraph.h TempAwareCommLB.def.h
 
 TempAwareGreedyLB.o: TempAwareGreedyLB.C TempAwareGreedyLB.h CentralLB.h \
  BaseLB.h LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1553,17 +1515,16 @@ TempAwareGreedyLB.o: TempAwareGreedyLB.C TempAwareGreedyLB.h CentralLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h TempAwareGreedyLB.decl.h \
- ckgraph.h TempAwareGreedyLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ TempAwareGreedyLB.decl.h ckgraph.h TempAwareGreedyLB.def.h
 
 TempAwareRefineLB.o: TempAwareRefineLB.C TempAwareRefineLB.h CentralLB.h \
  BaseLB.h LBDatabase.h lbdb.h converse.h conv-header.h conv-config.h \
@@ -1578,18 +1539,17 @@ TempAwareRefineLB.o: TempAwareRefineLB.C TempAwareRefineLB.h CentralLB.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h manager.h RefinerTemp.h elements.h \
- ckset.h ckheap.h TempAwareRefineLB.decl.h ckgraph.h \
- TempAwareRefineLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h manager.h \
+ RefinerTemp.h elements.h ckset.h ckheap.h TempAwareRefineLB.decl.h \
+ ckgraph.h TempAwareRefineLB.def.h
 
 TopoCentLB.o: TopoCentLB.C TopoCentLB.decl.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1602,18 +1562,18 @@ TopoCentLB.o: TopoCentLB.C TopoCentLB.decl.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
TopoCentLB.h CentralLB.h BaseLB.h manager.h topology.h TopoCentLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h TopoCentLB.h CentralLB.h BaseLB.h \
+ manager.h topology.h TopoCentLB.def.h
 
 TopoLB.o: TopoLB.C TopoLB.decl.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1626,18 +1586,18 @@ TopoLB.o: TopoLB.C TopoLB.decl.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CentralLB.decl.h CentralLBMsg.h \
TopoLB.h CentralLB.h BaseLB.h manager.h topology.h TopoLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
CentralLB.decl.h CentralLBMsg.h TopoLB.h CentralLB.h BaseLB.h manager.h \
+ topology.h TopoLB.def.h
 
 TreeMatchLB.o: TreeMatchLB.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1649,20 +1609,20 @@ TreeMatchLB.o: TreeMatchLB.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h tm_tree.h tm_mapping.h TreeMatchLB.h \
- CentralLB.h BaseLB.h LBDatabase.h CentralLB.decl.h charm++.h envelope.h \
sdag.h BaseLB.decl.h LBDatabase.decl.h CentralLBMsg.h pup_stl.h \
manager.h TreeMatchLB.decl.h ckgraph.h TreeMatchLB.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
tm_tree.h tm_mapping.h TreeMatchLB.h CentralLB.h BaseLB.h LBDatabase.h \
+ CentralLB.decl.h charm++.h envelope.h sdag.h BaseLB.decl.h \
LBDatabase.decl.h CentralLBMsg.h pup_stl.h manager.h TreeMatchLB.decl.h \
+ ckgraph.h TreeMatchLB.def.h
 
 WSLB.o: WSLB.C elements.h converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -1677,16 +1637,15 @@ WSLB.o: WSLB.C elements.h converse.h conv-header.h conv-config.h \
  CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h charm.h \
  converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- WSLB.decl.h topology.h WSLB.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h WSLB.decl.h topology.h WSLB.def.h
 
 arrayRedistributor.o: arrayRedistributor.C charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -1698,12 +1657,12 @@ arrayRedistributor.o: arrayRedistributor.C charm++.h charm.h converse.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -1774,17 +1733,16 @@ bigsim_proc.o: bigsim_proc.C blue.h converse.h conv-header.h \
  ckbitvector.h ckstream.h init.h charm-api.h ckhashtable.h debug-charm.h \
  debug-conv++.h register.h simd.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckarrayindex.h cksection.h ckcallback.h ckrdmawrapper.h ckrdma.h \
- ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckevacuation.h trace.h trace-bluegene.h pathHistory.h \
- PathHistory.decl.h ckcallback-ccs.h conv-ccs.h CkCallback.decl.h \
- bigsim_ooc.h bigsim_debug.h
+ ckarrayindex.h cksection.h ckcallback.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h conv-ccs.h \
+ CkCallback.decl.h bigsim_ooc.h bigsim_debug.h
 
 bigsim_read.o: bigsim_read.C blue.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1836,17 +1794,16 @@ blue_stub.o: blue_stub.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 ccs-builtins.o: ccs-builtins.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -1869,7 +1826,7 @@ charmProjections.o: charmProjections.C converse.h conv-header.h \
  pup_stl.h conv-config.h objid.h init.h charm-api.h charm++.h middle.h \
  cklists.h ckbitvector.h ckstream.h debug-charm.h debug-conv++.h \
  register.h simd.h ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h \
- envelope.h debug-charm.h ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h \
+ envelope.h debug-charm.h ckrdma.h ckobjQ.h ckreduction.h \
  CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
  ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
  LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
@@ -1893,11 +1850,11 @@ ck.o: ck.C ck.h qd.h ckcallback.h cksection.h charm.h converse.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -1917,17 +1874,17 @@ ckIgetControl.o: ckIgetControl.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ckIgetControl.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ ckIgetControl.h
 
 ckarray.o: ckarray.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -1938,19 +1895,18 @@ ckarray.o: ckarray.C charm++.h charm.h converse.h conv-header.h \
  ckstream.h init.h charm-api.h ckhashtable.h debug-charm.h debug-conv++.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
- envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkArray.def.h
 
 ckarrayoptions.o: ckarrayoptions.C charm++.h charm.h converse.h \
@@ -1964,18 +1920,17 @@ ckarrayoptions.o: ckarrayoptions.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h
 
 ckbitvector.o: ckbitvector.C charm++.h charm.h converse.h conv-header.h \
@@ -1988,17 +1943,16 @@ ckbitvector.o: ckbitvector.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 ckcallback.o: ckcallback.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -2010,17 +1964,17 @@ ckcallback.o: ckcallback.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h CkCallback.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ CkCallback.def.h
 
 ckcausalmlog.o: ckcausalmlog.C charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -2033,11 +1987,11 @@ ckcausalmlog.o: ckcausalmlog.C charm.h converse.h conv-header.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2057,18 +2011,17 @@ ckcheckpoint.o: ckcheckpoint.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkCheckpoint.decl.h CkCheckpoint.def.h \
  CkCheckpointStatus.def.h
 
@@ -2091,18 +2044,17 @@ ckfutures.o: ckfutures.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkFutures.def.h
 
 ckgraph.o: ckgraph.C ckgraph.h BaseLB.h LBDatabase.h lbdb.h converse.h \
@@ -2117,15 +2069,15 @@ ckgraph.o: ckgraph.C ckgraph.h BaseLB.h LBDatabase.h lbdb.h converse.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h
 
 ckgraphTemp.o: ckgraphTemp.C ckgraphTemp.h BaseLB.h LBDatabase.h lbdb.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -2139,15 +2091,15 @@ ckgraphTemp.o: ckgraphTemp.C ckgraphTemp.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h ckgraph.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h ckgraph.h
 
 ckhashtable.o: ckhashtable.C ckhashtable.h pup.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -2180,18 +2132,17 @@ cklocation.o: cklocation.C hilbert.h partitioning_strategies.h charm++.h \
  simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkLocation.def.h
 
 ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
@@ -2205,18 +2156,17 @@ ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkMemCheckpoint.def.h
 
 ckmessagelogging.o: ckmessagelogging.C charm.h converse.h conv-header.h \
@@ -2230,11 +2180,11 @@ ckmessagelogging.o: ckmessagelogging.C charm.h converse.h conv-header.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2254,18 +2204,17 @@ ckmulticast.o: ckmulticast.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h spanningTree.h XArraySectionReducer.h \
- CkMulticast.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ spanningTree.h XArraySectionReducer.h CkMulticast.def.h
 
 ckobjQ.o: ckobjQ.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -2277,17 +2226,16 @@ ckobjQ.o: ckobjQ.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 ckobjid.o: ckobjid.C charm.h converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -2300,11 +2248,11 @@ ckobjid.o: ckobjid.C charm.h converse.h conv-header.h conv-config.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2322,18 +2270,17 @@ ckrdma.o: ckrdma.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h
 
 ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-header.h \
@@ -2346,18 +2293,17 @@ ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h ck.h \
+ qd.h stats.h ../include/TopoManager.h ../include/topomanager_config.h \
  ../include/converse.h CkReduction.def.h
 
 ckregex.o: ckregex.C ckregex.h
@@ -2380,17 +2326,17 @@ cktiming.o: cktiming.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h cktiming.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ cktiming.h
 
 cldb.o: cldb.c queueing.h conv-config.h conv-autoconfig.h conv-common.h \
  conv-mach-common.h conv-mach.h conv-mach-opt.h lrts-common.h cldb.h \
@@ -2406,7 +2352,9 @@ cmipool.o: cmipool.c cmipool.h converse.h conv-header.h conv-config.h \
 
 cmirdmautils.o: cmirdmautils.c cmirdmautils.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
- conv-mach-opt.h lrts-common.h
+ conv-mach-opt.h lrts-common.h converse.h cmiqueue.h pup_c.h lrtslock.h \
+ queueing.h conv-cpm.h conv-cpath.h conv-qd.h conv-random.h conv-lists.h \
+ conv-trace.h persistent.h conv-rdma.h debug-conv.h
 
 cmitls.o: cmitls.c converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -2427,12 +2375,12 @@ controlPoints.o: controlPoints.C charm++.h charm.h converse.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2515,12 +2463,12 @@ cp_effects.o: cp_effects.C cp_effects.h charm++.h charm.h converse.h \
  ckhashtable.h debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
  pup.h CkMarshall.decl.h charm++.h envelope.h charm.h middle.h cklists.h \
  objid.h sdag.h pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h \
- cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ cksection.h ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2572,17 +2520,16 @@ custom_partitioner.o: custom_partitioner.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 debug-charm.o: debug-charm.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -2594,12 +2541,12 @@ debug-charm.o: debug-charm.C converse.h conv-header.h conv-config.h \
  middle.h middle-conv.h ck.h qd.h ckcallback.h cksection.h ckarrayindex.h \
  objid.h envelope.h pup.h charm.h middle.h cklists.h objid.h init.h \
  charm-api.h charm++.h ckbitvector.h ckstream.h simd.h ckmessage.h \
- CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2633,7 +2580,7 @@ debug-message.o: debug-message.C ck.h qd.h ckcallback.h cksection.h \
  pup_stl.h conv-config.h objid.h init.h charm-api.h charm++.h middle.h \
  cklists.h ckbitvector.h ckstream.h debug-charm.h debug-conv++.h \
  register.h simd.h ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h \
- envelope.h debug-charm.h ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h \
+ envelope.h debug-charm.h ckrdma.h ckobjQ.h ckreduction.h \
  CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
  ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
  LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
@@ -2692,17 +2639,16 @@ init.o: init.C ckcheckpoint.h pup.h converse.h conv-header.h \
  ckbitvector.h ckstream.h init.h charm-api.h ckhashtable.h debug-charm.h \
  debug-conv++.h register.h simd.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckarrayindex.h cksection.h ckcallback.h ckrdmawrapper.h ckrdma.h \
- ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckevacuation.h trace.h conv-trace.h trace-bluegene.h \
- pathHistory.h PathHistory.decl.h ckcallback-ccs.h conv-ccs.h \
- CkCallback.decl.h ck.h qd.h stats.h ../include/TopoManager.h \
+ ckarrayindex.h cksection.h ckcallback.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckevacuation.h trace.h conv-trace.h \
+ trace-bluegene.h pathHistory.h PathHistory.decl.h ckcallback-ccs.h \
+ conv-ccs.h CkCallback.decl.h ck.h qd.h stats.h ../include/TopoManager.h \
  ../include/topomanager_config.h ../include/converse.h \
  CkCheckpoint.decl.h spanningTree.h
 
@@ -2725,8 +2671,8 @@ lbdb.o: lbdb.C converse.h conv-header.h conv-config.h conv-autoconfig.h \
  debug-conv++.h register.h simd.h ckmessage.h pup.h CkMarshall.decl.h \
  envelope.h charm.h middle.h cklists.h objid.h charm.h pup.h sdag.h \
  pup_stl.h envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h \
- ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h \
- ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
+ ckcallback.h conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h \
+ ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
  CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
  RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
@@ -2746,17 +2692,16 @@ lbdbf.o: lbdbf.C charm++.h charm.h converse.h conv-header.h conv-config.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 lz4.o: lz4.c lz4.h
 
@@ -2778,11 +2723,11 @@ main.o: main.C ck.h qd.h ckcallback.h cksection.h charm.h converse.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2802,16 +2747,15 @@ manager.o: manager.C manager.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h NullLB.decl.h BaseLB.decl.h \
- CentralLB.decl.h CentralLBMsg.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h MetaBalancer.h RandomForestModel.h \
+ MetaBalancer.decl.h CkLocation.decl.h ckarrayoptions.h ckmulticast.h \
+ CkMulticast.decl.h cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h
 
 mem-arena.o: mem-arena.c converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -2854,17 +2798,16 @@ modifyScheduler.o: modifyScheduler.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 mpi-interoperate.o: mpi-interoperate.C mpi-interoperate.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -2877,12 +2820,12 @@ mpi-interoperate.o: mpi-interoperate.C mpi-interoperate.h converse.h \
  middle-conv.h converse.h cklists.h pup.h pup_stl.h conv-config.h objid.h \
  init.h charm-api.h charm++.h middle.h cklists.h ckbitvector.h ckstream.h \
  debug-charm.h debug-conv++.h register.h simd.h ckmessage.h \
- CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h ckrdma.h \
+ ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2901,19 +2844,18 @@ mpi-mainmodule.o: mpi-mainmodule.C mpi_main.decl.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h mpi-interoperate.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
- ../include/converse.h mpi_main.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ mpi-interoperate.h ck.h qd.h stats.h ../include/TopoManager.h \
+ ../include/topomanager_config.h ../include/converse.h mpi_main.def.h
 
 msgalloc.o: msgalloc.C ck.h qd.h ckcallback.h cksection.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -2926,11 +2868,11 @@ msgalloc.o: msgalloc.C ck.h qd.h ckcallback.h cksection.h charm.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -2967,19 +2909,19 @@ pathHistory.o: pathHistory.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h PathHistory.decl.h LBDatabase.h \
pathHistory.h register.h trace-projections.h trace.h trace-common.h \
ckhashtable.h pup.h PathHistory.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
PathHistory.decl.h LBDatabase.h pathHistory.h register.h \
trace-projections.h trace.h trace-common.h ckhashtable.h pup.h \
+ PathHistory.def.h
 
 persist-comm.o: persist-comm.c converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -2997,21 +2939,20 @@ picsautoperf.o: picsautoperf.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ../include/TopoManager.h \
- ../include/topomanager_config.h ../include/converse.h picsdefs.h \
- picsdefscpp.h TraceAutoPerf.decl.h picsautoperf.h picstreenode.h \
- picsdecisiontree.h picsautoperfAPI.h trace-common.h trace-projections.h \
- trace-perf.h TraceAutoPerf.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ ../include/TopoManager.h ../include/topomanager_config.h \
+ ../include/converse.h picsdefs.h picsdefscpp.h TraceAutoPerf.decl.h \
+ picsautoperf.h picstreenode.h picsdecisiontree.h picsautoperfAPI.h \
+ trace-common.h trace-projections.h trace-perf.h TraceAutoPerf.def.h
 
 picsautoperfAPI.o: picsautoperfAPI.C picsdefs.h picsdefscpp.h \
  picsautoperf.h picstreenode.h charm++.h charm.h converse.h conv-header.h \
@@ -3024,19 +2965,18 @@ picsautoperfAPI.o: picsautoperfAPI.C picsdefs.h picsdefscpp.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h picsdecisiontree.h picsautoperfAPI.h \
- trace-common.h TraceAutoPerf.decl.h trace-projections.h \
- picsautoperfAPIC.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ picsdecisiontree.h picsautoperfAPI.h trace-common.h TraceAutoPerf.decl.h \
+ trace-projections.h picsautoperfAPIC.h
 
 picsdecisiontree.o: picsdecisiontree.C picsdefs.h picsdecisiontree.h \
  picstreenode.h charm++.h charm.h converse.h conv-header.h conv-config.h \
@@ -3049,17 +2989,17 @@ picsdecisiontree.o: picsdecisiontree.C picsdefs.h picsdecisiontree.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h picsdefscpp.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ picsdefscpp.h
 
 picstreenode.o: picstreenode.C picstreenode.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3072,17 +3012,17 @@ picstreenode.o: picstreenode.C picstreenode.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h picsdefs.h picsdefscpp.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ picsdefs.h picsdefscpp.h
 
 pup_c.o: pup_c.C pup.h converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -3143,11 +3083,11 @@ qd.o: qd.C ck.h qd.h ckcallback.h cksection.h charm.h converse.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -3192,11 +3132,11 @@ register.o: register.C ck.h qd.h ckcallback.h cksection.h charm.h \
  conv-config.h objid.h init.h charm-api.h charm++.h middle.h cklists.h \
  ckbitvector.h ckstream.h debug-charm.h debug-conv++.h register.h simd.h \
  ckmessage.h CkMarshall.decl.h sdag.h pup_stl.h envelope.h debug-charm.h \
- ckrdmawrapper.h ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h \
ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
- LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
+ ckrdma.h ckobjQ.h ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+ LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
+ RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
  ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
  ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
  waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
@@ -3235,18 +3175,18 @@ spanningTree.o: spanningTree.C spanningTree.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h ../include/TopoManager.h \
- ../include/topomanager_config.h ../include/converse.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
../include/TopoManager.h ../include/topomanager_config.h \
+ ../include/converse.h
 
 tm_bucket.o: tm_bucket.c tm_tree.h tm_bucket.h tm_timings.h
 
@@ -3276,19 +3216,19 @@ trace-bluegene.o: trace-bluegene.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-common.h blue.h blue-conv.h \
- blue_impl.h ckliststring.h blue_types.h bigsim_timing.h bigsim_logs.h \
blue_defs.h bigsim_network.h memory-isomalloc.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
trace-common.h blue.h blue-conv.h blue_impl.h ckliststring.h \
+ blue_types.h bigsim_timing.h bigsim_logs.h blue_defs.h bigsim_network.h \
+ memory-isomalloc.h
 
 trace-common.o: trace-common.C charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -3312,19 +3252,18 @@ trace-controlPoints.o: trace-controlPoints.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h trace-controlPoints.h trace-common.h \
- trace-controlPointsBOC.h TraceControlPoints.decl.h \
- TraceControlPoints.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ trace-controlPoints.h trace-common.h trace-controlPointsBOC.h \
+ TraceControlPoints.decl.h TraceControlPoints.def.h
 
 trace-converse.o: trace-converse.c conv-trace.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -3343,17 +3282,16 @@ trace-counter.o: trace-counter.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h
 
 trace-memory.o: trace-memory.C trace-memory.h charm++.h charm.h \
  converse.h conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3366,17 +3304,17 @@ trace-memory.o: trace-memory.C trace-memory.h charm++.h charm.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-common.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ trace-common.h
 
 trace-perf.o: trace-perf.C trace-perf.h charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3389,21 +3327,20 @@ trace-perf.o: trace-perf.C trace-perf.h charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h ../include/TopoManager.h \
- ../include/topomanager_config.h ../include/converse.h trace-common.h \
- picsdefs.h picsdefscpp.h picsautoperf.h picstreenode.h \
- picsdecisiontree.h picsautoperfAPI.h TraceAutoPerf.decl.h \
- trace-projections.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ ../include/TopoManager.h ../include/topomanager_config.h \
+ ../include/converse.h trace-common.h picsdefs.h picsdefscpp.h \
+ picsautoperf.h picstreenode.h picsdecisiontree.h picsautoperfAPI.h \
+ TraceAutoPerf.decl.h trace-projections.h
 
 trace-projections.o: trace-projections.C charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3416,18 +3353,18 @@ trace-projections.o: trace-projections.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-projections.h trace-common.h \
trace-projectionsBOC.h TraceProjections.decl.h ../include/TopoManager.h \
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
trace-projections.h trace-common.h trace-projectionsBOC.h \
+ TraceProjections.decl.h ../include/TopoManager.h \
  ../include/topomanager_config.h ../include/converse.h \
  TraceProjections.def.h
 
@@ -3442,22 +3379,21 @@ trace-projector.o: trace-projector.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h trace-projector.h ck.h qd.h stats.h \
- ../include/TopoManager.h ../include/topomanager_config.h \
- ../include/converse.h allEvents.h charmEvents.h charmProjections.h \
- traceCoreAPI.h converseEvents.h converseProjections.h machineEvents.h \
- machineProjections.h traceCore.h threadEvents.h traceCoreCommon.h \
- trace-common.h trace-projections.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ trace-projector.h ck.h qd.h stats.h ../include/TopoManager.h \
+ ../include/topomanager_config.h ../include/converse.h allEvents.h \
+ charmEvents.h charmProjections.h traceCoreAPI.h converseEvents.h \
+ converseProjections.h machineEvents.h machineProjections.h traceCore.h \
+ threadEvents.h traceCoreCommon.h trace-common.h trace-projections.h
 
 trace-simple.o: trace-simple.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -3469,18 +3405,18 @@ trace-simple.o: trace-simple.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-simple.h trace-common.h \
trace-simpleBOC.h TraceSimple.decl.h TraceSimple.def.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
trace-simple.h trace-common.h trace-simpleBOC.h TraceSimple.decl.h \
+ TraceSimple.def.h
 
 trace-summary.o: trace-summary.C charm++.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3493,18 +3429,18 @@ trace-summary.o: trace-summary.C charm++.h charm.h converse.h \
  pup.h CkMarshall.decl.h envelope.h charm.h middle.h cklists.h objid.h \
  charm.h converse.h pup.h sdag.h pup_stl.h envelope.h debug-charm.h \
  ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-summary.h trace-common.h \
trace-summaryBOC.h TraceSummary.decl.h TraceSummary.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
trace-summary.h trace-common.h trace-summaryBOC.h TraceSummary.decl.h \
+ TraceSummary.def.h
 
 trace-utilization.o: trace-utilization.C trace-utilization.h charm++.h \
  charm.h converse.h conv-header.h conv-config.h conv-autoconfig.h \
@@ -3517,18 +3453,17 @@ trace-utilization.o: trace-utilization.C trace-utilization.h charm++.h \
  simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h middle.h \
  cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h envelope.h \
  debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h conv-ccs.h \
- sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
- waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
- ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h trace-common.h \
- TraceUtilization.decl.h TraceUtilization.def.h
+ sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
+ CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
+ pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ trace-common.h TraceUtilization.decl.h TraceUtilization.def.h
 
 traceCore.o: traceCore.C converse.h conv-header.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach-common.h conv-mach.h \
@@ -3556,17 +3491,17 @@ tracec.o: tracec.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
ckcallback-ccs.h CkCallback.decl.h trace-common.h tracec.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ trace-common.h tracec.h
 
 tracef.o: tracef.C charm++.h charm.h converse.h conv-header.h \
  conv-config.h conv-autoconfig.h conv-common.h conv-mach-common.h \
@@ -3578,17 +3513,17 @@ tracef.o: tracef.C charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h charm++.h \
  envelope.h charm.h middle.h cklists.h objid.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h waitqd.h \
waitqd.decl.h ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h \
ckevacuation.h trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
- ckcallback-ccs.h CkCallback.decl.h charm-api.h
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
waitqd.h waitqd.decl.h ckcheckpoint.h ckcallback.h \
CkCheckpointStatus.decl.h ckevacuation.h trace.h trace-bluegene.h \
pathHistory.h PathHistory.decl.h ckcallback-ccs.h CkCallback.decl.h \
+ charm-api.h
 
 treeTest.o: treeTest.C RandomForestModel.h charm.h converse.h \
  conv-header.h conv-config.h conv-autoconfig.h conv-common.h \
@@ -3607,14 +3542,13 @@ waitqd.o: waitqd.C waitqd.h charm++.h charm.h converse.h conv-header.h \
  register.h simd.h ckmessage.h pup.h CkMarshall.decl.h envelope.h charm.h \
  middle.h cklists.h objid.h charm.h converse.h pup.h sdag.h pup_stl.h \
  envelope.h debug-charm.h ckarrayindex.h objid.h cksection.h ckcallback.h \
- conv-ccs.h sockRoutines.h ccs-server.h ckrdmawrapper.h ckrdma.h ckobjQ.h \
- ckreduction.h CkReduction.decl.h ckmemcheckpoint.h \
- CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
- lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
- LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h MetaBalancer.h \
- RandomForestModel.h MetaBalancer.decl.h CkLocation.decl.h \
- ckarrayoptions.h ckmulticast.h CkMulticast.decl.h cklocrec.h \
- ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
+ conv-ccs.h sockRoutines.h ccs-server.h ckrdma.h ckobjQ.h ckreduction.h \
+ CkReduction.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h \
+ ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h \
+ LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h \
+ BaseLB.decl.h MetaBalancer.h RandomForestModel.h MetaBalancer.decl.h \
+ CkLocation.decl.h ckarrayoptions.h ckmulticast.h CkMulticast.decl.h \
+ cklocrec.h ckmigratable.h CkArray.decl.h ckfutures.h CkFutures.decl.h \
  ckcheckpoint.h ckcallback.h CkCheckpointStatus.decl.h ckevacuation.h \
  trace.h trace-bluegene.h pathHistory.h PathHistory.decl.h \
  ckcallback-ccs.h CkCallback.decl.h waitqd.decl.h waitqd.def.h
index ee058b7c2296ae0e312ded018e5d8ac924996268..c7dfdecdc0d10c9d8f74cd99819a27a406599aaa 100644 (file)
@@ -251,7 +251,7 @@ CKHEADERS=ck.h ckstream.h objid.h envelope.h init.h qd.h charm.h charm++.h \
          ckcallback.h CkCallback.decl.h ckcallback-ccs.h       \
          cksection.h ckmessage.h cklocrec.h ckmigratable.h \
          ckarrayindex.h ckarrayoptions.h ckarray.h cklocation.h ckmulticast.h ckreduction.h \
-         ckcheckpoint.h ckmemcheckpoint.h ckevacuation.h ckrdmawrapper.h ckrdma.h\
+         ckcheckpoint.h ckmemcheckpoint.h ckevacuation.h ckrdma.h\
           ckmessagelogging.h ckcausalmlog.h ckobjid.h\
          ckobjQ.h readonly.h charm++_type_traits.h \
           $(UTILHEADERS) \
index 1c331c1d92f00334cedaf239f01ff579dcfe3ef2..0dac1812cf782c64397fbd2121e0aae3dc36a36d 100644 (file)
@@ -1,4 +1,5 @@
 #include "cmirdmautils.h"
+#include "converse.h" // for CmiAbort usage to avoid undeclared warning
 
 #include <stdio.h>
 #include <string.h>
@@ -34,20 +35,39 @@ void setNcpyOpInfo(
     const void *destRef,
     NcpyOperationInfo *ncpyOpInfo) {
 
+  char *base = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
+
+  ncpyOpInfo->srcLayerInfo = NULL;
+  ncpyOpInfo->destLayerInfo = NULL;
+  ncpyOpInfo->srcAck = NULL;
+  ncpyOpInfo->destAck = NULL;
+
   // memcpy srcLayerInfo
-  memcpy((char *)ncpyOpInfo + sizeof(NcpyOperationInfo), srcLayerInfo, srcLayerSize);
-  ncpyOpInfo->srcLayerInfo = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
+  if(srcLayerInfo != NULL && srcLayerSize != 0) {
+    memcpy(base, srcLayerInfo, srcLayerSize);
+    ncpyOpInfo->srcLayerInfo = base;
+    base = base + srcLayerSize;
+  }
+
   // memcpy srcAckInfo
-  memcpy(ncpyOpInfo->srcLayerInfo + srcLayerSize, srcAck, srcAckSize);
-  ncpyOpInfo->srcAck = ncpyOpInfo->srcLayerInfo + srcLayerSize;
+  if(srcAck != NULL && srcAckSize != 0) {
+    memcpy(base, srcAck, srcAckSize);
+    ncpyOpInfo->srcAck = base;
+    base = base + srcAckSize;
+  }
 
   // memcpy destLayerInfo
-  memcpy(ncpyOpInfo->srcAck + srcAckSize, destLayerInfo, destLayerSize);
-  ncpyOpInfo->destLayerInfo = ncpyOpInfo->srcAck + srcAckSize;
+  if(srcLayerInfo != NULL && destLayerSize != 0) {
+    memcpy(base, destLayerInfo, destLayerSize);
+    ncpyOpInfo->destLayerInfo = base;
+    base = base + destLayerSize;
+  }
 
   // memcpy destAck Info
-  memcpy(ncpyOpInfo->destLayerInfo + destLayerSize, destAck, destAckSize);
-  ncpyOpInfo->destAck = ncpyOpInfo->destLayerInfo + destLayerSize;
+  if(destAck != NULL && destAckSize != 0) {
+    memcpy(base, destAck, destAckSize);
+    ncpyOpInfo->destAck = base;
+  }
 
   ncpyOpInfo->srcPtr = srcPtr;
   ncpyOpInfo->srcPe = srcPe;
@@ -67,19 +87,46 @@ void setNcpyOpInfo(
   ncpyOpInfo->destMode = destMode;
   ncpyOpInfo->isDestRegistered = isDestRegistered;
 
-  ncpyOpInfo->ackMode = 0;
-  ncpyOpInfo->freeMe  = 1;
+  ncpyOpInfo->opMode  = CMK_DIRECT_API; // default operation mode is CMK_DIRECT_API
+  ncpyOpInfo->ackMode = CMK_SRC_DEST_ACK; // default ack mode is CMK_SRC_DEST_ACK
+  ncpyOpInfo->freeMe  = CMK_FREE_NCPYOPINFO; // default ack mode is CMK_FREE_NCPYOPINFO
 
   ncpyOpInfo->ncpyOpInfoSize = sizeof(NcpyOperationInfo) + srcLayerSize + destLayerSize + srcAckSize + destAckSize;
 }
 
 
 void resetNcpyOpInfoPointers(NcpyOperationInfo *ncpyOpInfo) {
-  ncpyOpInfo->srcLayerInfo = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
 
-  ncpyOpInfo->srcAck = (char *)(ncpyOpInfo->srcLayerInfo) + ncpyOpInfo->srcLayerSize;
+  char *base = (char *)ncpyOpInfo + sizeof(NcpyOperationInfo);
+
+  if(ncpyOpInfo->srcLayerInfo) {
+    ncpyOpInfo->srcLayerInfo = base;
+    base = base + ncpyOpInfo->srcLayerSize;
+  }
 
-  ncpyOpInfo->destLayerInfo = (char *)(ncpyOpInfo->srcAck) + ncpyOpInfo->srcAckSize;
+  if(ncpyOpInfo->srcAck) {
+    ncpyOpInfo->srcAck = base;
+    base = base + ncpyOpInfo->srcAckSize;
+  }
+
+  if(ncpyOpInfo->destLayerInfo) {
+    ncpyOpInfo->destLayerInfo = base;
+    base = base + ncpyOpInfo->destLayerSize;
+  }
+
+  if(ncpyOpInfo->destAck) {
+    ncpyOpInfo->destAck = base;
+  }
+
+}
 
-  ncpyOpInfo->destAck = (char *)(ncpyOpInfo->destLayerInfo) + ncpyOpInfo->destLayerSize;
+void setReverseModeForNcpyOpInfo(NcpyOperationInfo *ncpyOpInfo) {
+  switch(ncpyOpInfo->opMode) {
+    case CMK_EM_API          : ncpyOpInfo->opMode = CMK_EM_API_REVERSE;
+                               break;
+    case CMK_DIRECT_API      : // Do nothing
+                               break;
+    default                  : CmiAbort("Unknown opcode");
+                               break;
+  }
 }
index 9de55e527c45e9b70cc6ba1200fc09c94d6c87cd..74343d39ee243e894db2d06e2c3407a8e1a40ec8 100644 (file)
@@ -34,11 +34,20 @@ typedef struct ncpystruct{
   unsigned short int isDestRegistered;
   const void *destRef;
 
+  unsigned char opMode; // CMK_DIRECT_API for p2p direct api
+                        // CMK_DIRECT_API_REVERSE for p2p direct api with inverse operation
+                        // CMK_EM_API for p2p entry method api
+                        // CMK_EM_API_REVERSE for p2p entry method api with inverse operation
+
   // Variables used for ack handling
-  int ackMode; // 0 for call both src and dest acks
-               // 1 for call just src ack
-               // 2 for call just dest ack
-  int freeMe; // 1 for free, 0 for do not free
+  unsigned char ackMode; // CMK_SRC_DEST_ACK for call both src and dest acks
+                         // CMK_SRC_ACK for call just src ack
+                         // CMK_DEST_ACK for call just dest ack
+
+  unsigned char freeMe;  // CMK_FREE_NCPYOPINFO in order to free NcpyOperationInfo
+                         // CMK_DONT_FREE_NCPYOPINFO in order to not free NcpyOperationInfo
+
+  void *refPtr;
 
   int ncpyOpInfoSize;
 
@@ -74,4 +83,6 @@ void setNcpyOpInfo(
   NcpyOperationInfo *ncpyOpInfo);
 
 void resetNcpyOpInfoPointers(NcpyOperationInfo *ncpyOpInfo);
+
+void setReverseModeForNcpyOpInfo(NcpyOperationInfo *ncpyOpInfo);
 #endif
index 6b94329de4b5606fa51d5b787a0c29241478387c..f0033e5c3b6cf00f820e93740f771a62cc65188c 100644 (file)
@@ -45,11 +45,11 @@ void generateLocalWrapper(XStr& decls, XStr& defs, int isVoid, XStr& signature,
             if (var.isFirstRdma)
               defs << "  genClosure->getP" << i++ << "() = " << numRdmaParams << ";\n";
             defs << "  genClosure->getP" << i << "() = "
-                 << "rdmawrapper_" << var.name << ";\n";
+                 << "ncpyBuffer_" << var.name << ";\n";
             defs << "#else\n";
             defs << "  genClosure->getP" << i << "() = "
                  << "(" << var.type << "*)"
-                 << "rdmawrapper_" << var.name << ".ptr"
+                 << "ncpyBuffer_" << var.name << ".ptr"
                  << ";\n";
             defs << "#endif\n";
           } else
@@ -81,7 +81,7 @@ void CEntry::generateCode(XStr& decls, XStr& defs) {
       if (sv->byConst) signature << "const ";
 
       if (sv->isRdma)
-        signature << "CkRdmaWrapper ";
+        signature << "CkNcpyBuffer ";
       else
         signature << sv->type << " ";
       if (sv->arrayLength != 0)
@@ -94,7 +94,7 @@ void CEntry::generateCode(XStr& decls, XStr& defs) {
       }
       if (sv->name != 0) {
         if (sv->isRdma) {
-          signature << "rdmawrapper_" << sv->name;
+          signature << "ncpyBuffer_" << sv->name;
         } else {
           signature << sv->name;
         }
index 08e60daea60878b1b9b8c5d6a50410eb878a8b70..09f55c1f3120ee3f4c898ad2c5631d4d20dbce87 100644 (file)
@@ -303,13 +303,13 @@ int SdagConstruct::unravelClosuresBegin(XStr& defs, bool child) {
                  << "getP" << i++ << "();\n";
           }
           indentBy(defs, cur + 2);
-          defs << "CkRdmaWrapper "
-               << "& rdmawrapper_" << var.name << " = ";
+          defs << "CkNcpyBuffer "
+               << "& ncpyBuffer_" << var.name << " = ";
           defs << "gen" << cur << "->"
                << "getP" << i << "();\n";
           indentBy(defs, cur + 2);
           defs << var.type << "* " << var.name << " = "
-               << "(" << var.type << "*) (rdmawrapper_" << var.name << ".ptr);\n";
+               << "(" << var.type << "*) (ncpyBuffer_" << var.name << ".ptr);\n";
           defs << "#else\n";
           indentBy(defs, cur + 2);
           defs << var.type << "*"
index 84b635b08e6c99fbdb67ed5a91ac1688ad2ddce4..c44ac6672058a18cdf152925ab17f1dc48b2927d 100644 (file)
@@ -383,9 +383,6 @@ void Entry::genChareDefs(XStr& str) {
       opts << ",0";
       if (isSkipscheduler()) opts << "+CK_MSG_EXPEDITED";
       if (isInline()) opts << "+CK_MSG_INLINE";
-      if (param->hasRdma()) {
-        opts << "+CK_MSG_RDMA";
-      }
       str << "    CkSendMsg(" << params << opts << ");\n";
       str << "  }\n";
     }
@@ -621,9 +618,6 @@ void Entry::genArrayDefs(XStr& str) {
         if (isInline()) opts << "+CK_MSG_INLINE";
         if (!isIget()) {
           if (container->isForElement() || container->isForSection()) {
-            if (param->hasRdma()) {
-              opts << "+CK_MSG_RDMA";
-            }
             str << "  ckSend(impl_amsg, " << epIdx() << opts << ");\n";
           } else
             str << "  ckBroadcast(impl_amsg, " << epIdx() << opts << ");\n";
@@ -923,9 +917,6 @@ void Entry::genGroupDefs(XStr& str) {
               << parampg << ");\n";
         }
         str << "  } else {\n";
-        if (param->hasRdma()) {
-          opts << "+CK_MSG_RDMA";
-        }
         str << "    CkSendMsg" << node << "Branch"
             << "(" << parampg << opts << ");\n";
         str << "  }\n";
@@ -2140,8 +2131,8 @@ void Entry::genClosure(XStr& decls, bool isDef) {
           structure << "      "
                     << "int num_rdma_fields;\n";
         structure << "      "
-                  << "CkRdmaWrapper "
-                  << "rdmawrapper_" << sv->name << ";\n";
+                  << "CkNcpyBuffer "
+                  << "ncpyBuffer_" << sv->name << ";\n";
         structure << "#else\n";
         structure << "      " << sv->type << " "
                   << "*" << sv->name << ";\n";
@@ -2155,10 +2146,10 @@ void Entry::genClosure(XStr& decls, bool isDef) {
                  << " num_rdma_fields;}\n";
         }
         getter << "      "
-               << "CkRdmaWrapper "
+               << "CkNcpyBuffer "
                << "& "
                << "getP" << i << "() { return "
-               << "rdmawrapper_" << sv->name << ";}\n";
+               << "ncpyBuffer_" << sv->name << ";}\n";
         getter << "#else\n";
         getter << sv->type << " "
                << "*";
@@ -2179,12 +2170,12 @@ void Entry::genClosure(XStr& decls, bool isDef) {
          * to adjust the pointer back to the actual buffer that exists within the message.
          */
         toPup << "        if (__p.isPacking()) {\n";
-        toPup << "          rdmawrapper_" << sv->name << ".ptr = ";
-        toPup << "(void *)((char *)(rdmawrapper_" << sv->name << ".ptr) - impl_buf);\n";
+        toPup << "          ncpyBuffer_" << sv->name << ".ptr = ";
+        toPup << "(void *)((char *)(ncpyBuffer_" << sv->name << ".ptr) - impl_buf);\n";
         toPup << "        }\n";
         toPup << "        "
               << "__p | "
-              << "rdmawrapper_" << sv->name << ";\n";
+              << "ncpyBuffer_" << sv->name << ";\n";
         toPup << "#endif\n";
       } else {
         structure << "      ";
@@ -2217,9 +2208,9 @@ void Entry::genClosure(XStr& decls, bool isDef) {
       if (sv->isMessage()) {
         isMessage = true;
         if (sv->isRdma()) {
-          structure << "CkRdmaWrapper"
+          structure << "CkNcpyBuffer"
                     << " "
-                    << "rdmawrapper_" << sv->name << ";\n";
+                    << "ncpyBuffer_" << sv->name << ";\n";
         } else {
           structure << sv->type << " " << sv->name << ";\n";
         }
index 109a94679fbe2c0e997138048ecc638bcd2b9732..758ccc4abc06b66e00296b3e8b268f63d1e6130b 100644 (file)
@@ -108,7 +108,7 @@ void ParamList::printTypes(XStr& str, int withDefaultValues, int useConst) {
 
 int Parameter::print(XStr& str, int withDefaultValues, int useConst, int fwdNum) {
   if (isRdma()) {
-    str << "CkRdmaWrapper rdmawrapper_" << name;
+    str << "CkNcpyBuffer ncpyBuffer_" << name;
   } else if (arrLen != NULL) {  // Passing arrays by const pointer-reference
     if (useConst) str << "const ";
     str << type << " *";
@@ -304,8 +304,9 @@ void Parameter::marshallRdmaParameters(XStr& str, bool genRdma) {
     Type* dt = type->deref();  // Type, without &
     if (genRdma) {
       str << "  impl_num_rdma_fields++;\n";
-      str << "  rdmawrapper_" << name << ".cnt=sizeof(" << dt << ")*(" << arrLen
+      str << "  ncpyBuffer_" << name << ".cnt=sizeof(" << dt << ")*(" << arrLen
           << ");\n";
+      str << "  ncpyBuffer_" << name << ".registerMem()" << ";\n";
     } else {
       marshallArraySizes(str, dt);
     }
@@ -315,7 +316,7 @@ void Parameter::marshallRdmaParameters(XStr& str, bool genRdma) {
 void Parameter::pupRdma(XStr& str, bool genRdma) {
   if (isRdma()) {
     if (genRdma)
-      str << "    implP|rdmawrapper_" << name << ";\n";
+      str << "    implP|ncpyBuffer_" << name << ";\n";
     else
       pupArray(str);
   }
@@ -341,14 +342,13 @@ void Parameter::pup(XStr& str) {
 void Parameter::marshallRdmaArrayData(XStr& str) {
   if (isRdma()) {
     str << "  memcpy(impl_buf+impl_off_" << name << ","
-        << "rdmawrapper_" << name << ".ptr"
+        << "ncpyBuffer_" << name << ".ptr"
         << ",impl_cnt_" << name << ");\n";
-    str << "  rdmawrapper_" << name << ".callback->send("
-        << "sizeof(void *)"
+    str << "  ncpyBuffer_" << name << ".cb.send("
+        << "sizeof(CkNcpyBuffer)"
         << ","
-        << "&rdmawrapper_" << name << ".ptr"
+        << "&ncpyBuffer_" << name
         << ");\n";
-    str << "  free(rdmawrapper_" << name << ".callback);\n";
   }
 }
 
@@ -512,8 +512,8 @@ void Parameter::beginUnmarshallRdma(XStr& str,
   Type* dt = type->deref();                          // Type, without &
   if (isRdma()) {
     if (genRdma) {
-      str << "  CkRdmaWrapper rdmawrapper_" << name << ";\n";
-      str << "  implP|rdmawrapper_" << name << ";\n";
+      str << "  CkNcpyBuffer ncpyBuffer_" << name << ";\n";
+      str << "  implP|ncpyBuffer_" << name << ";\n";
     } else
       beginUnmarshallArray(str);
   }
@@ -537,7 +537,7 @@ void Parameter::beginUnmarshallSDAGCallRdma(XStr& str, bool genRdma) {
       if (isFirstRdma()) {
         str << "  implP|genClosure->num_rdma_fields;\n";
       }
-      str << "  implP|genClosure->rdmawrapper_" << name << ";\n";
+      str << "  implP|genClosure->ncpyBuffer_" << name << ";\n";
     } else {
       beginUnmarshallArray(str);
     }
@@ -633,8 +633,8 @@ void Parameter::unmarshallRegArrayDataSDAG(XStr& str) {
 
 void Parameter::adjustUnmarshalledRdmaPtrsSDAG(XStr& str) {
   if (isRdma()) {
-    str << "  rdmawrapper_" << name << ".ptr = ";
-    str << "(void *)(impl_buf + (size_t)(rdmawrapper_" << name << ".ptr));\n";
+    str << "  ncpyBuffer_" << name << ".ptr = ";
+    str << "(void *)(impl_buf + (size_t)(ncpyBuffer_" << name << ".ptr));\n";
   }
 }
 
@@ -698,7 +698,7 @@ void ParamList::unmarshall(XStr& str, bool isInline, bool isFirst)  // Pass-by-v
     if (param->isRdma()) {
       str << "\n#if CMK_ONESIDED_IMPL\n";
       str << "(" << (param->getType())->deref() << " *)";
-      str << "rdmawrapper_" << param->getName() << ".ptr";
+      str << "ncpyBuffer_" << param->getName() << ".ptr";
       str << "\n#else\n";
       str << param->getName();
       str << "\n#endif\n";
@@ -771,7 +771,7 @@ void Parameter::pupAllValues(XStr& str) {
            "  implDestP.synchronize(PUP::sync_end_array);\n";
   } else if (isRdma()) {
     str << "#if CMK_ONESIDED_IMPL\n";
-    str << "  implDestP|rdmawrapper_" << name << ";\n";
+    str << "  implDestP|ncpyBuffer_" << name << ";\n";
     str << "#else\n";
     str << "  implDestP.synchronize(PUP::sync_begin_array);\n"
            "  { for (int impl_i=0;impl_i*(sizeof(*"