fix for migration
[charm.git] / src / ck-core / ckmemcheckpoint.C
index 510fbd0c6a10ddc035c01720c76c568010676c74..c4dc028c8dd537c689026fdc02c8c05a34bcbdc7 100644 (file)
@@ -436,6 +436,9 @@ void CkMemCheckPT::pup(PUP::er& p)
 void CkMemCheckPT::getIter(){
   localDecided = true;
   localMaxIter = maxIter+1;
 void CkMemCheckPT::getIter(){
   localDecided = true;
   localMaxIter = maxIter+1;
+  if(CkMyPe()==0){
+    CkPrintf("local max iter is %d\n",localMaxIter);
+  }
   contribute(sizeof(int),&localMaxIter,CkReduction::max_int,CkCallback(CkReductionTarget(CkMemCheckPT,recvMaxIter),thisProxy));
   int elemCount = CkCountChkpSyncElements();
   if(CkMyPe()==0)
   contribute(sizeof(int),&localMaxIter,CkReduction::max_int,CkCallback(CkReductionTarget(CkMemCheckPT,recvMaxIter),thisProxy));
   int elemCount = CkCountChkpSyncElements();
   if(CkMyPe()==0)
@@ -455,12 +458,15 @@ void CkMemCheckPT::recvIter(int iter){
 
 void CkMemCheckPT::recvMaxIter(int iter){
   localDecided = false;
 
 void CkMemCheckPT::recvMaxIter(int iter){
   localDecided = false;
+  if(CkMyPe()==0)
+    CkPrintf("checkpoint iteration is %d\n",iter);
   CKLOCMGR_LOOP(mgr->recvChkpIter(iter););
 }
 
 void CkMemCheckPT::reachChkpIter(){
   recvIterCount++;
   elemCount = CkCountChkpSyncElements();
   CKLOCMGR_LOOP(mgr->recvChkpIter(iter););
 }
 
 void CkMemCheckPT::reachChkpIter(){
   recvIterCount++;
   elemCount = CkCountChkpSyncElements();
+  //CkPrintf("[%d] received %d local %d\n",CkMyPe(),recvIterCount, elemCount);
   if(recvIterCount == elemCount){
     recvIterCount = 0;
     contribute(CkCallback(CkReductionTarget(CkMemCheckPT,startChkp),thisProxy[0]));
   if(recvIterCount == elemCount){
     recvIterCount = 0;
     contribute(CkCallback(CkReductionTarget(CkMemCheckPT,startChkp),thisProxy[0]));
@@ -655,7 +661,7 @@ void pupAllElements(PUP::er &p){
   p | numElements;
   if(!p.isUnpacking()){
     MemElementPacker packer(p);
   p | numElements;
   if(!p.isUnpacking()){
     MemElementPacker packer(p);
-    CKLOCMGR_LOOP(mgr->iterate(packer););
+    CKLOCMGR_LOOP(mgr->iterateLocal(packer););
     packer.writeCheckpoint();
   }
 #endif
     packer.writeCheckpoint();
   }
 #endif
@@ -797,7 +803,7 @@ void CkMemCheckPT::startCheckpoint(){
           CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
           CmiRemoteSyncSendAndFree(CkMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
           if(CkMyPe() == CpvAccess(_remoteCrashedNode)){
           CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
           CmiRemoteSyncSendAndFree(CkMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
           if(CkMyPe() == CpvAccess(_remoteCrashedNode)){
-            CkPrintf("[%d] sendProcdata\n",CkMyPe());
+            CkPrintf("[%d] sendProcdata at %lf\n",CkMyPe(),CmiWallTimer());
           }
         }
         //send the array checkpoint data
           }
         }
         //send the array checkpoint data
@@ -808,7 +814,7 @@ void CkMemCheckPT::startCheckpoint(){
         CmiSetHandler(env,recoverRemoteArrayDataHandlerIdx);
         CmiRemoteSyncSendAndFree(CkMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
         if(CkMyPe() == CpvAccess(_remoteCrashedNode))
         CmiSetHandler(env,recoverRemoteArrayDataHandlerIdx);
         CmiRemoteSyncSendAndFree(CkMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
         if(CkMyPe() == CpvAccess(_remoteCrashedNode))
-          CkPrintf("[%d] sendArraydata\n",CkMyPe());
+          CkPrintf("[%d] sendArraydata at %lf\n",CkMyPe(),CmiWallTimer());
       }else{
         if(CkMyPe() == CpvAccess(_remoteCrashedNode)){
           int pointer = CpvAccess(curPointer);
       }else{
         if(CkMyPe() == CpvAccess(_remoteCrashedNode)){
           int pointer = CpvAccess(curPointer);
@@ -850,20 +856,20 @@ void CkMemCheckPT::doneRComparison(int ret){
     }else{
       CmiPrintf("[%d][%d] Local checkpoint finished in %f seconds at %lf, waiting for replica ... \n", CmiMyPartition(),CkMyPe(), CmiWallTimer()-startTime,CmiWallTimer());
     }
     }else{
       CmiPrintf("[%d][%d] Local checkpoint finished in %f seconds at %lf, waiting for replica ... \n", CmiMyPartition(),CkMyPe(), CmiWallTimer()-startTime,CmiWallTimer());
     }
-    if(notifyReplica == 0){
-      //notify the replica am done
-      char *msg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes+sizeof(int));
-      *(int *)(msg+CmiMsgHeaderSizeBytes) = ret;
-      CmiSetHandler(msg,replicaChkpDoneHandlerIdx);
-      CmiRemoteSyncSendAndFree(0,CmiMyPartition()^1,CmiMsgHeaderSizeBytes+sizeof(int),(char *)msg);
-      notifyReplica = 1;
-    }
- // }
- /* else{
+  //}
+  /*else{
     CkPrintf("[%d][%d] going to RollBack %d at %lf checkpoint in %lf\n", CmiMyPartition(),CkMyPe(),ret,CmiWallTimer(), CmiWallTimer()-startTime);
     startTime = CmiWallTimer();
     thisProxy.RollBack();
   }*/
     CkPrintf("[%d][%d] going to RollBack %d at %lf checkpoint in %lf\n", CmiMyPartition(),CkMyPe(),ret,CmiWallTimer(), CmiWallTimer()-startTime);
     startTime = CmiWallTimer();
     thisProxy.RollBack();
   }*/
+  if(notifyReplica == 0){
+    //notify the replica am done
+    char *msg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes+sizeof(int));
+    *(int *)(msg+CmiMsgHeaderSizeBytes) = ret;
+    CmiSetHandler(msg,replicaChkpDoneHandlerIdx);
+    CmiRemoteSyncSendAndFree(0,CmiMyPartition()^1,CmiMsgHeaderSizeBytes+sizeof(int),(char *)msg);
+    notifyReplica = 1;
+  }
 }
 
 void CkMemCheckPT::doneBothComparison(){
 }
 
 void CkMemCheckPT::doneBothComparison(){
@@ -890,7 +896,7 @@ void CkMemCheckPT::doneBothComparison(){
 void CkMemCheckPT::RollBack(){
   //restore group data
   checkpointed = 0;
 void CkMemCheckPT::RollBack(){
   //restore group data
   checkpointed = 0;
-  CkMemCheckPT::inRestarting = 1;
+  inRestarting = 1;
   int pointer = CpvAccess(curPointer)^1;//use the previous one
   CkCheckPTMessage* chkpMsg = CpvAccess(chkpBuf)[pointer];
   PUP::fromMem p(chkpMsg->packData);   
   int pointer = CpvAccess(curPointer)^1;//use the previous one
   CkCheckPTMessage* chkpMsg = CpvAccess(chkpBuf)[pointer];
   PUP::fromMem p(chkpMsg->packData);   
@@ -1182,22 +1188,22 @@ void CkMemCheckPT::RollBack(){
     void CkMemCheckPT::restart(int diePe)
     {
 #if CMK_MEM_CHECKPOINT
     void CkMemCheckPT::restart(int diePe)
     {
 #if CMK_MEM_CHECKPOINT
+      thisFailedPe = diePe;
       double curTime = CmiWallTimer();
       double curTime = CmiWallTimer();
-      if (CkMyPe() == diePe){
+      if (CkMyPe() == thisFailedPe){
         restartT = CmiWallTimer();
         CkPrintf("[%d] Process data restored in %f seconds\n", CkMyPe(), curTime - startTime);
       }
       stage = (char*)"resetLB";
       startTime = curTime;
         restartT = CmiWallTimer();
         CkPrintf("[%d] Process data restored in %f seconds\n", CkMyPe(), curTime - startTime);
       }
       stage = (char*)"resetLB";
       startTime = curTime;
-      if (CkMyPe() == diePe)
+      if (CkMyPe() == thisFailedPe)
         CkPrintf("[%d] CkMemCheckPT ----- restart.\n",CkMyPe());
 
         CkPrintf("[%d] CkMemCheckPT ----- restart.\n",CkMyPe());
 
-#if CK_NO_PROC_POOL
-      failed(diePe);   // add into the list of failed pes
-#endif
-      thisFailedPe = diePe;
+//#if CK_NO_PROC_POOL
+//      failed(diePe); // add into the list of failed pes
+//#endif
 
 
-      if (CkMyPe() == diePe) CmiAssert(ckTable.length() == 0);
+//      if (CkMyPe() == diePe) CmiAssert(ckTable.length() == 0);
 
       inRestarting = 1;
 
 
       inRestarting = 1;
 
@@ -1247,7 +1253,8 @@ void CkMemCheckPT::RollBack(){
     void CkMemCheckPT::resetReductionMgr()
     {
       if (CkMyPe() == thisFailedPe) 
     void CkMemCheckPT::resetReductionMgr()
     {
       if (CkMyPe() == thisFailedPe) 
-        CkPrintf("[%d] CkMemCheckPT ----- resetReductionMgr\n",CkMyPe());
+        CkPrintf("[%d] CkMemCheckPT ----- resetReductionMgr at %lf\n",CkMyPe(),CmiWallTimer());
+      stage = (char *)"resetReductionMgr";
       int numGroups = CkpvAccess(_groupIDTable)->size();
       for(int i=0;i<numGroups;i++) {
         CkGroupID gID = (*CkpvAccess(_groupIDTable))[i];
       int numGroups = CkpvAccess(_groupIDTable)->size();
       for(int i=0;i<numGroups;i++) {
         CkGroupID gID = (*CkpvAccess(_groupIDTable))[i];
@@ -1260,8 +1267,11 @@ void CkMemCheckPT::RollBack(){
       if(CmiNumPartition()==1){
         barrier(CkCallback(CkIndex_CkMemCheckPT::recoverBuddies(), thisProxy));
       }
       if(CmiNumPartition()==1){
         barrier(CkCallback(CkIndex_CkMemCheckPT::recoverBuddies(), thisProxy));
       }
-      else
+      else{
+       if (CkMyPe() == thisFailedPe) 
+         CkPrintf("[%d] CkMemCheckPT ----- resetReductionMgr ends at %lf\n",CkMyPe(),CmiWallTimer());
         barrier(CkCallback(CkReductionTarget(CkMemCheckPT, recoverArrayElements), thisProxy));
         barrier(CkCallback(CkReductionTarget(CkMemCheckPT, recoverArrayElements), thisProxy));
+      }
     }
 
     // recover the lost buddies
     }
 
     // recover the lost buddies
@@ -1350,10 +1360,8 @@ void CkMemCheckPT::RollBack(){
     {
       double curTime = CmiWallTimer();
       int len = ckTable.length();
     {
       double curTime = CmiWallTimer();
       int len = ckTable.length();
-      //CkPrintf("[%d] CkMemCheckPT ----- %s len: %d in %f seconds \n",CkMyPe(), stage, len, curTime-startTime);
+      CkPrintf("[%d] CkMemCheckPT ----- %s len: %d in %f seconds \n",CkMyPe(), stage, len, curTime-startTime);
       stage = (char *)"recoverArrayElements";
       stage = (char *)"recoverArrayElements";
-      if (CkMyPe() == thisFailedPe)
-        CmiPrintf("[%d] CkMemCheckPT ----- %s starts at %f \n",CkMyPe(), stage, curTime);
       startTime = curTime;
       int flag = 0;
       // recover all array elements
       startTime = curTime;
       int flag = 0;
       // recover all array elements
@@ -1423,6 +1431,8 @@ void CkMemCheckPT::RollBack(){
       delete [] gmap;
 #endif
       DEBUGF("[%d] recoverArrayElements restore %d objects\n", CkMyPe(), count);
       delete [] gmap;
 #endif
       DEBUGF("[%d] recoverArrayElements restore %d objects\n", CkMyPe(), count);
+      CkPrintf("[%d] recoverArrayElements restore %d objects\n", CkMyPe(), count);
+      //if (CkMyPe() == thisFailedPe)
 
       CKLOCMGR_LOOP(mgr->doneInserting(););
 
 
       CKLOCMGR_LOOP(mgr->doneInserting(););
 
@@ -1498,7 +1508,7 @@ void CkMemCheckPT::RollBack(){
       }
       CKLOCMGR_LOOP(mgr->resumeFromChkp(););
       inRestarting = 0;
       }
       CKLOCMGR_LOOP(mgr->resumeFromChkp(););
       inRestarting = 0;
-
+      maxIter = -1;
 #if CMK_CONVERSE_MPI   
       if(CmiNumPartition()!=1){
         CpvAccess(recvdProcChkp) = 0;
 #if CMK_CONVERSE_MPI   
       if(CmiNumPartition()!=1){
         CpvAccess(recvdProcChkp) = 0;
@@ -1512,6 +1522,7 @@ void CkMemCheckPT::RollBack(){
         }
       }
       if (CmiMyPe() == BuddyPE(thisFailedPe)) {
         }
       }
       if (CmiMyPe() == BuddyPE(thisFailedPe)) {
+        lastPingTime = CmiWallTimer();
         CcdCallOnCondition(CcdPERIODIC_1s,(CcdVoidFn)pingCheckHandler,NULL);
       }
 #endif
         CcdCallOnCondition(CcdPERIODIC_1s,(CcdVoidFn)pingCheckHandler,NULL);
       }
 #endif
@@ -1532,10 +1543,19 @@ void CkMemCheckPT::RollBack(){
     void CkMemCheckPT::recoverFromSoftFailure()
     {
       inRestarting = 0;
     void CkMemCheckPT::recoverFromSoftFailure()
     {
       inRestarting = 0;
+      maxIter = -1;
       CpvAccess(recvdRemote) = 0;
       CpvAccess(recvdLocal) = 0;
       CpvAccess(localChkpDone) = 0;
       CpvAccess(remoteChkpDone) = 0;
       CpvAccess(recvdRemote) = 0;
       CpvAccess(recvdLocal) = 0;
       CpvAccess(localChkpDone) = 0;
       CpvAccess(remoteChkpDone) = 0;
+      CpvAccess(remoteReady) = 0;
+      CpvAccess(localReady) = 0;
+      inCheckpointing = 0;
+      notifyReplica = 0;
+      CpvAccess(remoteStarted) = 0;
+      CpvAccess(localStarted) = 0;
+      CpvAccess(_remoteCrashedNode) = -1;
+      CkMemCheckPT::replicaAlive = 1;
       inCheckpointing = 0;
       notifyReplica = 0;
       if(CkMyPe() == 0){
       inCheckpointing = 0;
       notifyReplica = 0;
       if(CkMyPe() == 0){
@@ -1620,7 +1640,7 @@ void CkMemCheckPT::RollBack(){
 #if CMK_MEM_CHECKPOINT
 #if CMK_USE_BARRIER
       if(CkMyPe()!=_diePE){
 #if CMK_MEM_CHECKPOINT
 #if CMK_USE_BARRIER
       if(CkMyPe()!=_diePE){
-        printf("restar begin on %d\n",CkMyPe());
+        printf("restart begin on %d at %lf\n",CkMyPe(),CmiWallTimer());
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
         CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
         CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
@@ -1633,7 +1653,7 @@ void CkMemCheckPT::RollBack(){
       CmiAssert(CkMyPe() == _diePE);
       count ++;
       if (count == CkNumPes()||(CpvAccess(resilience)==1&&count==1)) {
       CmiAssert(CkMyPe() == _diePE);
       count ++;
       if (count == CkNumPes()||(CpvAccess(resilience)==1&&count==1)) {
-        printf("restart begin on %d\n",CkMyPe());
+        printf("restart begin on %d at %lf\n",CkMyPe(),CmiWallTimer());
         CkRestartCheckPointCallback(NULL, NULL);
         count = 0;
       }
         CkRestartCheckPointCallback(NULL, NULL);
         count = 0;
       }
@@ -1815,7 +1835,9 @@ void CkMemCheckPT::RollBack(){
         CmiPrintf("replicaAlive %d\n",CkMemCheckPT::replicaAlive);
         fflush(stdout);
       }
         CmiPrintf("replicaAlive %d\n",CkMemCheckPT::replicaAlive);
         fflush(stdout);
       }
-      find_spare_mpirank(diePe,CmiMyPartition()^1);
+      if(CpvAccess(resilience)!=1||CkMyPe()!=diePe){
+        find_spare_mpirank(diePe,CmiMyPartition()^1);
+      }
       //broadcast to my partition to get local max iter
       if(CpvAccess(resilience)!=1){
         CkMemCheckPT::replicaAlive = 0;
       //broadcast to my partition to get local max iter
       if(CpvAccess(resilience)!=1){
         CkMemCheckPT::replicaAlive = 0;
@@ -1854,7 +1876,17 @@ void CkMemCheckPT::RollBack(){
         _diePE = CpvAccess(_crashedNode);
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
         _diePE = CpvAccess(_crashedNode);
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
-        CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+        //CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+#if CMK_USE_BARRIER
+      //CmiPrintf("before reduce\n");  
+       if(CpvAccess(resilience)==1){
+         CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+       }else
+         CmiReduce(restartmsg,CmiMsgHeaderSizeBytes,doNothingMsg);
+      //CmiPrintf("after reduce\n");   
+#else
+       CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+#endif 
       }
     }
 
       }
     }
 
@@ -1876,7 +1908,17 @@ void CkMemCheckPT::RollBack(){
         //CmiPrintf("[%d] send to die pe %d\n",CkMyPe(),_diePE);
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
         //CmiPrintf("[%d] send to die pe %d\n",CkMyPe(),_diePE);
         char *restartmsg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes);
         CmiSetHandler(restartmsg, restartBeginHandlerIdx);
-        CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+        //CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+#if CMK_USE_BARRIER
+      //CmiPrintf("before reduce\n");  
+       if(CpvAccess(resilience)==1){
+         CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+       }else
+         CmiReduce(restartmsg,CmiMsgHeaderSizeBytes,doNothingMsg);
+      //CmiPrintf("after reduce\n");   
+#else
+       CmiSyncSendAndFree(_diePE, CmiMsgHeaderSizeBytes, (char *)restartmsg);
+#endif 
       }
     }
 
       }
     }
 
@@ -1904,10 +1946,10 @@ void CkMemCheckPT::RollBack(){
     }
     
     static void askRecoverDataHandler(char * msg){
     }
     
     static void askRecoverDataHandler(char * msg){
+      if(CmiMyPe() == CpvAccess(_remoteCrashedNode))
+       CmiPrintf("[%d][%d] receive replica phase change at %lf\n",CmiMyPartition(),CmiMyPe(),CmiWallTimer());
       if(CpvAccess(resilience)!=1){
         CpvAccess(remoteReady)=1;
       if(CpvAccess(resilience)!=1){
         CpvAccess(remoteReady)=1;
-        if(CmiMyPe() == CpvAccess(_remoteCrashedNode))
-          CmiPrintf("[%d][%d] receive replica phase change at %lf\n",CmiMyPartition(),CmiMyPe(),CmiWallTimer());
         if(CpvAccess(localReady)==1){
           if(CmiMyPe() == CpvAccess(_remoteCrashedNode))
           {    
         if(CpvAccess(localReady)==1){
           if(CmiMyPe() == CpvAccess(_remoteCrashedNode))
           {    
@@ -1915,7 +1957,7 @@ void CkMemCheckPT::RollBack(){
             CkPackMessage(&env);
             CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
             CmiRemoteSyncSendAndFree(CmiMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
             CkPackMessage(&env);
             CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
             CmiRemoteSyncSendAndFree(CmiMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
-            CmiPrintf("[%d] sendProcdata after request\n",CmiMyPe());
+            CmiPrintf("[%d] sendProcdata after request at \n",CmiMyPe(),CmiWallTimer());
           }
           //send the array checkpoint data
           envelope * env = (envelope *)(UsrToEnv(CpvAccess(recoverArrayBuf)));
           }
           //send the array checkpoint data
           envelope * env = (envelope *)(UsrToEnv(CpvAccess(recoverArrayBuf)));
@@ -1926,6 +1968,7 @@ void CkMemCheckPT::RollBack(){
             CmiPrintf("[%d] sendArraydata after request\n",CmiMyPe());
         }
       }else{
             CmiPrintf("[%d] sendArraydata after request\n",CmiMyPe());
         }
       }else{
+        find_spare_mpirank(CkMyPe(),CmiMyPartition()^1);
         int pointer = CpvAccess(curPointer)^1;
         CkCheckPTMessage * procMsg = (CkCheckPTMessage *)CkCopyMsg((void **)&CpvAccess(localProcChkpBuf)[pointer]);
         procMsg->pointer = pointer;
         int pointer = CpvAccess(curPointer)^1;
         CkCheckPTMessage * procMsg = (CkCheckPTMessage *)CkCopyMsg((void **)&CpvAccess(localProcChkpBuf)[pointer]);
         procMsg->pointer = pointer;
@@ -1933,7 +1976,7 @@ void CkMemCheckPT::RollBack(){
         CkPackMessage(&env);
         CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
         CmiRemoteSyncSendAndFree(CmiMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
         CkPackMessage(&env);
         CmiSetHandler(env,recoverRemoteProcDataHandlerIdx);
         CmiRemoteSyncSendAndFree(CmiMyPe(),CmiMyPartition()^1,env->getTotalsize(),(char *)env);
-        CmiPrintf("[%d] sendProcdata after request\n",CmiMyPe());
+        CmiPrintf("[%d] sendProcdata after request at %lf\n",CmiMyPe(),CmiWallTimer());
         
         CkCheckPTMessage * arrayMsg = (CkCheckPTMessage *)CkCopyMsg((void **)&CpvAccess(chkpBuf)[pointer]);
         arrayMsg->pointer = pointer;
         
         CkCheckPTMessage * arrayMsg = (CkCheckPTMessage *)CkCopyMsg((void **)&CpvAccess(chkpBuf)[pointer]);
         arrayMsg->pointer = pointer;