Checked in the code for the evacuation scheme of fault tolerance.
authorSayantan Chakravorty <sayantan_chak@yahoo.com>
Sat, 25 Jun 2005 20:25:09 +0000 (20:25 +0000)
committerSayantan Chakravorty <sayantan_chak@yahoo.com>
Sat, 25 Jun 2005 20:25:09 +0000 (20:25 +0000)
Most of the code is in ckevacuation.C and .h. However there are
minor changes all over to make the scheme work.

Also fixed the reduction tree. Removed the ckreductionmgr tree.
Now there is just one tree the node based one.
Allowed ampi to do anytime migration with isomalloc.
Each thread now has a token and it is the token that gets enqueued in the scheduler and
not the thread itself. This is useful for doing any time migration and was a long standing
to do

35 files changed:
src/arch/net/machine.c
src/ck-core/charm++.h
src/ck-core/ck.C
src/ck-core/ckarray.C
src/ck-core/ckarray.h
src/ck-core/ckarrayreductionmgr.C
src/ck-core/ckarrayreductionmgr.ci
src/ck-core/ckarrayreductionmgr.h
src/ck-core/ckcheckpoint.C
src/ck-core/ckcheckpoint.h
src/ck-core/cklocation.C
src/ck-core/cklocation.h
src/ck-core/cknodegroupreduction.h
src/ck-core/ckreduction.C
src/ck-core/ckreduction.ci
src/ck-core/ckreduction.h
src/ck-core/init.C
src/ck-ldb/CentralLB.C
src/ck-ldb/GreedyLB.C
src/ck-ldb/LBDBManager.C
src/ck-ldb/RotateLB.C
src/ck-ldb/lbdb.C
src/conv-core/convcore.c
src/conv-core/converse.h
src/conv-core/isomalloc.c
src/conv-core/memory-isomalloc.c
src/conv-core/quiescence.c
src/conv-core/threads.c
src/libs/ck-libs/ampi/ampi.C
src/libs/ck-libs/ampi/ampi.h
src/libs/ck-libs/tcharm/tcharm.C
src/libs/ck-libs/tcharm/tcharm_impl.h
src/libs/ck-libs/tcharm/tcharmc.h
src/scripts/Make.depends
src/scripts/Makefile

index 48bf26ba6cdd3a903f889ed9a7c72c7354067001..969a77a12bdf11def8fbf5a2e33cdf91d190e42f 100644 (file)
@@ -867,12 +867,24 @@ int CmiMyRank(void)
 }
 #endif
 
+CpvExtern(char *,_validProcessors);
+CpvExtern(int,_charmEpoch);
+
 /*Add a message to this processor's receive queue 
   Must be called while holding comm. lock
 */
+
+extern double evacTime;
+
 static void CmiPushPE(int pe,void *msg)
 {
   CmiState cs=CmiGetStateN(pe);
+       /*
+               FAULT_EVAC
+       
+       if(CpvAccess(_charmEpoch)&&!(CpvAccess(_validProcessors)[CmiMyPe()])){
+               printf("[%d] Message after stop at %.6lf in %.6lf \n",CmiMyPe(),CmiWallTimer(),CmiWallTimer()-evacTime);
+       }*/
   MACHSTATE1(2,"Pushing message into %d's queue",pe);  
   MACHLOCK_ASSERT(comm_flag,"CmiPushPE")
 
@@ -893,7 +905,8 @@ static void CmiPushPE(int pe,void *msg)
 static void CmiPushNode(void *msg)
 {
   CmiState cs=CmiGetStateN(0);
-  MACHSTATE(2,"Pushing message into node queue");
+  
+       MACHSTATE(2,"Pushing message into node queue");
   MACHLOCK_ASSERT(comm_flag,"CmiPushNode")
   
 #if CMK_IMMEDIATE_MSG
@@ -1537,8 +1550,12 @@ void DeliverOutgoingMessage(OutgoingMsg ogm)
     SendHypercube(ogm, 1, 0, NULL, 0, DGRAM_BROADCAST);
 #else
     for (i=0; i<_Cmi_numnodes; i++)
-      if (i!=_Cmi_mynode)
-       DeliverViaNetwork(ogm, nodes + i, DGRAM_BROADCAST, DGRAM_ROOTPE_MASK);
+      if (i!=_Cmi_mynode){
+                               /*FAULT_EVAC : is the target processor valid*/
+                               if(CpvAccess(_validProcessors)[i]){
+                                       DeliverViaNetwork(ogm, nodes + i, DGRAM_BROADCAST, DGRAM_ROOTPE_MASK);
+                               }
+                       }       
 #endif
     GarbageCollectMsg(ogm);
     break;
@@ -1553,8 +1570,12 @@ void DeliverOutgoingMessage(OutgoingMsg ogm)
     SendHypercube(ogm, 1, 0, NULL, 0, DGRAM_BROADCAST);
 #else
     for (i = 0; i<_Cmi_numnodes; i++)
-      if (i!=_Cmi_mynode)
-       DeliverViaNetwork(ogm, nodes + i, DGRAM_BROADCAST, DGRAM_ROOTPE_MASK);
+      if (i!=_Cmi_mynode){
+                               /*FAULT_EVAC : is the target processor valid*/
+                               if(CpvAccess(_validProcessors)[i]){
+                                       DeliverViaNetwork(ogm, nodes + i, DGRAM_BROADCAST, DGRAM_ROOTPE_MASK);
+                               }
+                       }       
 #endif
     GarbageCollectMsg(ogm);
     break;
index a30cabb453eaaea4361a839a24bcc4029610d1ed..7c6dd8062ba82c4dcc86bbc16e3b317faa337361 100644 (file)
@@ -393,6 +393,11 @@ class IrrGroup : public Chare {
     virtual CmiBool isReductionMgr(void){ return CmiFalse; }
     static int isIrreducible(){ return 1;}
     virtual void flushStates() {}
+               /*
+                       FAULT_EVAC
+               */
+               virtual void evacuate(){};
+               virtual void doneEvacuate(){};
     virtual void CkAddThreadListeners(CthThread tid, void *msg);
 };
 
@@ -899,6 +904,7 @@ extern void CkStartQD(const CkCallback& cb);
 #include "waitqd.h"
 #include "sdag.h"
 #include "ckcheckpoint.h"
+#include "ckevacuation.h"
 #include "ckarrayreductionmgr.h"
 #include "trace.h"
 
index a2835baf61995936d2f7dcd48dec4e0113b1ea6d..30fe2cd9995ce135fa2963eaaa9432a6006443ed 100644 (file)
@@ -496,7 +496,9 @@ void _createGroup(CkGroupID groupID, envelope *env)
   int gIdx = _entryTable[epIdx]->chareIdx;  
   CkNodeGroupID rednMgr;
   if(_chareTable[gIdx]->isIrr == 0){
-       rednMgr = CProxy_CkArrayReductionMgr::ckNew();
+               CProxy_CkArrayReductionMgr rednMgrProxy = CProxy_CkArrayReductionMgr::ckNew();
+               rednMgr = rednMgrProxy;
+               rednMgrProxy.setAttachedGroup(groupID);
   }else{
        rednMgr.setZero();
   }
@@ -916,8 +918,16 @@ static void _skipCldHandler(void *converseMsg)
        (unsigned int *)env->getPrioPtr());
 }
 
+extern "C" void __dbgcheckMessageHandler();
+
 static void _skipCldEnqueue(int pe,envelope *env, int infoFn)
 {
+       if(pe == CkMyPe() ){
+               if(!CkpvAccess(_validProcessors)[CkMyPe()]){
+                       printf("[%d] Invalid processor sending itself a message \n",CkMyPe());
+                       return;
+               }
+       }
   if (pe == CkMyPe() && !CmiImmIsRunning()) {
 #if CMK_OBJECT_QUEUE_AVAILABLE
     Chare *obj = CkFindObjectPtr(env);
@@ -941,7 +951,9 @@ static void _skipCldEnqueue(int pe,envelope *env, int infoFn)
     CmiSetInfo(env,infoFn);
     if (pe==CLD_BROADCAST) { CmiSyncBroadcastAndFree(len, (char *)env); }
     else if (pe==CLD_BROADCAST_ALL) { CmiSyncBroadcastAllAndFree(len, (char *)env); }
-    else CmiSyncSendAndFree(pe, len, (char *)env);
+    else{
+                       CmiSyncSendAndFree(pe, len, (char *)env);
+               }       
   }
 }
 
@@ -1056,7 +1068,11 @@ extern "C"
 void CkSendMsgInline(int entryIndex, void *msg, const CkChareID *pCid, int opts)
 {
   if (pCid->onPE==CkMyPe())
-  { //Just directly call the chare (skip QD handling & scheduler)
+  { 
+               if(!CkpvAccess(_validProcessors)[CkMyPe()]){
+                       return;
+               }
+               //Just directly call the chare (skip QD handling & scheduler)
     register envelope *env = UsrToEnv(msg);
     if (env->isPacked()) CkUnpackMessage(&env);
     _STATS_RECORD_PROCESS_MSG_1();
@@ -1141,6 +1157,9 @@ void CkSendMsgBranchInline(int eIdx, void *msg, int destPE, CkGroupID gID, int o
 {
   if (destPE==CkMyPe())
   {
+               if(!CkpvAccess(_validProcessors)[CkMyPe()]){
+                       return;
+               }
     IrrGroup *obj=(IrrGroup *)_localBranch(gID);
     if (obj!=NULL)
     { //Just directly call the group:
index df2521550731d7de28100e58cdf582d3b44530b3..85b15c49c1ae492e35ed364204ed16674399595f 100644 (file)
@@ -57,6 +57,8 @@ Orion Sky Lawlor, olawlor@acm.org
 #include "LBDatabase.h"
 #endif // CMK_LBDB_ON
 
+CpvDeclare(int ,serializer);
+
 /************************** Debugging Utilities **************/
 
 //For debugging: convert given index to a string (NOT threadsafe)
@@ -370,6 +372,7 @@ CkArrayID CProxy_ArrayBase::ckCreateArray(CkArrayMessage *m,int ctor,
   CkMarshalledMessage marsh(m);
   CProxy_CkArrayReductionMgr nodereductionProxy = CProxy_CkArrayReductionMgr::ckNew();
   CkGroupID ag=CProxy_CkArray::ckNew(opts,marsh,nodereductionProxy);
+       nodereductionProxy.setAttachedGroup(ag);
   return (CkArrayID)ag;
 }
 CkArrayID CProxy_ArrayBase::ckCreateEmptyArray(void)
@@ -841,9 +844,9 @@ void CProxy_ArrayBase::ckBroadcast(CkArrayMessage *msg, int ep, int opts) const
        else 
        { //Broadcast message via serializer node
          _TRACE_CREATION_DETAILED(UsrToEnv(msg), ep);
-         int skipsched = opts & CK_MSG_EXPEDITED;
-         int serializer=0;//1623802937%CkNumPes();
-         if (CkMyPe()==serializer)
+         int skipsched = opts & CK_MSG_EXPEDITED; 
+         //int serializer=0;//1623802937%CkNumPes();
+         if (CkMyPe()==CpvAccess(serializer))
          {
                DEBB((AA"Sending array broadcast\n"AB));
                if (skipsched)
@@ -851,12 +854,12 @@ void CProxy_ArrayBase::ckBroadcast(CkArrayMessage *msg, int ep, int opts) const
                else
                        CProxy_CkArray(_aid).recvBroadcast(msg);
          } else {
-               DEBB((AA"Forwarding array broadcast to serializer node %d\n"AB,serializer));
+               DEBB((AA"Forwarding array broadcast to serializer node %d\n"AB,CpvAccess(serializer)));
                CProxy_CkArray ap(_aid);
                if (skipsched)
-                       ap[serializer].sendExpeditedBroadcast(msg);
+                       ap[CpvAccess(serializer)].sendExpeditedBroadcast(msg);
                else
-                       ap[serializer].sendBroadcast(msg);
+                       ap[CpvAccess(serializer)].sendBroadcast(msg);
          }
        }
 }
@@ -865,8 +868,12 @@ void CProxy_ArrayBase::ckBroadcast(CkArrayMessage *msg, int ep, int opts) const
 void CkArray::sendBroadcast(CkMessage *msg)
 {
        CK_MAGICNUMBER_CHECK
-       //Broadcast the message to all processors
-       thisProxy.recvBroadcast(msg);
+       if(CkMyPe() == CpvAccess(serializer)){
+               //Broadcast the message to all processors
+               thisProxy.recvBroadcast(msg);
+       }else{
+               thisProxy[CpvAccess(serializer)].sendBroadcast(msg);
+       }
 }
 void CkArray::sendExpeditedBroadcast(CkMessage *msg)
 {
index 40313de177a66cf00a359c71bcb2023f75e0c68b..4c7b96cc534cda5176fab04864481a4544254309 100644 (file)
@@ -32,6 +32,7 @@ Orion Sky Lawlor, olawlor@acm.org
        Utility defines, includes, etc.
 */
 extern void _registerCkArray(void);
+CpvExtern (int ,serializer);
 
 /**
 \addtogroup CkArray
index c33f272bbd1d9a77f3ac3429d8c6e48976d620ab..61e1d93c70237fb12cf5554b90694741fcd43460 100644 (file)
@@ -4,7 +4,7 @@
 #define ARRREDDEBUG 0
 
 #if ARRREDDEBUG
-#define ARPRINT ARPRINT
+#define ARPRINT CkPrintf
 #else
 #define ARPRINT //ARPRINT
 #endif
@@ -16,6 +16,8 @@ CkArrayReductionMgr::CkArrayReductionMgr(){
        count = 0;
        lockCount = CmiCreateLock();
        ctorDoneFlag = 1;
+       my_rednMgrs = new (CkReductionMgr *)[size];
+       attachedGroup.setZero();
 };
 
 void CkArrayReductionMgr::flushStates(){
@@ -32,7 +34,7 @@ void CkArrayReductionMgr::flushStates(){
 
 void CkArrayReductionMgr::collectAllMessages(){
        if(count == size){
-               ARPRINT("[%d] CollectAll messages  for %d with %d\n",CkMyNode(),redNo,count);
+               ARPRINT("[%d] CollectAll messages  for %d with %d on %p\n",CkMyNode(),redNo,count,this);
                CkReductionMsg *result = reduceMessages();
                result->redNo = redNo;
                /**keep a count of elements that contributed to the original reduction***/
@@ -54,7 +56,7 @@ void CkArrayReductionMgr::collectAllMessages(){
 }
 
 void CkArrayReductionMgr::contributeArrayReduction(CkReductionMsg *m){
-       ARPRINT("[%d]Contribute Array Reduction called for RedNo %d\n",CkMyNode(),m->getRedNo());
+       ARPRINT("[%d]Contribute Array Reduction called for RedNo %d group %d \n",CkMyNode(),m->getRedNo(),thisgroup.idx);
        /** store the contribution untill all procs have contributed. At that point reduce and
        carry out a reduction among nodegroups*/
        CmiLock(lockCount);
@@ -135,10 +137,57 @@ void CkArrayReductionMgr::pup(PUP::er &p){
        p(redNo);p(count);
        p|my_msgs;
        p|my_futureMsgs;
+       p|attachedGroup;
        if(p.isUnpacking()) {
          size = CkMyNodeSize();
          lockCount = CmiCreateLock();
+               my_rednMgrs = new (CkReductionMgr *)[size];
+               setAttachedGroup(attachedGroup);
        }
 }
 
+void CkArrayReductionMgr::setAttachedGroup(CkGroupID groupID){
+       attachedGroup = groupID;
+       CProxy_CkReductionMgr reductionMgrProxy(attachedGroup);
+       int firstPE = CkNodeFirst(CkMyNode());
+       for(int i=0;i<size;i++){
+               my_rednMgrs[i] = reductionMgrProxy[firstPE+i].ckLocalBranch();
+       }
+       ARPRINT("[%d] setAttachedGroup called with attachedGroup %d \n",CkMyNode(),attachedGroup);
+}
+
+void CkArrayReductionMgr::addRednMgr(CkReductionMgr *rednMgr,int rank){
+       ARPRINT("[%d] addRednMgr called with %p on %p \n",CkMyNode(),rednMgr,this);
+       my_rednMgrs[rank] = rednMgr;
+}
+
+
+void CkArrayReductionMgr::startNodeGroupReduction(int number,CkGroupID groupID){
+       ARPRINT("[%d] startNodeGroupReductions for red No %d my group %d attached group %d on %p \n",CkMyNode(),number,thisgroup.idx, attachedGroup.idx,this);
+       if(attachedGroup.isZero()){
+               setAttachedGroup(groupID);
+       }
+       startLocalGroupReductions(number);
+       CkReductionNumberMsg *msg = new CkReductionNumberMsg(number);
+       envelope::setSrcPe((char *)UsrToEnv(msg),CkMyNode());
+       ((CkNodeReductionMgr *)this)->ReductionStarting(msg);
+}
+void CkArrayReductionMgr::startLocalGroupReductions(int number){       
+       ARPRINT("[%d] startLocalGroupReductions for red No %d my group %d attached group %d number of rednMgrs %d on %p \n",CkMyNode(),number,thisgroup.idx, attachedGroup.idx,size,this);
+       if(attachedGroup.isZero()){
+               return;
+       }
+       int firstPE = CkNodeFirst(CkMyNode());
+       for(int i=0;i<size;i++){
+               CProxy_CkReductionMgr reductionMgrProxy(attachedGroup);
+               reductionMgrProxy[firstPE+i].ReductionStarting(new CkReductionNumberMsg(number));
+/*             if(my_rednMgrs[i]){
+                       my_rednMgrs[i]->ReductionStarting(new CkReductionNumberMsg(number));
+               }else{
+                       ARPRINT("[%d] my_rednMgr is null for %d for red No %d on %p \n",CkMyNode(),i,number,this);
+               }*/
+       }
+};
+
+
 #include "CkArrayReductionMgr.def.h"
index 7dce8c720f342c68958cf1cff768550417edccf5..b7b8ae641c0854c520c6c325d0434037fc1a5b9e 100644 (file)
@@ -2,6 +2,7 @@ module CkArrayReductionMgr {
        extern module CkReduction;
        nodegroup [migratable] CkArrayReductionMgr {
                entry CkArrayReductionMgr();
+               entry void setAttachedGroup(CkGroupID groupID);
                //entry  [immediate] void contributeArrayReduction(CkReductionMsg *);
        };
 };
index 991c9f796c88f731f0b62df2bb19575aac206243..1cae91598debc649c38bca4e4b3542f645500c32 100644 (file)
@@ -1,17 +1,18 @@
 #ifndef _CKARRAYREDNMGR_H
 #define _CKARRAYREDNMGR_H
 class CkArrayReductionMgr : public NodeGroup{
-       /** Group Reduction Managers use this class
-       to carry out their reductions. This class receives
-       contributions from group/array redn mgrs when all
-       data from elements on a processor has been collected
-       This is then contributed to this group.*/
+       /** This class receives contributions from all the CkReductionMgr s in a node, after each of them has 
+       collected all the contributions on one processor. The data from the different processors in a node
+       is collected and sent to the CkNodeReductionMgr for this node
+       */
        private:
                int size;
                int redNo;
                int count;
+               CkGroupID attachedGroup;
                CkMsgQ<CkReductionMsg> my_msgs;
                CkMsgQ<CkReductionMsg> my_futureMsgs;
+               CkReductionMgr **my_rednMgrs;
                CmiNodeLock lockCount;
                void collectAllMessages();
        public:
@@ -22,6 +23,10 @@ class CkArrayReductionMgr : public NodeGroup{
                CkReductionMsg *reduceMessages(void);
                 void flushStates();
                virtual void pup(PUP::er &p);
+               void setAttachedGroup(CkGroupID groupID);
+               void addRednMgr(CkReductionMgr *rednMgr,int rank);
+               void startNodeGroupReduction(int number,CkGroupID groupID);
+               virtual void startLocalGroupReductions(int number);
 };
 #endif
 
index a122cfa3fd96b865a30ea2f7c0b4380f29602627..31219fca3767d951acb69090be78c8d2d500d13c 100644 (file)
@@ -12,10 +12,13 @@ More documentation goes here...
 #include <string.h>
 #include "charm++.h"
 #include "ck.h"
-//#include "ckcheckpoint.h"
+#include "ckcheckpoint.h"
 
 #define DEBCHK  // CkPrintf
 
+#define DEBUGC(x) x
+//#define DEBUGC(x) 
+
 CkGroupID _sysChkptMgr;
 
 typedef struct _GroupInfo{
@@ -296,15 +299,6 @@ void CkPupNodeGroupData(PUP::er &p)
        delete [] tmpInfo;
 }
 
-// loop over all CkLocMgr and do "code"
-#define  CKLOCMGR_LOOP(code)   \
-  for(i=0;i<numGroups;i++) {   \
-    IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();   \
-    if(obj->isLocMgr())  {     \
-      CkLocMgr *mgr = (CkLocMgr*)obj;  \
-      code     \
-    }  \
-  }
 
 // handle chare array elements for this processor
 void CkPupArrayElementsData(PUP::er &p)
index 2060b7c5c4b9a6479375eccf19d058305126caeb..af67d8331454e049d6a4ffee451181a422f56688 100644 (file)
@@ -27,7 +27,15 @@ restarting of Charm++ programs. ...
 #define _CKCHECKPOINT_H
 
 #include "CkCheckpoint.decl.h"
-
+// loop over all CkLocMgr and do "code"
+#define  CKLOCMGR_LOOP(code)   \
+  for(i=0;i<numGroups;i++) {   \
+    IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();   \
+    if(obj->isLocMgr())  {     \
+      CkLocMgr *mgr = (CkLocMgr*)obj;  \
+      code     \
+    }  \
+  }
 /***
   * Location iterator that save each location
  ***/
@@ -59,4 +67,5 @@ void CkPupProcessorData(PUP::er &p);
 void CkStartCheckpoint(char* dirname,const CkCallback& cb);
 void CkRestartMain(const char* dirname);
 
+
 #endif //_CKCHECKPOINT_H
index 8007b34ffdfaf19b94b636d8ab93c18c6e844fb8..d16dd3b687239b69800ba3b33a782cf97931f307 100644 (file)
@@ -45,7 +45,7 @@ static const char *idx2str(const CkArrayMessage *m)
 #   define DEBS(x) CkPrintf x  //Send/recv/broadcast debug messages
 #   define DEBM(x) CkPrintf x  //Migration debug messages
 #   define DEBL(x) CkPrintf x  //Load balancing debug messages
-#   define DEBK(x) CkPrintf x  //Spring Cleaning debug messages
+#   define DEBK(x) //CkPrintf x  //Spring Cleaning debug messages
 #   define DEBB(x) CkPrintf x  //Broadcast debug messages
 #   define AA "LocMgr on %d: "
 #   define AB ,CkMyPe()
@@ -62,6 +62,9 @@ static const char *idx2str(const CkArrayMessage *m)
 #endif
 
 
+extern "C" void __dbgcheckMessageHandler();
+
+
 #if CMK_LBDB_ON
 /*LBDB object handles are fixed-sized, and not necc.
 the same size as ArrayIndices.
@@ -188,14 +191,23 @@ public:
 #if 1
     if (i.nInts==1) {
       //Map 1D integer indices in simple round-robin fashion
-      return (i.data()[0])%CkNumPes();
+      int ans= (i.data()[0])%CkNumPes();
+                       while(!CkpvAccess(_validProcessors)[ans] || (ans == CkMyPe() && CpvAccess(startedEvac))){
+                               ans = (ans +1 )%CkNumPes();
+                       }
+                       return ans;
     }
     else 
 #endif
       {
        //Map other indices based on their hash code, mod a big prime.
-       unsigned int hash=(i.hash()+739)%1280107;
-       return (hash % CkNumPes());
+                       unsigned int hash=(i.hash()+739)%1280107;
+                       int ans = (hash % CkNumPes());
+                       while(!CkpvAccess(_validProcessors)[ans]){
+                               ans = (ans +1 )%CkNumPes();
+                       }
+                       return ans;
+
       }
   }
 };
@@ -554,6 +566,10 @@ void CkMigratable::commonInit(void) {
        thisChareType=i.chareType;
        usesAtSync=CmiFalse;
        barrierRegistered=CmiFalse;
+       /*
+       FAULT_EVAC
+       */
+       AsyncEvacuate(CmiTrue);
 }
 
 CkMigratable::CkMigratable(void) {
@@ -576,6 +592,12 @@ void CkMigratable::pup(PUP::er &p) {
        if (p.isUnpacking()) myRec->ReadyMigrate(readyMigrate);
 #endif
        if(p.isUnpacking()) barrierRegistered=CmiFalse;
+       /*
+               FAULT_EVAC
+       */
+       p | asyncEvacuate;
+       if(p.isUnpacking()){myRec->AsyncEvacuate(asyncEvacuate);}
+       
        ckFinishConstruction();
 }
 
@@ -588,6 +610,7 @@ void CkMigratable::ckAboutToMigrate(void) { }
 void CkMigratable::ckJustMigrated(void) { }
 
 CkMigratable::~CkMigratable() {
+       __dbgcheckMessageHandler();
        DEBC((AA"In CkMigratable::~CkMigratable %s\n"AB,idx2str(thisIndexMax)));
 #if CMK_OUT_OF_CORE
        isInCore=CmiFalse;
@@ -613,6 +636,7 @@ CkMigratable::~CkMigratable() {
 #endif
        //To detect use-after-delete
        thisIndexMax.nInts=-123456;
+       __dbgcheckMessageHandler();
 }
 
 void CkMigratable::CkAbort(const char *why) const {
@@ -756,32 +780,41 @@ CkLocRec_local::CkLocRec_local(CkLocMgr *mgr,CmiBool fromMigration,
        nextPe = -1;
        asyncMigrate = CmiFalse;
        readyMigrate = CmiTrue;
+       bounced  = CmiFalse;
        the_lbdb=mgr->getLBDB();
        ldHandle=the_lbdb->RegisterObj(mgr->getOMHandle(),
                idx2LDObjid(idx),(void *)this,1);
        if (fromMigration) {
                DEBL((AA"Element %s migrated in\n"AB,idx2str(idx)));
-               the_lbdb->Migrated(ldHandle, !ignoreArrival);
-               if (ignoreArrival)  {
+               if (!ignoreArrival)  {
+                       the_lbdb->Migrated(ldHandle, CmiTrue);
                  // load balancer should ignore this objects movement
-                 AsyncMigrate(CmiTrue);
+               //  AsyncMigrate(CmiTrue);
                }
        }
 #endif
+       /*
+               FAULT_EVAC
+       */
+       asyncEvacuate = CmiTrue;
 }
 CkLocRec_local::~CkLocRec_local()
 {
        if (deletedMarker!=NULL) *deletedMarker=CmiTrue;
+       __dbgcheckMessageHandler();
        myLocMgr->reclaim(idx,localIdx);
+       __dbgcheckMessageHandler();
 #if CMK_LBDB_ON
        stopTiming();
        DEBL((AA"Unregistering element %s from load balancer\n"AB,idx2str(idx)));
        the_lbdb->UnregisterObj(ldHandle);
 #endif
+       __dbgcheckMessageHandler();
 }
 void CkLocRec_local::migrateMe(int toPe) //Leaving this processor
 {
        //This will pack us up, send us off, and delete us
+//     printf("[%d] migrating migrateMe to %d \n",CkMyPe(),toPe);
        myLocMgr->emigrate(this,toPe);
 }
 
@@ -1049,13 +1082,23 @@ public:
   
        //Send a message for this element.
        virtual CmiBool deliver(CkArrayMessage *msg,CkDeliver_t type,int opts=0) {
+               /*FAULT_EVAC*/
+               int destPE = onPe;
+               if((!CpvAccess(_validProcessors)[onPe] && onPe != allowMessagesOnly)){
+//                     printf("Delivery failed because process %d is invalid\n",onPe);
+                       /*
+                               Send it to its home processor instead
+                       */
+                       const CkArrayIndex &idx=msg->array_index();
+                       destPE = getNextPE(idx);
+               }
                access();//Update our modification date
                msg->array_hops()++;
                DEBS((AA"   Forwarding message for element %s to %d (REMOTE)\n"AB,
-                     idx2str(msg->array_index()),onPe));
+                     idx2str(msg->array_index()),destPE));
                if (opts & CK_MSG_KEEP)
                        msg = (CkArrayMessage *)CkCopyMsg((void **)&msg);
-               CkArrayManagerDeliver(onPe,msg,opts);
+               CkArrayManagerDeliver(destPE,msg,opts);
                return CmiTrue;
        }
        //Return if this element is now obsolete
@@ -1332,6 +1375,10 @@ void CkLocMgr::inform(const CkArrayIndex &idx,int nowOnPe)
        CkLocRec *rec=elementNrec(idx);
        if (rec!=NULL && rec->type()==CkLocRec::local)
                return; //Never replace a local element's record!
+               /*
+                       FAULT_EVAC
+               */
+       //      CmiAssert(CpvAccess(_validProcessors)[nowOnPe]);
        insertRemote(idx,nowOnPe);
 }
 
@@ -1412,7 +1459,9 @@ void CkLocMgr::reclaim(const CkArrayIndex &idx,int localIdx) {
        DEBC((AA"Destroying element %s (local %d)\n"AB,idx2str(idx),localIdx));
        //Delete, and mark as empty, each array element
        for (ManagerRec *m=firstManager;m!=NULL;m=m->next) {
+               __dbgcheckMessageHandler();
                delete m->elts.get(localIdx);
+               __dbgcheckMessageHandler();
                m->elts.empty(localIdx);
        }
        
@@ -1458,6 +1507,7 @@ void CkLocMgr::removeFromTable(const CkArrayIndex &idx) {
 /************************** LocMgr: MESSAGING *************************/
 /// Deliver message to this element, going via the scheduler if local
 void CkLocMgr::deliver(CkMessage *m,CkDeliver_t type,int opts) {
+       DEBS((AA"deliver \n"AB));
        CK_MAGICNUMBER_CHECK
        CkArrayMessage *msg=(CkArrayMessage *)m;
        const CkArrayIndex &idx=msg->array_index();
@@ -1465,6 +1515,11 @@ void CkLocMgr::deliver(CkMessage *m,CkDeliver_t type,int opts) {
        if (type==CkDeliver_queue)
                _TRACE_CREATION_DETAILED(UsrToEnv(m),msg->array_ep());
        CkLocRec *rec=elementNrec(idx);
+       if(rec != NULL){
+               DEBS((AA"deliver %s of type %d \n"AB,idx2str(idx),rec->type()));
+       }else{
+               DEBS((AA"deliver %s rec is null\n"AB,idx2str(idx)));
+       }
 #if CMK_LBDB_ON
        if (type==CkDeliver_queue) {
                if (!(opts & CK_MSG_LB_NOTRACE)) {
@@ -1473,8 +1528,19 @@ void CkLocMgr::deliver(CkMessage *m,CkDeliver_t type,int opts) {
                }
        }
 #endif
-       if (rec!=NULL) rec->deliver(msg,type,opts);
-       else /* rec==NULL*/ {
+       /**FAULT_EVAC*/
+       if (rec!=NULL){
+               CmiBool result = rec->deliver(msg,type,opts);
+               if(!result){
+               /*      //DEBS((AA"deliver %s failed type %d \n"AB,idx2str(idx),rec->type()));
+                       DEBS((AA"deliver %s failed \n"AB,idx2str(idx)));
+                       if(rec->type() == CkLocRec::remote){
+                               if (opts & CK_MSG_KEEP)
+                                       msg = (CkArrayMessage *)CkCopyMsg((void **)&msg);
+                               deliverUnknown(msg,type);
+                       }*/
+               }
+       }else /* rec==NULL*/ {
                if (opts & CK_MSG_KEEP)
                        msg = (CkArrayMessage *)CkCopyMsg((void **)&msg);
                deliverUnknown(msg,type);
@@ -1489,7 +1555,7 @@ CmiBool CkLocMgr::deliverUnknown(CkArrayMessage *msg,CkDeliver_t type)
        int onPe=homePe(idx);
        if (onPe!=CkMyPe()) 
        {// Forward the message to its home processor
-               DEBM((AA"Forwarding message for unknown %s\n"AB,idx2str(idx)));
+               DEBM((AA"Forwarding message for unknown %s to home %d \n"AB,idx2str(idx),onPe));
                msg->array_hops()++;
                CkArrayManagerDeliver(onPe,msg);
                return CmiTrue;
@@ -1658,6 +1724,14 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
 {
        CK_MAGICNUMBER_CHECK
        if (toPe==CkMyPe()) return; //You're already there!
+       /*
+               FAULT_EVAC
+               if the toProcessor is already marked as invalid, dont emigrate
+               Shouldn't happen but might
+       */
+       if(!(CkpvAccess(_validProcessors)[toPe])){
+               return;
+       }
 
        int localIdx=rec->getLocalIndex();
        CkArrayIndexMax idx=rec->getIndex();
@@ -1672,6 +1746,8 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
 
        //Let all the elements know we're leaving
        callMethod(rec,&CkMigratable::ckAboutToMigrate);
+       /*EVAC*/
+       __dbgcheckMessageHandler();
 
 //First pass: find size of migration message
        int bufSize;
@@ -1681,7 +1757,9 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
                pupElementsFor(p,rec,CkElementCreation_migrate);
                bufSize=p.size(); 
        }
-       
+               /*EVAC*/
+       __dbgcheckMessageHandler();
+
 //Allocate and pack into message
        int doubleSize=bufSize/sizeof(double)+1;
        CkArrayElementMigrateMessage *msg = 
@@ -1691,6 +1769,10 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
 #if CMK_LBDB_ON
        msg->ignoreArrival = rec->isAsyncMigrate()?1:0;
 #endif
+       /*
+               FAULT_EVAC
+       */
+       msg->bounced = rec->isBounced();
        {
                PUP::toMem p(msg->packData); 
                p.becomeDeleting(); 
@@ -1703,16 +1785,26 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
                        CkAbort("Array element's pup routine has a direction mismatch.\n");
                }
        }
-       DEBM((AA"Migrated index size %s\n"AB,idx2str(idx)));    
+               /*EVAC*/
+       __dbgcheckMessageHandler();
+
+       DEBM((AA"Migrated index size %s to %d \n"AB,idx2str(idx),toPe));        
 
 //Send off message and delete old copy
+       __dbgcheckMessageHandler();
        thisProxy[toPe].immigrate(msg);
+       __dbgcheckMessageHandler();
        duringMigration=CmiTrue;
+       __dbgcheckMessageHandler();
        delete rec; //Removes elements, hashtable entries, local index
+       
+       __dbgcheckMessageHandler();
+       
        duringMigration=CmiFalse;
        //The element now lives on another processor-- tell ourselves and its home
        inform(idx,toPe);
        informHome(idx,toPe);
+       CK_MAGICNUMBER_CHECK
 }
 
 /**
@@ -1722,6 +1814,7 @@ void CkLocMgr::immigrate(CkArrayElementMigrateMessage *msg)
 {
        CkArrayMessage *amsg=(CkArrayMessage *)msg;
        const CkArrayIndex &idx=msg->idx;
+               
        PUP::fromMem p(msg->packData); 
        
        int nMsgMan;
@@ -1749,9 +1842,38 @@ void CkLocMgr::immigrate(CkArrayElementMigrateMessage *msg)
                
                CkAbort("Array element's pup routine has a direction mismatch.\n");
        }
+       /*
+               FAULT_EVAC
+                       if this element came in as a result of being bounced off some other process,
+                       then it needs to be resumed. It is assumed that it was bounced because load 
+                       balancing caused it to move into a processor which later crashed
+       */
+       if(msg->bounced){
+               callMethod(rec,&CkMigratable::ResumeFromSync);
+       }
        
        //Let all the elements know we've arrived
        callMethod(rec,&CkMigratable::ckJustMigrated);
+       /*FAULT_EVAC
+               If this processor has started evacuating array elements on it 
+               dont let new immigrants in. If they arrive send them to what
+               would be their correct homePE.
+               Leave a record here mentioning the processor where it got sent
+       */
+       
+       if(CpvAccess(startedEvac)){
+               int newhomePE = getNextPE(idx);
+               DEBM((AA"Migrated into failed processor index size %s resent to %d \n"AB,idx2str(idx),newhomePE));      
+               CkLocMgr *mgr = rec->getLocMgr();
+               int targetPE=getNextPE(idx);
+               //set this flag so that load balancer is not informed when
+               //this element migrates
+               rec->AsyncMigrate(CmiTrue);
+               rec->Bounced(CmiTrue);
+               mgr->emigrate(rec,targetPE);
+               
+       }
+
        delete msg;
 }
 
@@ -1786,13 +1908,23 @@ CkMigratable *CkLocMgr::lookup(const CkArrayIndex &idx,CkArrayID aid) {
        else return rec->lookupElement(aid);
 }
 //"Last-known" location (returns a processor number)
-int CkLocMgr::lastKnown(const CkArrayIndex &idx) const {
+int CkLocMgr::lastKnown(const CkArrayIndex &idx) {
        CkLocMgr *vthis=(CkLocMgr *)this;//Cast away "const"
        CkLocRec *rec=vthis->elementNrec(idx);
        int pe=-1;
        if (rec!=NULL) pe=rec->lookupProcessor();
        if (pe==-1) return homePe(idx);
-       else return pe;
+       else{
+               /*
+                       FAULT_EVAC
+                       if the lastKnownPE is invalid return homePE and delete this record
+               */
+               if(!(CkpvAccess(_validProcessors)[pe])){
+                       removeFromTable(idx);
+                       return homePe(idx);
+               }
+               return pe;
+       }       
 }
 /// Return true if this array element lives on another processor
 bool CkLocMgr::isRemote(const CkArrayIndex &idx,int *onPe) const
index d79b8f7d3050c142394ac580787f0adc959092a1..ce7bf2818db4253aee3d8a57f69525dded1bee80 100644 (file)
@@ -60,6 +60,7 @@ public:
        CkArrayIndexMax idx; // Array index that is migrating
        int ignoreArrival;   // if to inform LB of arrival
        int length;//Size in bytes of the packed data
+       CmiBool bounced;
        double* packData;
 };
 
@@ -222,6 +223,18 @@ private:
   CmiBool  readyMigrate;    /// status whether it is ready to migrate
   int  nextPe;              /// next migration dest processor
 #endif
+/**FAULT_EVAC*/
+private:
+       CmiBool asyncEvacuate; //can the element be evacuated anytime, false for tcharm 
+       CmiBool bounced; //did this element try to immigrate into a processor which was evacuating
+                                                                                       // and was bounced away to some other processor. This is assumed to happen
+                                                                                       //only if this object was migrated by a load balancer, but the processor
+                                                                                       // started crashing soon after
+public:        
+       CmiBool isAsyncEvacuate(){return asyncEvacuate;}
+       void AsyncEvacuate(CmiBool set){asyncEvacuate = set;}
+       CmiBool isBounced(){return bounced;}
+       void Bounced(CmiBool set){bounced = set;}
 };
 class CkLocRec_remote;
 
@@ -243,6 +256,7 @@ protected:
 private:
   int thisChareType;//My chare type
   void commonInit(void);
+       bool asyncEvacuate;
 public:
   CkArrayIndexMax thisIndexMax;
 
@@ -285,12 +299,14 @@ protected:
   virtual void CkAbort(const char *str) const;
 
   CmiBool usesAtSync;//You must set this in the constructor to use AtSync().
-  virtual void ResumeFromSync(void);
   CmiBool barrierRegistered;//True iff barrier handle below is set
 
 #if CMK_LBDB_ON  //For load balancing:
   void AtSync(int waitForMigration=1);
   int MigrateToPe()  { return myRec->MigrateToPe(); }
+
+public:
+  virtual void ResumeFromSync(void);
 private: //Load balancer state:
   LDBarrierClient ldBarrierHandle;//Transient (not migrated)  
   LDBarrierReceiver ldBarrierRecvHandle;//Transient (not migrated)  
@@ -314,6 +330,12 @@ private:
   int prefetchObjID; //From CooRegisterObject
   CmiBool isInCore; //If true, the object is present in memory
 #endif
+       /*
+               FAULT_EVAC
+       */
+       void AsyncEvacuate(CmiBool set){myRec->AsyncEvacuate(set);asyncEvacuate = set;};
+       public:
+       bool isAsyncEvacuate(){return asyncEvacuate;};
 };
 
 /** 
@@ -405,6 +427,9 @@ public:
        /// Find our location manager
        inline CkLocMgr *getManager(void) const {return mgr;}
        
+       /// Find the local record that refers to this element
+       inline CkLocRec_local *getLocalRecord(void) const {return rec;}
+       
        /// Look up and return the array index of this location.
        const CkArrayIndex &getIndex(void) const;
        
@@ -539,7 +564,7 @@ public:
        CkMigratable *lookup(const CkArrayIndex &idx,CkArrayID aid);
 
        /// Return the "last-known" location (returns a processor number)
-       int lastKnown(const CkArrayIndex &idx) const;
+       int lastKnown(const CkArrayIndex &idx);
 
        /// Return true if this array element lives on another processor
        bool isRemote(const CkArrayIndex &idx,int *onPe) const;
index 63f3dba4f3e42a93e5226a21f0d93f61610fdfa0..f600b311c1046b4b4b435b26fdddb378cd58f485 100644 (file)
@@ -147,7 +147,10 @@ public:
        void RecvMsg(CkReductionMsg *m);
        void doRecvMsg(CkReductionMsg *m);
 
-        virtual void flushStates();    // flush state varaibles
+  virtual void flushStates();  // flush state varaibles
+       virtual void startLocalGroupReductions(int number){}; // can be used to start reductions on all the 
+       //CkReductionMgrs on a particular node. It is overwritten by CkArrayReductionMgr to make the actual calls
+       // since it knows the CkReductionMgrs on a node.
 
 private:
 //Data members
@@ -175,6 +178,9 @@ private:
 
        int interrupt; /* flag for use in non-smp 0 means interrupt can occur 1 means not (also acts as a lock)*/
 
+       /*vector storing the children of this node*/
+       CkVec<int> kids;
+       
 //State:
        void startReduction(int number,int srcPE);
        void doAddContribution(CkReductionMsg *m);
@@ -188,10 +194,12 @@ private:
        unsigned label;
        int parent;
        int numKids;
-       int *kids;
+//     int *kids;
        void init_BinomialTree();
 
-       enum {TREE_WID=4};
+       
+       void init_BinaryTree();
+       enum {TREE_WID=2};
        int treeRoot(void);//Root PE
        CmiBool hasParent(void);
        int treeParent(void);//My parent PE
@@ -206,10 +214,35 @@ private:
        CmiBool isPresent(int num) const {return (CmiBool)(num==redNo);}
        CmiBool isFuture(int num) const {return (CmiBool)(num>redNo);}
 
+       /*FAULT_EVAC*/
+       bool oldleaf;
+       bool blocked;
+       int newParent;
+       CkVec<int> newKids;
+       CkMsgQ<CkReductionMsg> bufferedMsgs;
+       CkMsgQ<CkReductionMsg> bufferedRemoteMsgs;
+       enum {OLDPARENT,OLDCHILDREN,NEWPARENT,LEAFPARENT};
+       int numModificationReplies;
+       int maxModificationRedNo;
+       int tempModificationRedNo;
+       bool readyDeletion;
+       int killed;     
+       
 //Checkpointing utilities
  public:
        virtual void pup(PUP::er &p);
-
+       /*FAULT_EVAC*/
+       virtual void evacuate();
+       virtual void doneEvacuate();
+       void DeleteChild(int deletedChild);
+       void DeleteNewChild(int deletedChild);
+       void collectMaxRedNo(int maxRedNo);
+       void unblockNode(int maxRedNo);
+       void modifyTree(int code,int size,int *data);
+private:       
+       int findMaxRedNo();
+       void updateTree();
+       void clearBlockedMsgs();
 };
 
 
index 8a2395a0c0ed5d97d8064dd03f9351452703a315..35dc04541b07aba55d9a24c5caa0c8d94da8bd91 100644 (file)
@@ -55,8 +55,8 @@ waits for the migrant contributions to straggle in.
 // Reduction mananger internal information:
 #define DEBUGRED 1
 #define DEBR(x) CkPrintf x
-#define AA "Red PE%d Node%d #%d (%d,%d)> "
-#define AB ,CkMyPe(),CkMyNode(),redNo,nRemote,nContrib
+#define AA "Red PE%d Node%d #%d (%d,%d) Group %d> "
+#define AB ,CkMyPe(),CkMyNode(),redNo,nRemote,nContrib,thisgroup.idx
 
 #define DEBN(x) CkPrintf x
 #define AAN "Red Node%d "
@@ -84,7 +84,6 @@ Group::Group()
 #endif                 
        CProxy_CkArrayReductionMgr nodetemp(CkpvAccess(_currentGroupRednMgr));
        nodeProxy = nodetemp;
-
 }
 
 CK_REDUCTION_CONTRIBUTE_METHODS_DEF(Group,
@@ -146,12 +145,6 @@ void CkReductionClientBundle::callbackCfn(void *thisPtr,void *reductionMsg)
 }
 
 ///////////////// Reduction Manager //////////////////
-class CkReductionNumberMsg:public CMessage_CkReductionNumberMsg {
-public:
-  int num;
-  CkReductionNumberMsg(int n) {num=n;}
-};
-
 /*
 One CkReductionMgr runs a non-overlapping set of reductions.
 It collects messages from all local contributors, then sends
@@ -282,7 +275,7 @@ void CkReductionMgr::contributorDied(contributorInfo *ci)
   // contribution, which will never come.
     DEBR((AA"Dying guy %p must have been migrating-- he's at #%d!\n"AB,ci,ci->redNo));
     for (int r=ci->redNo;r<redNo;r++)
-      thisProxy[treeRoot()].MigrantDied(new CkReductionNumberMsg(r));
+      thisProxy[0].MigrantDied(new CkReductionNumberMsg(r));
   }
 
   //Add to the global count for all his future messages (wherever they are)
@@ -351,6 +344,7 @@ void CkReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
        //delete m;
        //return;
  }
+ DEBR((AA" Group ReductionStarting called for redNo %d\n"AB,m->num));
  int srcPE = (UsrToEnv(m))->getSrcPe();
   if (isPresent(m->num) && !inProgress)
   {
@@ -384,7 +378,7 @@ void CkReductionMgr::LateMigrantMsg(CkReductionMsg *m)
 //A late migrating contributor will never contribute to this reduction
 void CkReductionMgr::MigrantDied(CkReductionNumberMsg *m)
 {
-  if (hasParent() || m->num < completedRedNo) CkAbort("Late MigrantDied message recv'd!\n");
+  if (CkMyPe() != 0 || m->num < completedRedNo) CkAbort("Late MigrantDied message recv'd!\n");
   DEBR((AA"Migrant died before contributing to #%d\n"AB,m->num));
  // CkPrintf("[%d,%d]Migrant Died called\n",CkMyNode(),CkMyPe());                        
   adj(m->num).gcount--;//He won't be contributing to this one.
@@ -412,12 +406,35 @@ void CkReductionMgr::startReduction(int number,int srcPE)
   DEBR((AA"Starting reduction #%d  %d %d \n"AB,redNo,completedRedNo,number));
   inProgress=CmiTrue;
   //Sent start requests to our kids (in case they don't already know)
+
+       /*
+               FAULT_EVAC
+       */
+       if(!(CkpvAccess(_validProcessors)[CkMyPe()])){
+               return;
+       }
        
+       nodeProxy[CkMyNode()].ckLocalBranch()->startNodeGroupReduction(number,thisgroup);
        int temp;
+       /*
+  //making it a broadcast done only by PE 0
   if(!hasParent()){
                temp = completedRedNo+1;
-       /*      if(temp < 0)
-                       temp = 0;  */
+               for(int i=temp;i<=number;i++){
+                       for(int j=0;j<CkNumPes();j++){
+                               if(j != CkMyPe() && j != srcPE){
+                                       if((CkpvAccess(_validProcessors)[j])||allowMessagesOnly !=-1){
+                                               thisProxy[j].ReductionStarting(new CkReductionNumberMsg(i));
+                                       }
+                               }
+                       }
+               }       
+       }       else{
+               temp = number;
+       }*/
+/*  if(!hasParent()){
+               temp = completedRedNo+1;
        }       else{
                temp = number;
        }
@@ -426,27 +443,24 @@ void CkReductionMgr::startReduction(int number,int srcPE)
                if(hasParent()){
          // kick-start your parent too ...
                        if(treeParent() != srcPE){
-                       thisProxy[treeParent()].ReductionStarting(new CkReductionNumberMsg(i));
+                               if((CkpvAccess(_validProcessors)[treeParent()])||allowMessagesOnly !=-1){
+                               thisProxy[treeParent()].ReductionStarting(new CkReductionNumberMsg(i));
+                               }       
                        }       
                }
          for (int k=0;k<treeKids();k++)
          {
                        if(firstKid()+k != srcPE){
-                   DEBR((AA"Asking child PE %d to start #%d\n"AB,firstKid()+k,redNo));
-                   thisProxy[firstKid()+k].ReductionStarting(new CkReductionNumberMsg(i));
+                               if((CkpvAccess(_validProcessors)[kids[k]])||allowMessagesOnly !=-1){
+                           DEBR((AA"Asking child PE %d to start #%d\n"AB,kids[k],redNo));
+                           thisProxy[kids[k]].ReductionStarting(new CkReductionNumberMsg(i));
+                               }       
                        }       
        }
        }
+       */
 }      
- /* else{
-         // kick-start your parent too ...
-         thisProxy[treeParent()].ReductionStarting(new CkReductionNumberMsg(number));
-         for (int k=0;k<treeKids();k++)
-         {
-           DEBR((AA"Asking child PE %d to start #%d\n"AB,firstKid()+k,redNo));
-           thisProxy[firstKid()+k].ReductionStarting(new CkReductionNumberMsg(number));
-         }
-  }*/
+
 /*Handle a message from one element for the reduction*/
 void CkReductionMgr::addContribution(CkReductionMsg *m)
 {
@@ -455,7 +469,7 @@ void CkReductionMgr::addContribution(CkReductionMsg *m)
     DEBR((AA"Migrant %p gives late contribution for #%d!\n"AB,m->ci,m->redNo));
    // if (!hasParent()) //Root moved on too soon-- should never happen
    //   CkAbort("Late reduction contribution received at root!\n");
-    thisProxy[treeRoot()].LateMigrantMsg(m);
+    thisProxy[0].LateMigrantMsg(m);
   }
   else if (isFuture(m->redNo)) {//An early contribution-- add to future Q
     DEBR((AA"Contributor %p gives early contribution-- for #%d\n"AB,m->ci,m->redNo));
@@ -508,7 +522,7 @@ void CkReductionMgr::finishReduction(void)
   //Shift the count adjustment vector down one slot (to match new redNo)
   int i;
 
-  if(hasParent()){
+  if(CkMyPe()!=0){
        int i;
        completedRedNo++;
        for (i=1;i<adjVec.length();i++)
@@ -534,6 +548,8 @@ void CkReductionMgr::finishReduction(void)
 }
 
 //////////// Reduction Manager Utilities /////////////
+
+/*
 int CkReductionMgr::treeRoot(void)
 {
   return 0;
@@ -556,7 +572,7 @@ int CkReductionMgr::treeKids(void)//Number of children in tree
   if (nKids>TREE_WID) nKids=TREE_WID;
   if (nKids<0) nKids=0;
   return nKids;
-}
+}*/
 
 //Return the countAdjustment struct for the given redNo:
 countAdjustment &CkReductionMgr::adj(int number)
@@ -1156,6 +1172,8 @@ CkNodeReductionMgr::CkNodeReductionMgr()//Constructor
 {
 #ifdef BINOMIAL_TREE
   init_BinomialTree();
+#else
+       init_BinaryTree();
 #endif
   storedCallback=NULL;
   redNo=0;
@@ -1171,6 +1189,12 @@ CkNodeReductionMgr::CkNodeReductionMgr()//Constructor
   creating=CmiFalse;
   interrupt = 0;
   DEBR((AA"In NodereductionMgr constructor at %d \n"AB,this));
+       /*
+               FAULT_EVAC
+       */
+       blocked = false;
+       maxModificationRedNo = INT_MAX;
+       killed=0;
 }
 
 void CkNodeReductionMgr::flushStates()
@@ -1248,6 +1272,11 @@ void CkNodeReductionMgr::contributeWithCounter(contributorInfo *ci,CkReductionMs
 void CkNodeReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
 {
   CmiLock(lockEverything);
+       if(blocked){
+               delete m;
+       CmiUnlock(lockEverything);
+               return ;
+       }
        int srcNode = CmiNodeOf((UsrToEnv(m))->getSrcPe());
   if (isPresent(m->num) && !inProgress)
   {
@@ -1255,9 +1284,8 @@ void CkNodeReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
     startReduction(m->num,srcNode);
     finishReduction();
   } else if (isFuture(m->num)){
-       //CkPrintf("[%d][%d] Message num %d Present redNo %d \n",CkMyNode(),CkMyPe(),m->num,redNo);
-       CkAbort("My reduction tree parent somehow got ahead of me! in nodegroups\n");
-    }
+       DEBR(("[%d][%d] Message num %d Present redNo %d \n",CkMyNode(),CkMyPe(),m->num,redNo));
+  }
   else //is Past
     DEBR((AA"Ignoring node parent's late request to start #%d\n"AB,m->num));
   CmiUnlock(lockEverything);
@@ -1270,6 +1298,15 @@ void CkNodeReductionMgr::doRecvMsg(CkReductionMsg *m){
 #if DEBUGRED
        CkPrintf("[%d,%d] doRecvMsg called for  %d at %.6f[[[[[\n",CkMyNode(),CkMyPe(),m->redNo,CkWallTimer());
 #endif
+       /*
+               FAULT_EVAC
+       */
+       if(blocked){
+               DEBR(("[%d] This node is blocked, so remote message is being buffered as no %d\n",CkMyNode(),bufferedRemoteMsgs.length()));
+               bufferedRemoteMsgs.enq(m);
+               return;
+       }
+       
        if (isPresent(m->redNo)) { //Is a regular, in-order reduction message
            //DEBR((AA"Recv'd remote contribution %d for #%d at %d\n"AB,nRemote,m->redNo,this));
            startReduction(m->redNo,CkMyNode());
@@ -1281,12 +1318,11 @@ void CkNodeReductionMgr::doRecvMsg(CkReductionMsg *m){
            if (isFuture(m->redNo)) {
                   // DEBR((AA"Recv'd early remote contribution %d for #%d\n"AB,nRemote,m->redNo));
                    futureRemoteMsgs.enq(m);
-           }
-           else{
+           }else{
                   CkPrintf("BIG Problem Present %d Mesg RedNo %d \n",redNo,m->redNo);  
                   CkAbort("Recv'd late remote contribution!\n");
            }
-       }
+  }
 #if DEBUGRED        
        CkPrintf("[%d,%d]]]]] doRecvMsg called for  %d at %.6f\n",CkMyNode(),CkMyPe(),m->redNo,CkWallTimer());
 #endif       
@@ -1318,7 +1354,7 @@ void CkNodeReductionMgr::RecvMsg(CkReductionMsg *m)
 #endif 
 }
 
-void CkNodeReductionMgr::startReduction(int number,int srcPE)
+void CkNodeReductionMgr::startReduction(int number,int srcNode)
 {
        if (isFuture(number)) CkAbort("Can't start reductions out of order!\n");
        if (isPast(number)) CkAbort("Can't restart reduction that's already finished!\n");
@@ -1334,7 +1370,7 @@ void CkNodeReductionMgr::startReduction(int number,int srcPE)
        }
        
        //If none of these cases, we need to start the reduction--
-       DEBR((AA"Starting Node reduction #%d\n"AB,redNo));
+       DEBR((AA"Starting Node reduction #%d on %p srcNode %d\n"AB,redNo,this,srcNode));
        inProgress=CmiTrue;
        //Sent start requests to our kids (in case they don't already know)
 
@@ -1344,15 +1380,25 @@ void CkNodeReductionMgr::startReduction(int number,int srcPE)
                DEBR((AA"Asking child Node %d to start #%d\n"AB,kids[k],redNo));
                thisProxy[kids[k]].ReductionStarting(new CkReductionNumberMsg(redNo));
 #else
-               if(firstKid()+k != srcPE){
-                       DEBR((AA"Asking child Node %d to start #%d\n"AB,firstKid()+k,redNo));
-                       thisProxy[firstKid()+k].ReductionStarting(new CkReductionNumberMsg(redNo));
+               if(kids[k] != srcNode){
+                       DEBR((AA"Asking child Node %d to start #%d\n"AB,kids[k],redNo));
+                       thisProxy[kids[k]].ReductionStarting(new CkReductionNumberMsg(redNo));
                }       
 #endif
        }
+       startLocalGroupReductions(number);
 }
 
 void CkNodeReductionMgr::doAddContribution(CkReductionMsg *m){
+       /*
+               FAULT_EVAC
+       */
+       if(blocked){
+               DEBR(("[%d] This node is blocked, so local message is being buffered as no %d\n",CkMyNode(),bufferedMsgs.length()));
+               bufferedMsgs.enq(m);
+               return;
+       }
+       
        if (isFuture(m->redNo)) {//An early contribution-- add to future Q
                DEBR((AA"Contributor %p gives early node contribution-- for #%d\n"AB,m->ci,m->redNo));
                futureMsgs.enq(m);
@@ -1380,7 +1426,7 @@ up the reduction tree **/
 
 void CkNodeReductionMgr::finishReduction(void)
 {
-  DEBR((AA"in Nodegrp finishReduction %d \n"AB,inProgress));
+  DEBR((AA"in Nodegrp finishReduction %d treeKids %d \n"AB,inProgress,treeKids()));
   /***Check if reduction is finished in the next few ifs***/
   if ((!inProgress) | creating){
        DEBR((AA"Either not in Progress or creating\n"AB));
@@ -1410,11 +1456,16 @@ void CkNodeReductionMgr::finishReduction(void)
 
   if (hasParent())
   {//Pass data up tree to parent
-    DEBR((AA"Passing reduced data up to parent node %d. \n"AB,treeParent()));
+               if(CkpvAccess(_validProcessors)[CkMyNode()] || killed == 0){
+       DEBR((AA"Passing reduced data up to parent node %d. \n"AB,treeParent()));
 #if DEBUGRED
-    CkPrintf("[%d,%d] Passing data up to parentNode %d at %.6f for redNo %d with ncontrib %d\n",CkMyNode(),CkMyPe(),treeParent(),CkWallTimer(),redNo,nContrib);
+       CkPrintf("[%d,%d] Passing data up to parentNode %d at %.6f for redNo %d with ncontrib %d\n",CkMyNode(),CkMyPe(),treeParent(),CkWallTimer(),redNo,nContrib);
 #endif
-    thisProxy[treeParent()].RecvMsg(result);
+               /*
+                       FAULT_EVAC
+               */
+           thisProxy[treeParent()].RecvMsg(result);
+               }       
 
   }
   else
@@ -1438,7 +1489,7 @@ void CkNodeReductionMgr::finishReduction(void)
            storedCallback->send(result);
     }
     else{
-               DEBR((AA"Invalid Callback at %d %d\n"AB,result->callback,storedCallback));
+               DEBR((AA"Invalid Callback \n"AB));
            CkAbort("No reduction client!\n"
                    "You must register a client with either SetReductionClient or during contribute.\n");
        }
@@ -1447,6 +1498,7 @@ void CkNodeReductionMgr::finishReduction(void)
   // DEBR((AA"Reduction %d finished in group!\n"AB,redNo));
   //CkPrintf("[%d,%d]Reduction %d finished with %d\n",CkMyNode(),CkMyPe(),redNo,nContrib);
   redNo++;
+       updateTree();
   int i;
   inProgress=CmiFalse;
   startRequested=CmiFalse;
@@ -1485,6 +1537,20 @@ void CkNodeReductionMgr::finishReduction(void)
 }
 
 //////////// Reduction Manager Utilities /////////////
+
+void CkNodeReductionMgr::init_BinaryTree(){
+       parent = (CkMyNode()-1)/TREE_WID;
+       int firstkid = CkMyNode()*TREE_WID+1;
+       numKids=CkNumNodes()-firstkid;
+  if (numKids>TREE_WID) numKids=TREE_WID;
+  if (numKids<0) numKids=0;
+
+       for(int i=0;i<numKids;i++){
+               kids.push_back(firstkid+i);
+               newKids.push_back(firstkid+i);
+       }
+}
+
 void CkNodeReductionMgr::init_BinomialTree(){
        int depth = (int )ceil((log((double )CkNumNodes())/log((double)2)));
        /*upperSize = (unsigned )pow((double)2,depth);*/
@@ -1505,20 +1571,20 @@ void CkNodeReductionMgr::init_BinomialTree(){
        parent = upperSize -1 -parent;
        int temp;
        if(count != 0){
-               kids = new int[count];
                numKids = 0;
                for(int i=0;i<count;i++){
                        /*temp = label - rint(pow((double)2,i));*/
                        temp = label - (1<<i);
                        temp = upperSize-1-temp;
                        if(temp <= CkNumNodes()-1){
-                               kids[numKids] = temp;
+               //              kids[numKids] = temp;
+                               kids.push_back(temp);
                                numKids++;
                        }
                }
        }else{
                numKids = 0;
-               kids = NULL;
+       //      kids = NULL;
        }
 }
 
@@ -1536,7 +1602,7 @@ int CkNodeReductionMgr::treeParent(void) //My parent Node
 #ifdef BINOMIAL_TREE
        return parent;
 #else
-  return (CkMyNode()-1)/TREE_WID;
+  return parent;
 #endif
 }
 
@@ -1549,10 +1615,11 @@ int CkNodeReductionMgr::treeKids(void)//Number of children in tree
 #ifdef BINOMIAL_TREE
        return numKids;
 #else
-  int nKids=CkNumNodes()-firstKid();
+/*  int nKids=CkNumNodes()-firstKid();
   if (nKids>TREE_WID) nKids=TREE_WID;
   if (nKids<0) nKids=0;
-  return nKids;
+  return nKids;*/
+       return numKids;
 #endif
 }
 
@@ -1634,12 +1701,283 @@ void CkNodeReductionMgr::pup(PUP::er &p)
   p|msgs;
   p|futureMsgs;
   p|futureRemoteMsgs;
+       p|parent;
   if(p.isUnpacking()) {
     gcount=CkNumNodes();
     thisProxy = thisgroup;
     lockEverything = CmiCreateLock();
+#ifdef BINOMIAL_TREE
+               init_BinomialTree();
+#else
+               init_BinaryTree();
+#endif         
   }
+       p | blocked;
+       p | maxModificationRedNo;
+}
+
+/*
+       FAULT_EVAC
+       Evacuate - is called when this processor realizes it might crash. In that case, it tries to change 
+       the reduction tree. It also needs to decide a reduction number after which it shall use the new 
+       reduction tree. 
+*/
+void CkNodeReductionMgr::evacuate(){
+       DEBR(("[%d] Evacuate called on nodereductionMgr \n",CkMyNode()));
+       if(treeKids() == 0){
+       /*
+               if the node going down is a leaf
+       */
+               oldleaf=true;
+               DEBR(("[%d] Leaf Node marks itself for deletion when evacuation is complete \n",CkMyNode()));
+               /*
+                       Need to ask parent for the reduction number that it has seen. 
+                       Since it is a leaf, the tree does not need to be rewired. 
+                       We reuse the oldparent type of tree modification message to get 
+                       the parent to block and tell us about the highest reduction number it has seen.
+                       
+               */
+               int deleteNode=CkMyNode();
+               thisProxy[treeParent()].modifyTree(LEAFPARENT,1,&deleteNode);
+               newParent = treeParent();
+       }else{
+               DEBR(("[%d] Internal Node sends messages to change the redN tree \n",CkMyNode()));
+               oldleaf= false;
+       /*
+               It is not a leaf. It needs to rewire the tree around itself.
+               It also needs to decide on a reduction No after which the new tree will be used
+               Till it decides on the new tree and the redNo at which it becomes valid,
+               all received messages will be buffered
+       */
+               newParent = kids[0];
+               for(int i=numKids-1;i>=0;i--){
+                       newKids.remove(i);
+               }
+               /*
+                       Ask everybody for the highest reduction number they have seen and
+                       also tell them about the new tree
+               */
+               /*
+                       Tell parent about its new child;
+               */
+               int oldParentData[2];
+               oldParentData[0] = CkMyNode();
+               oldParentData[1] = newParent;
+               thisProxy[parent].modifyTree(OLDPARENT,2,oldParentData);
+
+               /*
+                       Tell the other children about their new parent
+               */
+               int childrenData=newParent;
+               for(int i=1;i<numKids;i++){
+                       thisProxy[kids[i]].modifyTree(OLDCHILDREN,1,&childrenData);
+               }
+               
+               /*
+                       Tell newParent (1st child) about its new children,
+                       the current node and its children except the newParent
+               */
+               int *newParentData = new int[numKids+1];
+               for(int i=1;i<numKids;i++){
+                       newParentData[i] = kids[i];
+               }
+               newParentData[0] = CkMyNode();
+               newParentData[numKids] = parent;
+               thisProxy[newParent].modifyTree(NEWPARENT,numKids+1,newParentData);
+       }
+       readyDeletion = false;
+       blocked = true;
+       numModificationReplies = 0;
+       tempModificationRedNo = findMaxRedNo();
+}
+
+/*
+       Depending on the code, use the data to change the tree
+       1. OLDPARENT : replace the old child with a new one
+       2. OLDCHILDREN: replace the parent
+       3. NEWPARENT:  add the children and change the parent
+       4. LEAFPARENT: delete the old child
+*/
+
+void CkNodeReductionMgr::modifyTree(int code,int size,int *data){
+       DEBR(("[%d] Received modifyTree request with code %d \n",CkMyNode(),code));
+       int sender;
+       newKids = kids;
+       readyDeletion = false;
+       switch(code){
+               case OLDPARENT: 
+                       for(int i=0;i<numKids;i++){
+                               if(newKids[i] == data[0]){
+                                       newKids[i] = data[1];
+                                       break;
+                               }
+                       }
+                       sender = data[0];
+                       newParent = parent;
+                       break;
+               case OLDCHILDREN:
+                       newParent = data[0];
+                       sender = parent;
+                       break;
+               case NEWPARENT:
+                       for(int i=0;i<size-1;i++){
+                               newKids.push_back(data[i]);
+                       }
+                       newParent = data[size-1];
+                       sender = parent;
+                       break;
+               case LEAFPARENT:
+                       sender = data[0];
+                       newParent = parent;
+                       break;
+       };
+       blocked = true;
+       int maxRedNo = findMaxRedNo();
+       
+       thisProxy[sender].collectMaxRedNo(maxRedNo);
+}
+
+void CkNodeReductionMgr::collectMaxRedNo(int maxRedNo){
+       /*
+               Find out the maximum redNo that has been seen by 
+               the affected nodes
+       */
+       numModificationReplies++;
+       if(maxRedNo > tempModificationRedNo){
+               tempModificationRedNo = maxRedNo;
+       }
+       if(numModificationReplies == numKids+1){
+               maxModificationRedNo = tempModificationRedNo;
+               /*
+                       when all the affected nodes have replied, tell them the maximum.
+                       Unblock yourself. deal with the buffered messages local and remote
+               */
+               if(maxModificationRedNo == -1){
+                       printf("[%d] This array has not started reductions yet \n",CkMyNode());
+               }else{
+                       DEBR(("[%d] maxModificationRedNo for this nodegroup %d \n",CkMyNode(),maxModificationRedNo));
+               }
+               thisProxy[parent].unblockNode(maxModificationRedNo);
+               for(int i=0;i<numKids;i++){
+                       thisProxy[kids[i]].unblockNode(maxModificationRedNo);
+               }
+               blocked = false;
+               updateTree();
+               clearBlockedMsgs();
+       }
+};
+
+void CkNodeReductionMgr::unblockNode(int maxRedNo){
+       maxModificationRedNo = maxRedNo;
+       updateTree();
+       blocked = false;
+       clearBlockedMsgs();
 }
 
 
+void CkNodeReductionMgr::clearBlockedMsgs(){
+       int len = bufferedMsgs.length();
+       for(int i=0;i<len;i++){
+               CkReductionMsg *m = bufferedMsgs.deq();
+               doAddContribution(m);
+       }
+       len = bufferedRemoteMsgs.length();
+       for(int i=0;i<len;i++){
+               CkReductionMsg *m = bufferedRemoteMsgs.deq();
+               doRecvMsg(m);
+       }
+
+}
+/*
+       if the reduction number exceeds the maxModificationRedNo, change the tree
+       to become the new one
+*/
+void CkNodeReductionMgr::updateTree(){
+       if(redNo > maxModificationRedNo){
+               parent = newParent;
+               kids = newKids;
+               maxModificationRedNo = INT_MAX;
+               numKids = kids.size();
+               readyDeletion = true;
+               DEBR(("[%d] Updating Tree numKids %d \n",CkMyNode(),numKids));
+       }
+}
+
+
+void CkNodeReductionMgr::doneEvacuate(){
+       DEBR(("[%d] doneEvacuate called \n",CkMyNode()));
+/*     if(oldleaf){
+               
+                       It used to be a leaf
+                       Then as soon as future messages have been emptied you can 
+                       send the parent a message telling them that they are not going
+                       to receive anymore messages from this child
+               
+               DEBR(("[%d] At the end of evacuation emptying future messages %d \n",CkMyNode(),futureMsgs.length()));
+               while(futureMsgs.length() != 0){
+                       int n = futureMsgs.length();
+                       for(int i=0;i<n;i++){
+                               CkReductionMsg *m = futureMsgs.deq();
+                               if(isPresent(m->redNo)){
+                                       msgs.enq(m);
+                               }else{
+                                       futureMsgs.enq(m);
+                               }
+                       }
+                       CkReductionMsg *result = reduceMessages();
+                       thisProxy[treeParent()].RecvMsg(result);
+                       redNo++;
+               }
+               DEBR(("[%d] Asking parent %d to remove myself from list \n",CkMyNode(),treeParent()));
+               thisProxy[treeParent()].DeleteChild(CkMyNode());
+       }else{*/
+               if(readyDeletion){
+                       thisProxy[treeParent()].DeleteChild(CkMyNode());
+               }else{
+                       thisProxy[newParent].DeleteNewChild(CkMyNode());
+               }
+//     }
+}
+
+void CkNodeReductionMgr::DeleteChild(int deletedChild){
+       DEBR(("[%d] Deleting child %d \n",CkMyNode(),deletedChild));
+       for(int i=0;i<numKids;i++){
+               if(kids[i] == deletedChild){
+                       kids.remove(i);
+                       break;
+               }
+       }
+       numKids = kids.length();
+       finishReduction();
+}
+
+void CkNodeReductionMgr::DeleteNewChild(int deletedChild){
+       for(int i=0;i<newKids.length();i++){
+               if(newKids[i] == deletedChild){
+                       newKids.remove(i);
+                       break;
+               }
+       }
+       DEBR(("[%d] Deleting  new child %d readyDeletion %d newKids %d \n",CkMyNode(),deletedChild,readyDeletion,newKids.size()));
+       finishReduction();
+}
+
+int CkNodeReductionMgr::findMaxRedNo(){
+       int max = redNo;
+       for(int i=0;i<futureRemoteMsgs.length();i++){
+               if(futureRemoteMsgs[i]->redNo  > max){
+                       max = futureRemoteMsgs[i]->redNo;
+               }
+       }
+       /*
+               if redNo is max (that is no future message) and the current reduction has not started
+               then tree can be changed before the reduction redNo can be started
+       */ 
+       if(redNo == max && msgs.length() == 0){
+               DEBR(("[%d] Redn %d has not received any contributions \n",CkMyNode(),max));
+               max--;
+       }
+       return max;
+}
+
 #include "CkReduction.def.h"
index 8373a9e1259a1b40f9c90b2fe50c4dc781190b77..ab385c3e505d83171434f4bb476bca993db88905 100644 (file)
@@ -29,17 +29,20 @@ module CkReduction {
   };
 
   nodegroup CkNodeReductionMgr : IrrGroup {
-       entry CkNodeReductionMgr(void);
-
+  entry CkNodeReductionMgr(void);
 
        //Sent down the reduction tree (used by barren PEs)
        entry  void ReductionStarting(CkReductionNumberMsg *);
-       //entry  void ReductionStarting(CkReductionNumberMsg *);
 
        //Sent up the reduction tree with reduced data
-       //entry [immediate] void RecvMsg(CkReductionMsg *);
-        entry void RecvMsg(CkReductionMsg *);
-
+       entry void RecvMsg(CkReductionMsg *);
+
+       entry void DeleteChild(int child);
+       entry void DeleteNewChild(int child);
+       entry void modifyTree(int code,int size,int data[size]);
+       entry void collectMaxRedNo(int redNo);
+       entry void unblockNode(int maxRedNo);
+       
   };
 
 
index 522652f48138ed191aa75bd8d0a7cbf537051d41..125d77b7ba08a42068e384da64c50ec9f3d27ee9 100644 (file)
@@ -65,7 +65,12 @@ protected:
 class CkReductionMsg;
 
 
-class CkReductionNumberMsg;
+class CkReductionNumberMsg:public CMessage_CkReductionNumberMsg {
+public:
+  int num;
+  CkReductionNumberMsg(int n) {num=n;}
+};
+
 
 /**
  * One CkReductionMgr runs a non-overlapping set of reductions.
@@ -168,13 +173,13 @@ private:
        void addContribution(CkReductionMsg *m);
        void finishReduction(void);
 
-//Reduction tree utilities
+/*//Reduction tree utilities
        enum {TREE_WID=2};
        int treeRoot(void);//Root PE
        CmiBool hasParent(void);
        int treeParent(void);//My parent PE
        int firstKid(void);//My first child PE
-       int treeKids(void);//Number of children in tree
+       int treeKids(void);//Number of children in tree*/
 
        //Combine (& free) the current message vector.
        CkReductionMsg *reduceMessages(void);
index 092f1b7bd81136f755797b260b90e3a352c80635..627a0833f5ea82dd63488909de83ea2226f78a60 100644 (file)
@@ -104,7 +104,13 @@ CkpvStaticDeclare(int,  _numInitsRecd); /* UInt changed to int */
 CkpvStaticDeclare(PtrQ*, _buffQ);
 CkpvStaticDeclare(PtrVec*, _bocInitVec);
 
-static int    _exitHandlerIdx;
+/*
+       FAULT_EVAC
+*/
+CkpvDeclare    (char *, _validProcessors);
+CpvDeclare(char ,startedEvac);
+
+int    _exitHandlerIdx;
 
 static Stats** _allStats = 0;
 
@@ -128,6 +134,13 @@ static char* _restartDir;
 int _defaultObjectQ = 0;            // for obejct queue
 int _ringexit = 0;                 // for charm exit
 
+/*
+       FAULT_EVAC
+
+       flag which marks whether or not to trigger the processor shutdowns
+*/
+int _raiseEvac=0;
+
 static inline void _parseCommandLineOpts(char **argv)
 {
   if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
@@ -173,6 +186,14 @@ static inline void _parseCommandLineOpts(char **argv)
     if (CkMyPe()==0)
       CkPrintf("Charm++> Program shutdown in ring.\n");
   }
+       /*
+               FAULT_EVAC
+
+               if the argument +raiseevac is present then cause faults
+       */
+       if(CmiGetArgFlagDesc(argv,"+raiseevac","Generates processor evacuation on random processors")){
+               _raiseEvac = 1;
+       }
 }
 
 static void _bufferHandler(void *msg)
@@ -265,17 +286,26 @@ static void _exitHandler(envelope *env)
       env->setMsgtype(ReqStatMsg);
       env->setSrcPe(CkMyPe());
       // if exit in ring, instead of broadcasting, send in ring
-      if (_ringexit)
+      if (_ringexit){
+                               DEBUGF(("[%d] Ring Exit \n",CkMyPe()));
         CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
-      else
-        CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
+      }else{
+             //  CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
+                               /*FAULT_EVAC*/
+                               for(int i=0;i<CkNumPes();i++){
+               CmiSyncSend(i, env->getTotalsize(), (char *)env);
+                               }       
+                       }       
       break;
     case ReqStatMsg:
       DEBUGF(("ReqStatMsg on %d\n", CkMyPe()));
       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
       CkNumberHandler(_nodeBocHandlerIdx, (CmiHandler)_discardHandler);
-      _sendStats();
+                       /*FAULT_EVAC*/
+                       if(CpvAccess(_validProcessors)[CkMyPe()]){
+             _sendStats();
+                       }       
       _mainDone = 1; // This is needed because the destructors for
                      // readonly variables will be called when the program
                     // exits. If the destructor is called while _mainDone
@@ -289,8 +319,10 @@ static void _exitHandler(envelope *env)
         CmiSyncSendAndFree(CkMyPe()+1, env->getTotalsize(), (char *)env);
       else
         CmiFree(env);
-      if(CkMyPe())
+      if(CkMyPe()){
+                               DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
         ConverseExit();
+                       }       
       break;
     case StatMsg:
       CkAssert(CkMyPe()==0);
@@ -299,8 +331,10 @@ static void _exitHandler(envelope *env)
 #endif
       _numStatsRecd++;
       DEBUGF(("StatMsg on %d with %d\n", CkMyPe(), _numStatsRecd));
-      if(_numStatsRecd==CkNumPes()) {
+                       /*FAULT_EVAC*/
+      if(_numStatsRecd==CkNumValidPes()) {
         _printStats();
+                               DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
         ConverseExit();
       }
       break;
@@ -495,7 +529,12 @@ void _CkExit(void)
     envelope *env = _allocEnv(ReqStatMsg);
     env->setSrcPe(CkMyPe());
     CmiSetHandler(env, _exitHandlerIdx);
-    CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
+               /*FAULT_EVAC*/
+ //   CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
+    for(int i=0;i<CmiNumPes();i++){
+                               CmiSyncSend(i,env->getTotalsize(),(char *)env);
+               }
+       
   } else {
     envelope *env = _allocEnv(ExitMsg);
     env->setSrcPe(CkMyPe());
@@ -518,6 +557,8 @@ CkQ<CkExitFn> _CkExitFnVec;
 extern "C"
 void CkExit(void)
 {
+       /*FAULT_EVAC*/
+       DEBUGF(("[%d] CkExit called \n",CkMyPe()));
   if (!_CkExitFnVec.isEmpty()) {
     CkExitFn fn = _CkExitFnVec.deq();
     fn();
@@ -592,6 +633,13 @@ void _initCharm(int unused_argc, char **argv)
        CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
        CkpvInitialize(MsgPool*, _msgPool);
        CkpvInitialize(CkCoreState *, _coreState);
+       /*
+               Added for evacuation-sayantan
+       */
+       CkpvInitialize(char *,_validProcessors);
+       CkpvInitialize(char ,startedEvac);
+       CkpvInitialize(int,serializer);
+       
 
        CksvInitialize(UInt, _numNodeGroups);
        CksvInitialize(GroupTable*, _nodeGroupTable);
@@ -757,7 +805,25 @@ void _initCharm(int unused_argc, char **argv)
        if (!inCommThread) {
          _TRACE_BEGIN_COMPUTATION();
        }
-
+       /*
+               FAULT_EVAC
+       */
+       CkpvAccess(_validProcessors) = new char[CkNumPes()];
+       CpvAccess(startedEvac) = 0;
+       _ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
+       _ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
+       CkpvAccess(serializer) = 0;
+
+       for(int vProc=0;vProc<CkNumPes();vProc++){
+               CkpvAccess(_validProcessors)[vProc]=1;
+       }
+       evacuate = 0;
+       CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
+       if(CkMyPe() == 1 && _raiseEvac){
+       //      CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
+               CcdCallOnCondition(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
+       }       
+       
        if (faultFunc) {
                if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
                if (!inCommThread) faultFunc(_restartDir);
index f6bfc33b5858274b47fb78d44a400cf1ba62928b..2476f2008932634737a36077a8337519886f3919 100644 (file)
@@ -16,7 +16,8 @@
 #include "LBDBManager.h"
 #include "LBSimulation.h"
 
-#define  DEBUGF(x)      // CmiPrintf x;
+//#define  DEBUGF(x)       CmiPrintf x;
+#define DEBUGF(x) 
 
 CkGroupID loadbalancer;
 int * lb_ptr;
@@ -157,7 +158,7 @@ void CentralLB::AtSync()
 void CentralLB::ProcessAtSync()
 {
 #if CMK_LBDB_ON
-
+       CmiAssert(CkpvAccess(_validProcessors)[CkMyPe()]);
   if (CkMyPe() == cur_ld_balancer) {
     start_lb_time = CkWallTimer();
   }
@@ -178,10 +179,10 @@ void CentralLB::ProcessAtSync()
   theLbdb->GetTime(&msg->total_walltime,&msg->total_cputime,
                   &msg->idletime, &msg->bg_walltime,&msg->bg_cputime);
   msg->pe_speed = myspeed;
-//  CkPrintf(
-//    "Processors %d Total time (wall,cpu) = %f %f Idle = %f Bg = %f %f\n",
-//    CkMyPe(),msg->total_walltime,msg->total_cputime,
-//    msg->idletime,msg->bg_walltime,msg->bg_cputime);
+  DEBUGF((
+    "Processors %d Total time (wall,cpu) = %f %f Idle = %f Bg = %f %f\n",
+    CkMyPe(),msg->total_walltime,msg->total_cputime,
+    msg->idletime,msg->bg_walltime,msg->bg_cputime));
 
   msg->n_objs = osz;
   theLbdb->GetObjData(msg->objData);
@@ -222,9 +223,9 @@ void CentralLB::Migrated(LDObjHandle h, int waitBarrier)
 {
 #if CMK_LBDB_ON
   if (waitBarrier) {
-    migrates_completed++;
-    //  CkPrintf("[%d] An object migrated! %d %d\n",
-    //            CkMyPe(),migrates_completed,migrates_expected);
+           migrates_completed++;
+      DEBUGF(("[%d] An object migrated! %d %d\n",
+          CkMyPe(),migrates_completed,migrates_expected));
     if (migrates_completed == migrates_expected) {
       MigrationDone(1);
     }
@@ -259,10 +260,13 @@ void CentralLB::buildStats()
     int nobj = 0;
     int ncom = 0;
     int nmigobj = 0;
-    // copy all data from individule message to this big structure
-    for (int pe=0; pe<stats_msg_count; pe++) {
+    // copy all data in individule message to this big structure
+    for (int pe=0; pe<CkNumPes(); pe++) {
        int i;
        CLBStatsMsg *msg = statsMsgsList[pe];
+                        if(msg == NULL){
+                               continue;
+                        }
        for (i=0; i<msg->n_objs; i++) {
          statsData->from_proc[nobj] = statsData->to_proc[nobj] = pe;
         statsData->objData[nobj] = msg->objData[i];
@@ -288,11 +292,14 @@ void CentralLB::ReceiveStats(CkMarshalledCLBStatsMessage &msg)
 #if CMK_LBDB_ON
   CLBStatsMsg *m = (CLBStatsMsg *)msg.getMessage();
   const int pe = m->from_pe;
-//  CkPrintf("Stats msg received, %d %d %d %p\n",
-//        pe,stats_msg_count,m->n_objs,m);
-
-  // update proc avail bit vector
-//  if (pe == cur_ld_balancer && m->avail_vector) {
+  DEBUGF(("Stats msg received, %d %d %d %d %p\n",
+          pe,stats_msg_count,m->n_objs,m->serial,m));
+       
+       if(!CkpvAccess(_validProcessors)[pe]){
+               DEBUGF(("[%d] ReceiveStats called from invalidProcessor %d\n",CkMyPe(),pe));
+               return;
+       }
+       
   if (m->avail_vector) {
       LBDatabaseObj()->set_avail_vector(m->avail_vector,  m->next_lb);
   }
@@ -321,7 +328,7 @@ void CentralLB::ReceiveStats(CkMarshalledCLBStatsMessage &msg)
   }
 
   DEBUGF(("[0] ReceiveStats from %d step: %d count: %d\n", pe, step(), stats_msg_count));
-  const int clients = CkNumPes();
+  const int clients = CkNumValidPes();
 
   if (stats_msg_count == clients) {
     thisProxy[CkMyPe()].LoadBalance();
@@ -378,7 +385,7 @@ void CentralLB::LoadBalance()
       getPredictedLoadWithMsg(statsData, clients, migrateMsg, info, 0);
   }
 
-  //  CkPrintf("calling recv migration\n");
+  DEBUGF(("[%d]calling recv migration\n",CkMyPe()));
   thisProxy.ReceiveMigration(migrateMsg);
 
   // Zero out data structures for next cycle
@@ -497,7 +504,12 @@ void CentralLB::ReceiveMigration(LBMigrateMsg *m)
 #if CMK_LBDB_ON
   int i;
   for (i=0; i<CkNumPes(); i++) theLbdb->lastLBInfo.expectedLoad[i] = m->expectedLoad[i];
-  
+  CmiAssert(migrates_expected <= 0 || migrates_completed == migrates_expected);
+/*FAULT_EVAC*/
+       if(!(CkpvAccess(_validProcessors)[CkMyPe()])){
+               delete m;
+               return;
+       }
   migrates_expected = 0;
   future_migrates_expected = 0;
   for(i=0; i < m->n_moves; i++) {
@@ -509,7 +521,7 @@ void CentralLB::ReceiveMigration(LBMigrateMsg *m)
       if (theLbdb->Migrate(move.obj,move.to_pe) == 0) 
          thisProxy[move.to_pe].MissMigrate(!move.async_arrival);
     } else if (move.from_pe != me && move.to_pe == me) {
-      // CkPrintf("[%d] expecting object from %d\n",move.to_pe,move.from_pe);
+       DEBUGF(("[%d] expecting object from %d\n",move.to_pe,move.from_pe));
       if (!move.async_arrival) migrates_expected++;
       else future_migrates_expected++;
     }
@@ -521,7 +533,6 @@ void CentralLB::ReceiveMigration(LBMigrateMsg *m)
     theLbdb->SetLBPeriod(theLbdb->GetLBPeriod()*2);
   }
 #endif
-
   cur_ld_balancer = m->next_lb;
   if((CkMyPe() == cur_ld_balancer) && (cur_ld_balancer != 0)){
       LBDatabaseObj()->set_avail_vector(m->avail_vector, -2);
@@ -530,6 +541,8 @@ void CentralLB::ReceiveMigration(LBMigrateMsg *m)
   if (migrates_expected == 0 || migrates_completed == migrates_expected)
     MigrationDone(1);
   delete m;
+
+//     CkEvacuatedElement();
 #endif
 }
 
@@ -543,6 +556,8 @@ void CentralLB::MigrationDone(int balancing)
   if (balancing) theLbdb->ClearLoads();
   // Increment to next step
   theLbdb->incStep();
+       DEBUGF(("[%d] Incrementing Step %d \n",CkMyPe(),step()));
+  // if sync resume, invoke a barrier
 
   LoadbalanceDone(balancing);        // callback
 
index 729c2d1d35715a8d72722626d320b8effc92df04..c8cd075994508b5ef12931674c6b9e2e349064bd 100644 (file)
@@ -151,10 +151,10 @@ GreedyLB::BuildCpuArray(BaseLB::LDStats* stats,
   for (pe=0; pe < count; pe++) {
     CentralLB::ProcStats &peData = stats->procs[pe];
  
-    data[*peCount].load = 0.0;
     map[pe] = -1;
     if (peData.available) 
     {
+       data[*peCount].load = 0.0;
       data[*peCount].load += peData.bg_walltime;
       data[*peCount].pe = data[*peCount].id = pe;
       map[pe] = *peCount;
@@ -188,14 +188,15 @@ void GreedyLB::work(BaseLB::LDStats* stats, int count)
 {
   int  obj, heapSize, objCount;
   int *pemap = new int [count];
+       CmiMemoryCheck();
   HeapData *cpuData = BuildCpuArray(stats, count, &heapSize);
+       CmiMemoryCheck();
   HeapData *objData = BuildObjectArray(stats, count, &objCount);
-
   if (_lb_args.debug()>1) 
     CkPrintf("[%d] In GreedyLB strategy\n",CkMyPe());
-
   heapSize--;
   for (obj=0; obj < objCount; obj++) {
+               CmiMemoryCheck();
     HeapData minCpu;  
     // Operation of extracting the least loaded processor
     // from the heap
@@ -203,6 +204,7 @@ void GreedyLB::work(BaseLB::LDStats* stats, int count)
     cpuData[0] = cpuData[heapSize];
     heapSize--;
     Heapify(cpuData, 0, heapSize, LT);    
+               CmiMemoryCheck();
 
     // Increment the time of the least loaded processor by the cpuTime of
     // the `heaviest' object
@@ -226,6 +228,7 @@ void GreedyLB::work(BaseLB::LDStats* stats, int count)
       location = (location-1)/2;
     }
     cpuData[location] = minCpu;
+               CmiMemoryCheck();
   }
 
   delete [] cpuData;
index e788f660c8ecedffc091e9fdbfd86d34563e5113..38e530cccb903ad3e5a71a5202883161232f4f11 100644 (file)
@@ -483,7 +483,7 @@ void LocalBarrier::CheckBarrier()
     for(int i=0; i < max_client; i++)
       if (clients[i] != 0 && ((client*)clients[i])->refcount >= cur_refcount)
        at_barrier = CmiTrue;
-
+               
     if (at_barrier) {
       at_count -= client_count;
       cur_refcount++;
index cb7907788490dd7c4c5bbeac49691493598e3b80..932fe560940744efa807d414e8b4acad59160dba 100644 (file)
@@ -73,8 +73,7 @@ void RotateLB::work (BaseLB::LDStats *stats, int count)
       }
       if (dest != stats->from_proc[obj]) {
        stats->to_proc[obj] = dest;
-        //CkPrintf ("[%d] Object %d is migrating from PE %d to PE %d\n",
-       //          CkMyPe (), obj, stats->from_proc[obj], dest);
+   //     CkPrintf ("[%d] Object %d is migrating from PE %d to PE %d\n",CkMyPe (), obj, stats->from_proc[obj], dest);
       }
     }
   }
index 1648a912545dfd7eec15d7498bd1af626ca10395..543214a212e9dcdd9ec3445ad455473e2d1c03f5 100644 (file)
@@ -378,7 +378,7 @@ LDRemoveLocalBarrierReceiver(LDHandle _db,LDBarrierReceiver h)
 extern "C" void LDAtLocalBarrier(LDHandle _db, LDBarrierClient h)
 {
   LBDB *const db = (LBDB*)(_db.handle);
-
+       
   db->AtLocalBarrier(h);
 }
 
index f48504eeed8adb32bb7f5956abae7851c62a8cbe..1ff2233a7b73c9c1b4787777366537aa87918cfb 100644 (file)
@@ -1109,6 +1109,9 @@ void CsdEndIdle(void)
 #define MESSAGE_PHASE_CHECK
 #endif
 
+CpvExtern(char *,_validProcessors);
+extern int _exitHandlerIdx;
+
 void CmiHandleMessage(void *msg)
 {
 /* this is wrong because it counts the Charm++ messages in sched queue
@@ -1120,6 +1123,13 @@ void CmiHandleMessage(void *msg)
        _LOG_E_HANDLER_BEGIN(handler); /* projector */
 #endif
 
+/*
+       FAULT_EVAC
+*/
+/*     if((!CpvAccess(_validProcessors)[CmiMyPe()]) && handler != _exitHandlerIdx){
+               return;
+       }*/
+       
         MESSAGE_PHASE_CHECK
 
        h=&CmiGetHandlerInfo(msg);
@@ -1235,6 +1245,19 @@ int CsdScheduler(int maxmsgs)
        CsdEndIdle();\
        break;\
       }\
+/*
+       EVAC
+*/
+extern void CkClearAllArrayElements();
+CmiUInt2 handler1;
+void *__dbgMsg;
+void __dbgcheckMessageHandler(){
+       CmiUInt2 handler2;
+       if(__dbgMsg){
+                       handler2=CmiGetHandler(__dbgMsg); 
+       //              CmiAssert(handler2 != 0);
+       }
+}
 
 void CsdScheduleForever(void)
 {
@@ -1242,6 +1265,12 @@ void CsdScheduleForever(void)
   SCHEDULE_TOP
   while (1) {
     msg = CsdNextMessage(&state);
+               __dbgMsg = msg;
+               if(msg){
+                       handler1=CmiGetHandler(msg); 
+               }       
+//             CkClearAllArrayElements();
+               __dbgcheckMessageHandler();
     if (msg) { /*A message is available-- process it*/
       if (isIdle) {isIdle=0;CsdEndIdle();}
       SCHEDULE_MESSAGE
@@ -1350,7 +1379,7 @@ CthThread CthSuspendNormalThread()
   return CpvAccess(CthSchedulingThread);
 }
 
-void CthEnqueueSchedulingThread(CthThread t, int, int, unsigned int*);
+void CthEnqueueSchedulingThread(CthThreadToken *token, int, int, unsigned int*);
 CthThread CthSuspendSchedulingThread();
 
 CthThread CthSuspendSchedulingThread()
@@ -1370,8 +1399,13 @@ CthThread CthSuspendSchedulingThread()
   return succ;
 }
 
-void CthResumeNormalThread(CthThread t)
+void CthResumeNormalThread(CthThreadToken* token)
 {
+       CthThread t = token->thread;
+       if(t == NULL){
+        free(token);
+        return;
+       }
 #ifndef CMK_OPTIMIZE
 #if ! CMK_TRACE_IN_CHARM
   if(CpvAccess(traceOn))
@@ -1384,11 +1418,12 @@ void CthResumeNormalThread(CthThread t)
   CthResume(t);
 }
 
-void CthResumeSchedulingThread(CthThread t)
+void CthResumeSchedulingThread(CthThreadToken  *token)
 {
+       CthThread t = token->thread;
   CthThread me = CthSelf();
   if (me == CpvAccess(CthMainThread)) {
-    CthEnqueueSchedulingThread(me,CQS_QUEUEING_FIFO, 0, 0);
+    CthEnqueueSchedulingThread(CthGetToken(me),CQS_QUEUEING_FIFO, 0, 0);
   } else {
     CthSetNext(me, CpvAccess(CthSleepingStandins));
     CpvAccess(CthSleepingStandins) = me;
@@ -1405,18 +1440,18 @@ void CthResumeSchedulingThread(CthThread t)
   CthResume(t);
 }
 
-void CthEnqueueNormalThread(CthThread t, int s, 
+void CthEnqueueNormalThread(CthThreadToken* token, int s, 
                                   int pb,unsigned int *prio)
 {
-  CmiSetHandler(t, CpvAccess(CthResumeNormalThreadIdx));
-  CsdEnqueueGeneral(t, s, pb, prio);
+  CmiSetHandler(token, CpvAccess(CthResumeNormalThreadIdx));
+  CsdEnqueueGeneral(token, s, pb, prio);
 }
 
-void CthEnqueueSchedulingThread(CthThread t, int s, 
+void CthEnqueueSchedulingThread(CthThreadToken* token, int s, 
                                       int pb,unsigned int *prio)
 {
-  CmiSetHandler(t, CpvAccess(CthResumeSchedulingThreadIdx));
-  CsdEnqueueGeneral(t, s, pb, prio);
+  CmiSetHandler(token, CpvAccess(CthResumeSchedulingThreadIdx));
+  CsdEnqueueGeneral(token, s, pb, prio);
 }
 
 void CthSetStrategyDefault(CthThread t)
index 6b8678c1f65a61c1d516565cbb9fc06913f58667..d15af0fa34fef768ea15a2412da78ab4eb5593f3 100644 (file)
@@ -940,9 +940,18 @@ void CmiIsomallocBlockListFree(void *doomedMallocedBlock);
 /****** CTH: THE LOW-LEVEL THREADS PACKAGE ******/
 
 typedef struct CthThreadStruct *CthThread;
+typedef struct {
+       /*Start with a message header so threads can be enqueued 
+    as messages (e.g., by CthEnqueueNormalThread in convcore.c)
+  */
+  char cmicore[CmiMsgHeaderSizeBytes];
+       CthThread thread;
+} CthThreadToken;
+
+CthThreadToken *CthGetToken(CthThread);
 
 typedef void        (*CthVoidFn)();
-typedef void        (*CthAwkFn)(CthThread,int,
+typedef void        (*CthAwkFn)(CthThreadToken *,int,
                                int prioBits,unsigned int *prioptr);
 typedef CthThread   (*CthThFn)();
 
index 9cd3e5f39c20ba8f0694f836f06ba725c1c2ddc9..003407a44857827ee086825670271efb0b3578ac 100644 (file)
@@ -18,7 +18,7 @@ generalized by Orion Lawlor November 2001.
 #include "converse.h"
 #include "memory-isomalloc.h"
 
-/* #define CMK_THREADS_DEBUG 1 */
+#define CMK_THREADS_DEBUG 0 
 
 #include <stdio.h>
 #include <stdlib.h>
index 4fd3e74590b03243f75fb9577b4de991e9ba27f1..98c70a9147e677ef43769d732abc7bd9d82affdf 100644 (file)
@@ -45,7 +45,7 @@ static void *meta_malloc(size_t size)
 }
 
 static void meta_free(void *mem)
-{
+{      
        if (CmiIsomallocInRange(mem)) 
        { /*Unlink this slot and isofree*/
                ISOMALLOC_PUSH
index faef1f5bbddc2b4b19bde181b9afe10b5879fbe8..6187d2a3c329235ab7b759e7e751bccc685b4f15 100644 (file)
@@ -24,7 +24,7 @@ quiescence detection!
 #include <stdio.h>
 #include <stdlib.h>
 #ifndef  DEBUGF
-#define  DEBUGF(x) /*printf x*/ 
+#define  DEBUGF(x) printf x 
 #endif
 
 CpvDeclare(CQdState, cQdState);
index 334933555e1e0ef5cfc82e3ea6872582f915eef6..372fc985ebe5a77710bf7593d9b6f6b30d5f071f 100644 (file)
 #endif
 
 /**************************** Shared Base Thread Class ***********************/
+/*
+       FAULT_EVAC
+       Moved the cmicore converse header from CthThreadBase to CthThreadToken.
+       The CthThreadToken gets enqueued in the converse queue instead of the
+       CthThread. This allows the thread to be moved out of a processor even
+       if there is an awaken call for it enqueued in the scheduler
+
+*/
+
 typedef struct CthThreadBase
 {
-  /*Start with a message header so threads can be enqueued 
-    as messages (e.g., by CthEnqueueNormalThread in convcore.c)
-  */
-  char cmicore[CmiMsgHeaderSizeBytes];
-  
+       CthThreadToken *token; /* token that shall be enqueued into the ready queue*/
+       int scheduled;                           /* has this thread been added to the ready queue ?
+                                                                                                       */
   CmiObjId   tid;        /* globally unique tid */
   CthAwkFn   awakenfn;   /* Insert this thread into the ready queue */
   CthThFn    choosefn;   /* Return the next ready thread */
@@ -165,6 +173,10 @@ typedef struct CthThreadBase
 #define S(t) ((CthThread)(t))
 
 
+CthThreadToken *CthGetToken(CthThread t){
+       return B(t)->token;
+}
+
 /*********************** Stack Aliasing *********************
   Stack aliasing: instead of consuming virtual address space
   with isomalloc, map all stacks to the same virtual addresses
@@ -340,6 +352,9 @@ static void CthThreadBaseInit(CthThreadBase *th)
   __pthread_find_self_with_pid=1;
   __pthread_nonstandard_stacks=1;
 #endif
+       th->token = (CthThreadToken *)malloc(sizeof(CthThreadToken));
+       th->token->thread = S(th);
+       th->scheduled = 0;
 
   th->awakenfn = 0;
   th->choosefn = 0;
@@ -387,6 +402,14 @@ static void *CthAllocateStack(CthThreadBase *th,int *stackSize,int useMigratable
 }
 static void CthThreadBaseFree(CthThreadBase *th)
 {
+       /*
+        * remove the token if it is not queued in the converse scheduler               
+        */
+       if(th->scheduled == 0){
+               free(th->token);
+       }else{
+               th->token->thread = NULL;
+       }
        /* Call the free function pointer on all the listeners on
                        this thread and also delete the thread listener objects
        */
@@ -443,6 +466,13 @@ void CthPupBase(pup_er p,CthThreadBase *t,int useMigratable)
        if ((CthThread)t==CthCpvAccess(CthCurrent))
                CmiAbort("CthPupBase: Cannot pack running thread!");
 #endif
+       /*recreate the token when unpacking and set the thread
+         pointer in it to NULL if the thread is being deleted*/
+       if(pup_isUnpacking(p)){
+               t->token = (CthThreadToken *)malloc(sizeof(CthThreadToken));
+               t->token->thread = S(t);
+               t->scheduled = 0;
+       }
        /*Really need a pup_functionPtr here:*/
        pup_bytes(p,&t->awakenfn,sizeof(t->awakenfn));
        pup_bytes(p,&t->choosefn,sizeof(t->choosefn));
@@ -505,6 +535,7 @@ void CthSetStrategy(CthThread t, CthAwkFn awkfn, CthThFn chsfn)
 
 static void CthBaseResume(CthThread t)
 {
+       
        struct CthThreadListener *l;
        for(l=B(t)->listener;l!=NULL;l=l->next){
                        l->resume(l);
@@ -542,9 +573,9 @@ void CthSuspend(void)
        for(l=cur->listener;l!=NULL;l=l->next){
                        l->suspend(l);
        }
-       
   if (cur->choosefn == 0) CthNoStrategy();
   next = cur->choosefn();
+       cur->scheduled=0;
 #ifndef CMK_OPTIMIZE
 #if !CMK_TRACE_IN_CHARM
   if(CpvAccess(traceOn))
@@ -563,7 +594,8 @@ void CthAwaken(CthThread th)
     traceAwaken(th);
 #endif
 #endif
-  B(th)->awakenfn(th, CQS_QUEUEING_FIFO, 0, 0);
+  B(th)->awakenfn(B(th)->token, CQS_QUEUEING_FIFO, 0, 0);
+       B(th)->scheduled = 1;
 }
 
 void CthYield()
@@ -581,7 +613,8 @@ void CthAwakenPrio(CthThread th, int s, int pb, unsigned int *prio)
     traceAwaken(th);
 #endif
 #endif
-  B(th)->awakenfn(th, s, pb, prio);
+  B(th)->awakenfn(B(th)->token, s, pb, prio);
+       B(th)->scheduled = 1;
 }
 
 void CthYieldPrio(int s, int pb, unsigned int *prio)
index d80b109c146009849add316b6d030c73b6c6cc1c..7c824ae88709792fe6079d71f4087a561a5a0217 100644 (file)
 #define AMPI_PRINT_IDLE 0
 
 /* change this define to "x" to trace all send/recv's */
-#define MSG_ORDER_DEBUG(x) /* empty */
+#define MSG_ORDER_DEBUG(x) // x /* empty */
 /* change this define to "x" to trace user calls */
-#define USER_CALL_DEBUG(x) /* ckout<<"vp "<<TCHARM_Element()<<": "<<x<<endl; */
-#define STARTUP_DEBUG(x)  /* ckout<<"ampi[pe "<<CkMyPe()<<"] "<< x <<endl; */
+#define USER_CALL_DEBUG(x) // ckout<<"vp "<<TCHARM_Element()<<": "<<x<<endl; 
+#define STARTUP_DEBUG(x)  // ckout<<"ampi[pe "<<CkMyPe()<<"] "<< x <<endl; 
+
+
 
 //------------- startup -------------
 static mpi_comm_worlds mpi_worlds;
@@ -588,16 +590,20 @@ ampiParent::ampiParent(MPI_Comm worldNo_,CProxy_TCharm threads_)
   prepareCtv();
 
   thread->semaPut(AMPI_BARRIER_SEMAID,&barrier);
+       AsyncEvacuate(CmiFalse);
 }
 
 ampiParent::ampiParent(CkMigrateMessage *msg):CBase_ampiParent(msg) {
   thread=NULL;
   worldPtr=NULL;
   myDDT=&myDDTsto;
+       AsyncEvacuate(CmiFalse);
 }
 
 void ampiParent::pup(PUP::er &p) {
+
   ArrayElement1D::pup(p);
+//     printf("[%d] Ampiparent being pupped \n",thisIndex);
   p|threads;
 
   p|worldStruct;
@@ -624,6 +630,7 @@ void ampiParent::ckJustMigrated(void) {
 }
 
 ampiParent::~ampiParent() {
+       STARTUP_DEBUG("ampiParent> destructor called");
 }
 
 //Children call this when they are first created or just migrated
@@ -788,6 +795,7 @@ void ampi::init(void) {
   thread=NULL;
   msgs=NULL;
   resumeOnRecv=false;
+       AsyncEvacuate(CmiFalse);
 }
 
 ampi::ampi()
@@ -850,8 +858,8 @@ ampi::ampi(CkMigrateMessage *msg):CBase_ampi(msg)
 
 void ampi::ckJustMigrated(void)
 {
-       ArrayElement1D::ckJustMigrated();
        findParent(true);
+       ArrayElement1D::ckJustMigrated();
 }
 
 void ampi::findParent(bool forMigration) {
@@ -860,11 +868,13 @@ void ampi::findParent(bool forMigration) {
        if (parent==NULL) CkAbort("AMPI can't find its parent!");
        thread=parent->registerAmpi(this,myComm,forMigration);
        if (thread==NULL) CkAbort("AMPI can't find its thread!");
+//     printf("[%d] ampi index %d TCharm thread pointer %p \n",CkMyPe(),thisIndex,thread);
 }
 
 static void cmm_pup_ampi_message(pup_er p,void **msg) {
        CkPupMessage(*(PUP::er *)p,msg,1);
        if (pup_isDeleting(p)) delete (AmpiMsg *)*msg;
+//     printf("[%d] pupping ampi message %p \n",CkMyPe(),*msg);
 }
 
 void ampi::pup(PUP::er &p)
@@ -877,6 +887,7 @@ void ampi::pup(PUP::er &p)
   p|nbcasts;
   p|tmpVec;
   p|remoteProxy;
+       p|resumeOnRecv;
   p|comlibProxy;
   p|ciStreaming;
   p|ciBcast;
@@ -891,10 +902,12 @@ void ampi::pup(PUP::er &p)
   }
 
   msgs=CmmPup((pup_er)&p,msgs,cmm_pup_ampi_message);
+//     printf("[%d] ampi index %d msgs table pointer %p\n",CkMyPe(),thisIndex,msgs);
 
   p|seqEntries;
   p|oorder;
 }
+extern "C" void __dbgcheckMessageHandler();
 
 ampi::~ampi()
 {
@@ -907,7 +920,9 @@ ampi::~ampi()
     msg = (AmpiMsg *) CmmGet(msgs, 3, tags, sts);
   }
 */
+       __dbgcheckMessageHandler();
   CmmFree(msgs);
+       __dbgcheckMessageHandler();
 }
 
 //------------------------ Communicator Splitting ---------------------
@@ -1287,8 +1302,8 @@ void
 ampi::generic(AmpiMsg* msg)
 {
 MSG_ORDER_DEBUG(
-  CkPrintf("AMPI vp %d arrival: tag=%d, src=%d, comm=%d  (from %d, seq %d)\n",
-       thisIndex,msg->tag,msg->srcRank,msg->comm, msg->srcIdx, msg->seq);
+  CkPrintf("AMPI vp %d arrival: tag=%d, src=%d, comm=%d  (from %d, seq %d) resumeOnRecv %d\n",
+       thisIndex,msg->tag,msg->srcRank,msg->comm, msg->srcIdx, msg->seq,resumeOnRecv);
 )
 
 //     AmpiMsg *msgcopy = msg;
@@ -1388,6 +1403,8 @@ ampi::delesend(int t, int sRank, const void* buf, int count, int type,  int rank
 int
 ampi::recv(int t, int s, void* buf, int count, int type, int comm, int *sts)
 {
+       ampi *dis;
+       MPI_Comm disComm = myComm.getComm();
   if(s==MPI_PROC_NULL) {
     ((MPI_Status *)sts)->MPI_SOURCE = MPI_PROC_NULL;
     ((MPI_Status *)sts)->MPI_TAG = MPI_ANY_TAG;
@@ -1408,24 +1425,25 @@ ampi::recv(int t, int s, void* buf, int count, int type, int comm, int *sts)
 
   int tags[3];
   AmpiMsg *msg = 0;
-  CkDDT_DataType *ddt = getDDT()->getType(type);
-  int len = ddt->getSize(count);
   
  MSG_ORDER_DEBUG(
   CkPrintf("AMPI vp %d blocking recv: tag=%d, src=%d, comm=%d\n",thisIndex,t,s,comm);
  )
 
   resumeOnRecv=true;
-//  int counter=0;
   while(1) {
+               //This is done to take into account the case in which an ampi thread has migrated
+               //while waiting for a message
+               dis = getAmpiInstance(disComm);
     tags[0] = t; tags[1] = s; tags[2] = comm;
-    msg = (AmpiMsg *) CmmGet(msgs, 3, tags, sts);
- //   if(msg) counters.push_back(counter);
+    msg = (AmpiMsg *) CmmGet(dis->msgs, 3, tags, sts);
     if (msg) break;
-    thread->suspend();
-//    counter++;
+    dis->thread->suspend();
+               dis = getAmpiInstance(disComm);
   }
-  resumeOnRecv=false;
+  dis->resumeOnRecv=false;
+  CkDDT_DataType *ddt = dis->getDDT()->getType(type);
+  int len = ddt->getSize(count);
   
   if(sts)
     ((MPI_Status*)sts)->MPI_LENGTH = msg->length;
@@ -1688,6 +1706,14 @@ CDECL void AMPI_Migrate(void)
 #endif
 }
 
+
+CDECL void AMPI_Evacuate(void)
+{
+  TCHARM_Evacuate();
+}
+
+
+
 CDECL void AMPI_Migrateto(int destPE)
 {
   AMPIAPI("AMPI_MigrateTo");
@@ -1700,6 +1726,11 @@ CDECL void AMPI_Migrateto(int destPE)
 #endif
 }
 
+CDECL void AMPI_MigrateTo(int destPE)
+{
+       AMPI_Migrateto(destPE);
+}
+
 CDECL void AMPI_Async_Migrate(void)
 {
   AMPIAPI("AMPI_Async_Migrate");
@@ -1949,6 +1980,7 @@ const int MPI_REDUCE_SOURCE=0;
 const int MPI_REDUCE_COMM=MPI_COMM_WORLD;
 void ampi::reduceResult(CkReductionMsg *msg)
 {
+       MSG_ORDER_DEBUG(printf("[%d] reduceResult called \n",thisIndex));
   ampi::sendraw(MPI_REDUCE_TAG, MPI_REDUCE_SOURCE, msg->getData(), msg->getSize(),
              thisArrayID,thisIndex);
   delete msg;
@@ -1994,6 +2026,7 @@ int AMPI_Reduce(void *inbuf, void *outbuf, int count, int type, MPI_Op op,
   int rootIdx=ptr->comm2CommStruct(comm).getIndexForRank(root);
   CkCallback reduceCB(CkIndex_ampi::reduceResult(0),CkArrayIndex1D(rootIdx),ptr->getProxy(),true);
   msg->setCallback(reduceCB);
+       MSG_ORDER_DEBUG(CkPrintf("[%d] AMPI_Reduce called on comm %d root %d \n",ptr->thisIndex,comm,rootIdx));
   ptr->contribute(msg);
   if (ptr->thisIndex == rootIdx){
     /*HACK: Use recv() to block until reduction data comes back*/
@@ -2255,27 +2288,27 @@ inline void sortedIndex(int n, int* arr, int* idx){
       if (arr[idx[j+1]] < arr[idx[j]]) 
        swapInt(idx[j+1],idx[j]);
 }
-CkVec<CkVec<int> > vecIndex(int count, int* arr){
+CkVec<CkVec<int> > *vecIndex(int count, int* arr){
   CkAssert(count!=0);
   int *newidx = new int [count];
   int flag;
   sortedIndex(count,arr,newidx);
-  CkVec<CkVec<int> > vec;
+  CkVec<CkVec<int> > *vec = new CkVec<CkVec<int> >;
   CkVec<int> slot;
-  slot.push_back(newidx[0]);
-  vec.push_back(slot);
+  vec->push_back(slot);
+  (*vec)[0].push_back(newidx[0]);
   for(int i=1;i<count;i++){
     flag=0;
-    for(int j=0;j<vec.size();j++){
-      if(matchReq(arr[newidx[i]],arr[(vec[j])[0]])){
-        (vec[j]).push_back(newidx[i]);
+    for(int j=0;j<vec->size();j++){
+      if(matchReq(arr[newidx[i]],arr[((*vec)[j])[0]])){
+        ((*vec)[j]).push_back(newidx[i]);
        flag++;
       }
     }
     if(!flag){
       CkVec<int> newslot;
       newslot.push_back(newidx[i]);
-      vec.push_back(newslot);
+      vec->push_back(newslot);
     }else{
       CkAssert(flag==1);
     }
@@ -2355,16 +2388,24 @@ int AMPI_Waitall(int count, MPI_Request request[], MPI_Status sts[])
   AMPIAPI("AMPI_Waitall");
   if(count==0) return MPI_SUCCESS;
   checkRequests(count,request);
-  int i,j;
+  int i,j,oldPe;
   AmpiRequestList* reqs = getReqs();
-  CkVec<CkVec<int> > reqvec = vecIndex(count,request);
-  for(i=0;i<reqvec.size();i++){
-    for(j=0;j<(reqvec[i]).size();j++){
-      if(request[(reqvec[i])[j]] == MPI_REQUEST_NULL){
-        stsempty(sts[(reqvec[i])[j]]);
+  CkVec<CkVec<int> > *reqvec = vecIndex(count,request);
+  for(i=0;i<reqvec->size();i++){
+    for(j=0;j<((*reqvec)[i]).size();j++){
+      if(request[((*reqvec)[i])[j]] == MPI_REQUEST_NULL){
+        stsempty(sts[((*reqvec)[i])[j]]);
         continue;
       }
-      (*reqs)[request[(reqvec[i])[j]]]->wait(&sts[(reqvec[i])[j]]);
+                       oldPe = CkMyPe();
+                       AmpiRequest *waitReq = ((*reqs)[request[((*reqvec)[i])[j]]]);
+      waitReq->wait(&sts[((*reqvec)[i])[j]]);
+                       if(oldPe != CkMyPe()){
+#if 1
+                       reqs = getReqs();
+                       reqvec  = vecIndex(count,request);
+#endif
+                       }
     }
   }
 #if CMK_BLUEGENE_CHARM
@@ -2379,6 +2420,7 @@ int AMPI_Waitall(int count, MPI_Request request[], MPI_Status sts[])
       request[i] = MPI_REQUEST_NULL;
     }
   }
+       delete reqvec;
   return 0;
 }
 
@@ -2395,12 +2437,12 @@ int AMPI_Waitany(int count, MPI_Request *request, int *idx, MPI_Status *sts)
     return MPI_SUCCESS;
   }
   int flag=0;
-  CkVec<CkVec<int> > reqvec = vecIndex(count,request);
+  CkVec<CkVec<int> > *reqvec = vecIndex(count,request);
   while(count>0){ /* keep looping until some request finishes: */
-    for(int i=0;i<reqvec.size();i++){
-      AMPI_Test(&request[(reqvec[i])[0]], &flag, sts);
+    for(int i=0;i<reqvec->size();i++){
+      AMPI_Test(&request[((*reqvec)[i])[0]], &flag, sts);
       if(flag == 1 && sts->MPI_COMM != 0){ // to skip MPI_REQUEST_NULL
-        *idx = (reqvec[i])[0];
+        *idx = ((*reqvec)[i])[0];
        USER_CALL_DEBUG("AMPI_Waitany returning "<<*idx);
         return 0;
       }
@@ -2410,6 +2452,7 @@ int AMPI_Waitany(int count, MPI_Request *request, int *idx, MPI_Status *sts)
   }
   *idx = MPI_UNDEFINED;
   USER_CALL_DEBUG("AMPI_Waitany returning UNDEFINED");
+       delete reqvec;
   return 0;
 }
 
@@ -2426,13 +2469,13 @@ int AMPI_Waitsome(int incount, MPI_Request *array_of_requests, int *outcount,
   MPI_Status sts;
   int i;
   int flag=0, realflag=0;
-  CkVec<CkVec<int> > reqvec = vecIndex(incount,array_of_requests);
+  CkVec<CkVec<int> > *reqvec = vecIndex(incount,array_of_requests);
   *outcount = 0;
   while(1){
-    for(i=0;i<reqvec.size();i++){
-      AMPI_Test(&array_of_requests[(reqvec[i])[0]], &flag, &sts);
+    for(i=0;i<reqvec->size();i++){
+      AMPI_Test(&array_of_requests[((*reqvec)[i])[0]], &flag, &sts);
       if(flag == 1){ 
-        array_of_indices[(*outcount)]=(reqvec[i])[0];
+        array_of_indices[(*outcount)]=((*reqvec)[i])[0];
        array_of_statuses[(*outcount)++]=sts;
         if(sts.MPI_COMM != 0)
          realflag=1; // there is real(non null) request
@@ -2440,6 +2483,7 @@ int AMPI_Waitsome(int incount, MPI_Request *array_of_requests, int *outcount,
     }
     if(realflag && outcount>0) break;
   }
+       delete reqvec;
   return 0;
 }
 
@@ -2511,16 +2555,17 @@ int AMPI_Testany(int count, MPI_Request *request, int *index, int *flag, MPI_Sta
     stsempty(*sts);
     return MPI_SUCCESS;
   }
-  CkVec<CkVec<int> > reqvec = vecIndex(count,request);
+  CkVec<CkVec<int> > *reqvec = vecIndex(count,request);
   *flag=0;
-  for(int i=0;i<reqvec.size();i++){
-    AMPI_Test(&request[(reqvec[i])[0]], flag, sts);
+  for(int i=0;i<reqvec->size();i++){
+    AMPI_Test(&request[((*reqvec)[i])[0]], flag, sts);
     if(*flag==1 && sts->MPI_COMM!=0){ // skip MPI_REQUEST_NULL
-      *index = (reqvec[i])[0];
+      *index = ((*reqvec)[i])[0];
       return 0;
     }
   }
   *index = MPI_UNDEFINED;
+       delete reqvec;
   return 0;
 }
 
@@ -2533,18 +2578,19 @@ int AMPI_Testall(int count, MPI_Request *request, int *flag, MPI_Status *sts)
   int tmpflag;
   int i,j;
   AmpiRequestList* reqs = getReqs();
-  CkVec<CkVec<int> > reqvec = vecIndex(count,request);
+  CkVec<CkVec<int> > *reqvec = vecIndex(count,request);
   *flag = 1;  
-  for(i=0;i<reqvec.size();i++){
-    for(j=0;j<(reqvec[i]).size();j++){
-      if(request[(reqvec[i])[j]] == MPI_REQUEST_NULL)
+  for(i=0;i<reqvec->size();i++){
+    for(j=0;j<((*reqvec)[i]).size();j++){
+      if(request[((*reqvec)[i])[j]] == MPI_REQUEST_NULL)
         continue;
-      tmpflag = (*reqs)[request[(reqvec[i])[j]]]->test(&sts[(reqvec[i])[j]]);
+      tmpflag = (*reqs)[request[((*reqvec)[i])[j]]]->test(&sts[((*reqvec)[i])[j]]);
       *flag *= tmpflag;
     }
   }
   if(flag) 
     MPI_Waitall(count,request,sts);
+       delete reqvec;  
   return 0;
 }
 
@@ -2561,15 +2607,16 @@ int AMPI_Testsome(int incount, MPI_Request *array_of_requests, int *outcount,
   MPI_Status sts;
   int flag;
   int i;
-  CkVec<CkVec<int> > reqvec = vecIndex(incount,array_of_requests);
+  CkVec<CkVec<int> > *reqvec = vecIndex(incount,array_of_requests);
   *outcount = 0;
-  for(i=0;i<reqvec.size();i++){
-    AMPI_Test(&array_of_requests[(reqvec[i])[0]], &flag, &sts);
+  for(i=0;i<reqvec->size();i++){
+    AMPI_Test(&array_of_requests[((*reqvec)[i])[0]], &flag, &sts);
     if(flag == 1){
-      array_of_indices[(*outcount)]=(reqvec[i])[0];
+      array_of_indices[(*outcount)]=((*reqvec)[i])[0];
       array_of_statuses[(*outcount)++]=sts;
     }
   }
+       delete reqvec;
   return 0;
 }
 
index d7309b057258091eae6ee27ef54b65a56a5239ad..50f6384d675e0eb6fa5dcbacbfbd8fa9bb760a8f 100644 (file)
@@ -576,8 +576,13 @@ void AMPI_Print(char *str);
 int AMPI_Register(void *, MPI_PupFn);
 #define MPI_Migrate AMPI_Migrate
 void AMPI_Migrate(void);
+
+#define MPI_Evacuate AMPI_Evacuate
+void AMPI_Evacuate(void);
+
 #define MPI_Migrateto AMPI_Migrateto
 void AMPI_Migrateto(int destPE);
+
 #define MPI_Async_Migrate AMPI_Async_Migrate
 void AMPI_Async_Migrate(void);
 #define MPI_Allow_Migrate AMPI_Allow_Migrate
index b1b36c681b3f1ded2f0d25bb01b9fed709fb2521..5b3e2485c2e92b1c256a97d7961400432fd824e6 100644 (file)
@@ -9,7 +9,7 @@ Orion Sky Lawlor, olawlor@acm.org, 11/19/2001
 
 #if 0
     /*Many debugging statements:*/
-#    define DBG(x) ckout<<"["<<thisIndex<<"] TCHARM> "<<x<<endl;
+#    define DBG(x) ckout<<"["<<thisIndex<<","<<CkMyPe()<<"] TCHARM> "<<x<<endl;
 #    define DBGX(x) ckout<<"PE("<<CkMyPe()<<") TCHARM> "<<x<<endl;
 #else
     /*No debugging statements*/
@@ -144,6 +144,8 @@ TCharm::TCharm(TCharmInitMsg *initMsg_)
   CtvAccessOther(tid,_curTCharm)=this;
   isStopped=true;
   resumeAfterMigration=false;
+       /* FAULT_EVAC*/
+       AsyncEvacuate(CmiTrue);
   skipResume=false;
   exitWhenDone=initMsg->opts.exitWhenDone;
   threadInfo.tProxy=CProxy_TCharm(thisArrayID);
@@ -165,6 +167,7 @@ TCharm::TCharm(CkMigrateMessage *msg)
   tid=NULL;
   threadGlobals=NULL;
   threadInfo.tProxy=CProxy_TCharm(thisArrayID);
+       AsyncEvacuate(CmiTrue);
   heapBlocks=0;
 }
 
@@ -192,8 +195,9 @@ void TCharm::pup(PUP::er &p) {
 
 #ifndef CMK_OPTIMIZE
   DBG("Packing thread");
-  if (!isStopped)
+  if (!isStopped && !CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC)){
     CkAbort("Cannot pup a running thread.  You must suspend before migrating.\n");
+       }       
   if (tcharm_nomig) CkAbort("Cannot migrate with the +tcharm_nomig option!\n");
 #endif
 
@@ -293,7 +297,7 @@ void TCharm::migrateTo(int destPE) {
        if (destPE==CkMyPe()) return;
        // Make sure migrateMe gets called *after* we suspend:
        thisProxy[thisIndex].migrateDelayed(destPE);
-       resumeAfterMigration=true;
+//     resumeAfterMigration=true;
        suspend();
 }
 void TCharm::migrateDelayed(int destPE) {
@@ -302,11 +306,23 @@ void TCharm::migrateDelayed(int destPE) {
 void TCharm::ckJustMigrated(void) {
        ArrayElement::ckJustMigrated();
        if (resumeAfterMigration) {
-               resumeAfterMigration=false;
+               resumeAfterMigration=false;
                resume(); //Start the thread running
        }
 }
 
+/*
+       FAULT_EVAC
+
+       If a Tcharm object is about to migrate it should be suspended first
+*/
+void TCharm::ckAboutToMigrate(void){
+       ArrayElement::ckAboutToMigrate();
+       resumeAfterMigration = true;
+       isStopped = true;
+//     suspend();
+}
+
 // clear the data before restarting from disk
 void TCharm::clear()
 {
@@ -354,7 +370,7 @@ void TCharm::stop(void)
   isStopped=true;
   DBG("thread suspended");
   CthSuspend();
-  DBG("thread resumed");
+//     DBG("thread resumed");
   /*SUBTLE: We have to do the get() because "this" may have changed
     during a migration-suspend.  If you access *any* members
     from this point onward, you'll cause heap corruption if
@@ -363,6 +379,7 @@ void TCharm::stop(void)
   TCharm *dis=TCharm::get();
   dis->isStopped=false;
   dis->startTiming();
+//     printf("[%d] Thread resumed  for tid %p\n",dis->thisIndex,dis->tid);
 }
 
 //Resume the waiting thread
@@ -393,6 +410,23 @@ void TCharm::migrate(void)
 #endif
 }
 
+
+void TCharm::evacuate(){
+       /*
+               FAULT_EVAC
+       */
+       //CkClearAllArrayElementsCPP();
+       if(CpvAccess(startedEvac)){
+               int nextPE = getNextPE(CkArrayIndex1D(thisIndex));
+//             resumeAfterMigration=true;
+               CcdCallFnAfter((CcdVoidFn)CkEmmigrateElement, (void *)myRec, 1);
+               suspend();
+               return;
+       }
+       return;
+
+}
+
 //calls atsync with async mode
 void TCharm::async_migrate(void)
 {
@@ -721,6 +755,13 @@ CDECL void TCHARM_Migrate_to(int destPE)
        TCHARMAPI("TCHARM_Migrate_to");
        TCharm::get()->migrateTo(destPE);
 }
+
+CDECL void TCHARM_Evacuate()
+{
+       TCHARMAPI("TCHARM_Migrate_to");
+       TCharm::get()->evacuate();
+}
+
 FORTRAN_AS_C(TCHARM_MIGRATE_TO,TCHARM_Migrate_to,tcharm_migrate_to,
        (int *destPE),(*destPE))
 
index 2075c0d588a425ea5e711d3196844420e388c339..c4b48ea94f37e4408045a39033aaf10ad54982ca 100644 (file)
@@ -155,6 +155,8 @@ class TCharm: public CBase_TCharm
        ~TCharm();
        
        virtual void ckJustMigrated(void);
+       virtual void ckAboutToMigrate(void);
+       
        void migrateDelayed(int destPE);
        void atBarrier(CkReductionMsg *);
        void atExit(CkReductionMsg *);
@@ -175,6 +177,8 @@ class TCharm: public CBase_TCharm
        //Block, migrate to destPE, and resume
        void migrateTo(int destPE);
 
+       void evacuate();
+
        //Thread finished running
        void done(void);
 
@@ -207,7 +211,10 @@ class TCharm: public CBase_TCharm
        void start(void);
        //Aliases:
        inline void suspend(void) {stop();}
-       inline void resume(void) { if (isStopped) start();}
+       inline void resume(void) { if (isStopped) start(); else {
+       //      printf("[%d] TCharm resume called on already running thread pe %d \n",thisIndex,CkMyPe());
+               }
+       }
 
        //Go to sync, block, possibly migrate, and then resume
        void migrate(void);
index 47a6ae77ab4c83286ef1d105f8bd1e312b2a17f3..86dc78b513f4ced67c2c79510f869fa3bc2e84b0 100644 (file)
@@ -47,6 +47,8 @@ void TCHARM_Migrate(void);
 void TCHARM_Async_Migrate(void);
 void TCHARM_Allow_Migrate(void);
 void TCHARM_Migrate_to(int destPE);
+void TCHARM_Evacuate();
+
 int TCHARM_System(const char *shell_command);
 void TCHARM_Done(void);
 void TCHARM_Yield(void);
index 96f7af87ac498a63185aaff15b7e309ec530cb57..9c520fb0dad39357fa5ac511e2d2e6e263d23699 100644 (file)
@@ -168,8 +168,9 @@ trace-projections.o: trace-projections.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h trace-common.h trace-projections.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-common.h \
+  trace-projections.h
        $(CHARMC) -c -I. trace-projections.C
 
 trace-summary.o: trace-summary.C charm++.h charm.h converse.h \
@@ -186,9 +187,10 @@ trace-summary.o: trace-summary.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h trace-summary.h register.h trace-common.h \
-  trace-summaryBOC.h TraceSummary.decl.h TraceSummary.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-summary.h \
+  register.h trace-common.h trace-summaryBOC.h TraceSummary.decl.h \
+  TraceSummary.def.h
        $(CHARMC) -c -I. trace-summary.C
 
 trace-counter.o: trace-counter.C charm++.h charm.h converse.h \
@@ -205,8 +207,8 @@ trace-counter.o: trace-counter.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. trace-counter.C
 
 trace-bluegene.o: trace-bluegene.C charm++.h charm.h converse.h \
@@ -223,10 +225,10 @@ trace-bluegene.o: trace-bluegene.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h trace-common.h blue.h blue-conv.h blue_impl.h \
-  ckliststring.h blue_types.h blue_timing.h blue_logs.h blue_defs.h \
-  blue_network.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-common.h blue.h \
+  blue-conv.h blue_impl.h ckliststring.h blue_types.h blue_timing.h \
+  blue_logs.h blue_defs.h blue_network.h
        $(CHARMC) -c -I. trace-bluegene.C
 
 trace-projector.o: trace-projector.C charm++.h charm.h converse.h \
@@ -243,12 +245,12 @@ trace-projector.o: trace-projector.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h trace-projector.h ck.h qd.h register.h stats.h \
-  ckfutures.h allEvents.h charmEvents.h charmProjections.h traceCoreAPI.h \
-  converseEvents.h converseProjections.h machineEvents.h \
-  machineProjections.h traceCore.h threadEvents.h traceCoreCommon.h \
-  trace-common.h trace-projections.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-projector.h ck.h \
+  qd.h register.h stats.h ckfutures.h allEvents.h charmEvents.h \
+  charmProjections.h traceCoreAPI.h converseEvents.h \
+  converseProjections.h machineEvents.h machineProjections.h traceCore.h \
+  threadEvents.h traceCoreCommon.h trace-common.h trace-projections.h
        $(CHARMC) -c -I. trace-projector.C
 
 trace-converse.o: trace-converse.c conv-trace.h converse.h conv-config.h \
@@ -274,8 +276,9 @@ trace-recordreplay.o: trace-recordreplay.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h trace-recordreplay.h register.h trace-common.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-recordreplay.h \
+  register.h trace-common.h
        $(CHARMC) -c -I. trace-recordreplay.C
 
 trace-common.o: trace-common.C charm.h converse.h conv-config.h \
@@ -303,8 +306,8 @@ tracef.o: tracef.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h charm-api.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h charm-api.h
        $(CHARMC) -c -I. tracef.C
 
 init.o: init.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -321,8 +324,9 @@ init.o: init.C ck.h charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h
        $(CHARMC) -c -I. init.C
 
 register.o: register.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -339,8 +343,9 @@ register.o: register.C ck.h charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h
        $(CHARMC) -c -I. register.C
 
 qd.o: qd.C ck.h charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -357,8 +362,9 @@ qd.o: qd.C ck.h charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h
        $(CHARMC) -c -I. qd.C
 
 ck.o: ck.C ck.h charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -375,9 +381,9 @@ ck.o: ck.C ck.h charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h queueing.h \
-  CkMarshall.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h queueing.h CkMarshall.def.h
        $(CHARMC) -c -I. ck.C
 
 main.o: main.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -394,8 +400,9 @@ main.o: main.C ck.h charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h
        $(CHARMC) -c -I. main.C
 
 msgalloc.o: msgalloc.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -412,8 +419,9 @@ msgalloc.o: msgalloc.C ck.h charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h queueing.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h queueing.h
        $(CHARMC) -c -I. msgalloc.C
 
 ckfutures.o: ckfutures.C charm++.h charm.h converse.h conv-config.h \
@@ -430,9 +438,9 @@ ckfutures.o: ckfutures.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ck.h qd.h register.h stats.h ckfutures.h \
-  CkFutures.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h CkFutures.def.h
        $(CHARMC) -c -I. ckfutures.C
 
 debug-message.o: debug-message.C ck.h charm++.h charm.h converse.h \
@@ -449,8 +457,9 @@ debug-message.o: debug-message.C ck.h charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h queueing.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h queueing.h
        $(CHARMC) -c -I. debug-message.C
 
 debug-charm.o: debug-charm.C converse.h conv-config.h conv-common.h \
@@ -467,8 +476,8 @@ debug-charm.o: debug-charm.C converse.h conv-config.h conv-common.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h stats.h ckfutures.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h stats.h ckfutures.h
        $(CHARMC) -c -I. debug-charm.C
 
 ckcallback.o: ckcallback.C charm++.h charm.h converse.h conv-config.h \
@@ -485,8 +494,9 @@ ckcallback.o: ckcallback.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ckcallback-ccs.h CkCallback.decl.h CkCallback.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ckcallback-ccs.h \
+  CkCallback.decl.h CkCallback.def.h
        $(CHARMC) -c -I. ckcallback.C
 
 cklocation.o: cklocation.C charm++.h charm.h converse.h conv-config.h \
@@ -503,9 +513,9 @@ cklocation.o: cklocation.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h register.h ck.h qd.h stats.h ckfutures.h \
-  CkLocation.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h register.h ck.h qd.h \
+  stats.h ckfutures.h CkLocation.def.h
        $(CHARMC) -c -I. cklocation.C
 
 ckarray.o: ckarray.C charm++.h charm.h converse.h conv-config.h \
@@ -522,8 +532,9 @@ ckarray.o: ckarray.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h register.h ck.h qd.h stats.h ckfutures.h CkArray.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h register.h ck.h qd.h \
+  stats.h ckfutures.h CkArray.def.h
        $(CHARMC) -c -I. ckarray.C
 
 ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-config.h \
@@ -540,9 +551,9 @@ ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ck.h qd.h register.h stats.h ckfutures.h \
-  CkReduction.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h CkReduction.def.h
        $(CHARMC) -c -I. ckreduction.C
 
 ckarrayreductionmgr.o: ckarrayreductionmgr.C charm++.h charm.h converse.h \
@@ -559,9 +570,9 @@ ckarrayreductionmgr.o: ckarrayreductionmgr.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ck.h qd.h register.h stats.h ckfutures.h \
-  CkArrayReductionMgr.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h CkArrayReductionMgr.def.h
        $(CHARMC) -c -I. ckarrayreductionmgr.C
 
 tempo.o: tempo.C charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -578,8 +589,8 @@ tempo.o: tempo.C charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h tempo.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h tempo.def.h
        $(CHARMC) -c -I. tempo.C
 
 waitqd.o: waitqd.C waitqd.h charm++.h charm.h converse.h conv-config.h \
@@ -596,8 +607,8 @@ waitqd.o: waitqd.C waitqd.h charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  waitqd.decl.h waitqd.def.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h waitqd.decl.h waitqd.def.h
        $(CHARMC) -c -I. waitqd.C
 
 LBDatabase.o: LBDatabase.C converse.h conv-config.h conv-common.h \
@@ -614,9 +625,10 @@ LBDatabase.o: LBDatabase.C converse.h conv-config.h conv-common.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h LBDatabase.def.h LBSimulation.h CentralLB.h \
-  BaseLB.h CentralLB.decl.h CentralLBMsg.h topology.h NullLB.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h LBDatabase.def.h \
+  LBSimulation.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
+  topology.h NullLB.h
        $(CHARMC) -c -I. LBDatabase.C
 
 lbdb.o: lbdb.C converse.h conv-config.h conv-common.h conv-mach.h \
@@ -633,8 +645,8 @@ lbdb.o: lbdb.C converse.h conv-config.h conv-common.h conv-mach.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h NullLB.decl.h \
-  BaseLB.decl.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
+  NullLB.decl.h BaseLB.decl.h
        $(CHARMC) -c -I. lbdb.C
 
 lbdbf.o: lbdbf.C charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -651,8 +663,8 @@ lbdbf.o: lbdbf.C charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h charm-api.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h charm-api.h
        $(CHARMC) -c -I. lbdbf.C
 
 charisma.o: charisma.C charisma.h charisma.decl.h charm++.h charm.h \
@@ -669,8 +681,8 @@ charisma.o: charisma.C charisma.h charisma.decl.h charm++.h charm.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  charisma.def.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h charisma.def.h
        $(CHARMC) -c -I. charisma.C
 
 ckobjQ.o: ckobjQ.C charm++.h charm.h converse.h conv-config.h \
@@ -687,8 +699,8 @@ ckobjQ.o: ckobjQ.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h queueing.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h queueing.h
        $(CHARMC) -c -I. ckobjQ.C
 
 LBAgent.o: LBAgent.C LBAgent.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
@@ -705,8 +717,9 @@ LBAgent.o: LBAgent.C LBAgent.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h topology.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
+  CentralLBMsg.h topology.h
        $(CHARMC) -c -I. LBAgent.C
 
 LBProfit.o: LBProfit.C converse.h conv-config.h conv-common.h conv-mach.h \
@@ -723,8 +736,8 @@ LBProfit.o: LBProfit.C converse.h conv-config.h conv-common.h conv-mach.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h
        $(CHARMC) -c -I. LBProfit.C
 
 ckcheckpoint.o: ckcheckpoint.C charm++.h charm.h converse.h conv-config.h \
@@ -741,9 +754,9 @@ ckcheckpoint.o: ckcheckpoint.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ck.h qd.h register.h stats.h ckfutures.h \
-  CkCheckpoint.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h CkCheckpoint.def.h
        $(CHARMC) -c -I. ckcheckpoint.C
 
 ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
@@ -760,11 +773,30 @@ ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ck.h qd.h register.h stats.h ckfutures.h \
-  CkMemCheckpoint.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h CkMemCheckpoint.def.h
        $(CHARMC) -c -I. ckmemcheckpoint.C
 
+ckevacuation.o: ckevacuation.C charm++.h charm.h converse.h conv-config.h \
+  conv-common.h conv-mach.h conv-autoconfig.h conv-mach-opt.h pup_c.h \
+  conv-cpm.h conv-cpath.h conv-qd.h conv-random.h conv-lists.h \
+  conv-trace.h persistent.h debug-conv.h pup.h middle.h middle-conv.h \
+  cklists.h ckbitvector.h ckstream.h init.h ckhashtable.h debug-charm.h \
+  CkMarshall.decl.h cksection.h ckcallback.h conv-ccs.h sockRoutines.h \
+  ccs-server.h ckobjQ.h ckreduction.h CkReduction.decl.h \
+  cknodegroupreduction.h CkArrayReductionMgr.decl.h ckmemcheckpoint.h \
+  CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
+  lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h \
+  LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h CkLocation.decl.h \
+  CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
+  convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
+  charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ck.h qd.h register.h \
+  stats.h ckfutures.h
+       $(CHARMC) -c -I. ckevacuation.C
+
 LBDBManager.o: LBDBManager.C charm++.h charm.h converse.h conv-config.h \
   conv-common.h conv-mach.h conv-autoconfig.h conv-mach-opt.h pup_c.h \
   conv-cpm.h conv-cpath.h conv-qd.h conv-random.h conv-lists.h \
@@ -779,8 +811,8 @@ LBDBManager.o: LBDBManager.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. LBDBManager.C
 
 LBComm.o: LBComm.C converse.h conv-config.h conv-common.h conv-mach.h \
@@ -803,8 +835,8 @@ LBObj.o: LBObj.C charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. LBObj.C
 
 LBMachineUtil.o: LBMachineUtil.C LBDatabase.h lbdb.h converse.h \
@@ -821,8 +853,8 @@ LBMachineUtil.o: LBMachineUtil.C LBDatabase.h lbdb.h converse.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h
        $(CHARMC) -c -I. LBMachineUtil.C
 
 CentralPredictor.o: CentralPredictor.C charm++.h charm.h converse.h \
@@ -839,8 +871,9 @@ CentralPredictor.o: CentralPredictor.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h
        $(CHARMC) -c -I. CentralPredictor.C
 
 BaseLB.o: BaseLB.C charm++.h charm.h converse.h conv-config.h \
@@ -857,9 +890,9 @@ BaseLB.o: BaseLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h BaseLB.h LBSimulation.h CentralLB.h CentralLB.decl.h \
-  CentralLBMsg.h BaseLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h BaseLB.h LBSimulation.h \
+  CentralLB.h CentralLB.decl.h CentralLBMsg.h BaseLB.def.h
        $(CHARMC) -c -I. BaseLB.C
 
 CentralLB.o: CentralLB.C charm++.h charm.h converse.h conv-config.h \
@@ -876,9 +909,9 @@ CentralLB.o: CentralLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
-  LBSimulation.h CentralLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h LBSimulation.h CentralLB.def.h
        $(CHARMC) -c -I. CentralLB.C
 
 HybridBaseLB.o: HybridBaseLB.C charm++.h charm.h converse.h conv-config.h \
@@ -895,13 +928,13 @@ HybridBaseLB.o: HybridBaseLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h BaseLB.h HybridBaseLB.h CentralLB.h CentralLB.decl.h \
-  CentralLBMsg.h HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h \
-  topology.h GreedyLB.h GreedyLB.decl.h GreedyCommLB.h \
-  GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h RefineCommLB.h \
-  RefinerComm.h Refiner.h RefineLB.h RefineLB.decl.h RefineCommLB.decl.h \
-  HybridBaseLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h BaseLB.h HybridBaseLB.h \
+  CentralLB.h CentralLB.decl.h CentralLBMsg.h HybridBaseLB.decl.h \
+  NeighborLBMsg.h HybridLBMsg.h topology.h GreedyLB.h GreedyLB.decl.h \
+  GreedyCommLB.h GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h \
+  RefineCommLB.h RefinerComm.h Refiner.h RefineLB.h RefineLB.decl.h \
+  RefineCommLB.decl.h HybridBaseLB.def.h
        $(CHARMC) -c -I. HybridBaseLB.C
 
 NborBaseLB.o: NborBaseLB.C charm++.h charm.h converse.h conv-config.h \
@@ -918,9 +951,9 @@ NborBaseLB.o: NborBaseLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h BaseLB.h NborBaseLB.h NborBaseLB.decl.h \
-  NeighborLBMsg.h topology.h NborBaseLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h BaseLB.h NborBaseLB.h \
+  NborBaseLB.decl.h NeighborLBMsg.h topology.h NborBaseLB.def.h
        $(CHARMC) -c -I. NborBaseLB.C
 
 WSLB.o: WSLB.C charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -937,9 +970,9 @@ WSLB.o: WSLB.C charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h BaseLB.h heap.h elements.h Set.h WSLB.h WSLB.decl.h \
-  topology.h WSLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h BaseLB.h heap.h \
+  elements.h Set.h WSLB.h WSLB.decl.h topology.h WSLB.def.h
        $(CHARMC) -c -I. WSLB.C
 
 ObjGraph.o: ObjGraph.C ObjGraph.h lbdb.h converse.h conv-config.h \
@@ -956,9 +989,9 @@ ObjGraph.o: ObjGraph.C ObjGraph.h lbdb.h converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
-  CentralLBMsg.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h NullLB.decl.h \
+  BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h
        $(CHARMC) -c -I. ObjGraph.C
 
 graph.o: graph.c converse.h conv-config.h conv-common.h conv-mach.h \
@@ -981,8 +1014,9 @@ LButil.o: LButil.C elements.h converse.h conv-config.h conv-common.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h NullLB.decl.h BaseLB.decl.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h NullLB.decl.h \
+  BaseLB.decl.h
        $(CHARMC) -c -I. LButil.C
 
 Refiner.o: Refiner.C Refiner.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
@@ -999,9 +1033,9 @@ Refiner.o: Refiner.C Refiner.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h elements.h \
-  Set.h heap.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
+  CentralLBMsg.h elements.h Set.h heap.h
        $(CHARMC) -c -I. Refiner.C
 
 RefinerApprox.o: RefinerApprox.C RefinerApprox.h CentralLB.h BaseLB.h \
@@ -1018,9 +1052,10 @@ RefinerApprox.o: RefinerApprox.C RefinerApprox.h CentralLB.h BaseLB.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
-  CentralLBMsg.h Refiner.h elements.h Set.h heap.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h NullLB.decl.h \
+  BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h Refiner.h elements.h \
+  Set.h heap.h
        $(CHARMC) -c -I. RefinerApprox.C
 
 RefinerComm.o: RefinerComm.C RefinerComm.h CentralLB.h BaseLB.h \
@@ -1037,9 +1072,10 @@ RefinerComm.o: RefinerComm.C RefinerComm.h CentralLB.h BaseLB.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
-  CentralLBMsg.h elements.h Set.h heap.h Refiner.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h NullLB.decl.h \
+  BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h elements.h Set.h heap.h \
+  Refiner.h
        $(CHARMC) -c -I. RefinerComm.C
 
 bitvecset.o: bitvecset.c bitvecset.h
@@ -1073,8 +1109,8 @@ CommLBHeap.o: CommLBHeap.C CommLBHeap.h charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. CommLBHeap.C
 
 NullLB.o: NullLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1091,9 +1127,9 @@ NullLB.o: NullLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h BaseLB.h NullLB.h ck.h qd.h register.h stats.h \
-  ckfutures.h NullLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h BaseLB.h NullLB.h ck.h \
+  qd.h register.h stats.h ckfutures.h NullLB.def.h
        $(CHARMC) -c -I. NullLB.C
 
 LBSimulation.o: LBSimulation.C LBSimulation.h charm++.h charm.h \
@@ -1110,8 +1146,9 @@ LBSimulation.o: LBSimulation.C LBSimulation.h charm++.h charm.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h
        $(CHARMC) -c -I. LBSimulation.C
 
 ComlibStrategy.o: ComlibStrategy.C charm++.h charm.h converse.h \
@@ -1128,8 +1165,8 @@ ComlibStrategy.o: ComlibStrategy.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h register.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h register.h
        $(CHARMC) -c -I. ComlibStrategy.C
 
 ComlibArrayListener.o: ComlibArrayListener.C charm++.h charm.h converse.h \
@@ -1146,8 +1183,8 @@ ComlibArrayListener.o: ComlibArrayListener.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. ComlibArrayListener.C
 
 charmProjections.o: charmProjections.C converse.h conv-config.h \
@@ -1165,8 +1202,9 @@ charmProjections.o: charmProjections.C converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h qd.h register.h stats.h ckfutures.h trace-common.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h qd.h register.h stats.h \
+  ckfutures.h trace-common.h
        $(CHARMC) -c -I. charmProjections.C
 
 EveryLB.o: EveryLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1183,13 +1221,13 @@ EveryLB.o: EveryLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h EveryLB.decl.h DummyLB.decl.h CentralLB.decl.h \
-  CentralLBMsg.h ComboCentLB.decl.h RandCentLB.decl.h RefineLB.decl.h \
-  RefineKLB.decl.h RefineCommLB.decl.h GreedyLB.decl.h \
-  GreedyCommLB.decl.h GreedyAgentLB.decl.h GridCommLB.decl.h \
-  Comm1LB.decl.h OrbLB.decl.h RecBisectBfLB.decl.h MetisLB.decl.h \
-  PhasebyArrayLB.decl.h RotateLB.decl.h NeighborLB.decl.h \
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h EveryLB.decl.h \
+  DummyLB.decl.h CentralLB.decl.h CentralLBMsg.h ComboCentLB.decl.h \
+  RandCentLB.decl.h RefineLB.decl.h RefineKLB.decl.h RefineCommLB.decl.h \
+  GreedyLB.decl.h GreedyCommLB.decl.h GreedyAgentLB.decl.h \
+  GridCommLB.decl.h Comm1LB.decl.h OrbLB.decl.h RecBisectBfLB.decl.h \
+  MetisLB.decl.h PhasebyArrayLB.decl.h RotateLB.decl.h NeighborLB.decl.h \
   NborBaseLB.decl.h NeighborLBMsg.h NeighborCommLB.decl.h WSLB.decl.h \
   HybridLB.decl.h HybridBaseLB.decl.h HybridLBMsg.h TopoLB.decl.h \
   RefineTopoLB.decl.h TopoCentLB.decl.h EveryLB.def.h
@@ -1209,9 +1247,9 @@ DummyLB.o: DummyLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h DummyLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h DummyLB.decl.h DummyLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h DummyLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h DummyLB.decl.h DummyLB.def.h
        $(CHARMC) -c -I. DummyLB.C
 
 ComboCentLB.o: ComboCentLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1228,9 +1266,10 @@ ComboCentLB.o: ComboCentLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ComboCentLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h ComboCentLB.decl.h ComboCentLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ComboCentLB.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h ComboCentLB.decl.h \
+  ComboCentLB.def.h
        $(CHARMC) -c -I. ComboCentLB.C
 
 RandCentLB.o: RandCentLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1247,9 +1286,10 @@ RandCentLB.o: RandCentLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RandCentLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h RandCentLB.decl.h RandCentLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RandCentLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h RandCentLB.decl.h \
+  RandCentLB.def.h
        $(CHARMC) -c -I. RandCentLB.C
 
 RefineLB.o: RefineLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1266,10 +1306,10 @@ RefineLB.o: RefineLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RefineLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h Refiner.h elements.h Set.h heap.h RefineLB.decl.h \
-  RefineLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RefineLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h Refiner.h elements.h Set.h \
+  heap.h RefineLB.decl.h RefineLB.def.h
        $(CHARMC) -c -I. RefineLB.C
 
 RefineKLB.o: RefineKLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1286,10 +1326,10 @@ RefineKLB.o: RefineKLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RefineKLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h RefinerApprox.h Refiner.h elements.h Set.h heap.h \
-  RefineKLB.decl.h RefineKLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RefineKLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h RefinerApprox.h Refiner.h \
+  elements.h Set.h heap.h RefineKLB.decl.h RefineKLB.def.h
        $(CHARMC) -c -I. RefineKLB.C
 
 RefineCommLB.o: RefineCommLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1306,10 +1346,11 @@ RefineCommLB.o: RefineCommLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RefineCommLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h RefinerComm.h elements.h Set.h heap.h Refiner.h \
-  RefineLB.h RefineLB.decl.h RefineCommLB.decl.h RefineCommLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RefineCommLB.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h RefinerComm.h \
+  elements.h Set.h heap.h Refiner.h RefineLB.h RefineLB.decl.h \
+  RefineCommLB.decl.h RefineCommLB.def.h
        $(CHARMC) -c -I. RefineCommLB.C
 
 GreedyLB.o: GreedyLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1326,9 +1367,9 @@ GreedyLB.o: GreedyLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h GreedyLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h GreedyLB.decl.h GreedyLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h GreedyLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h GreedyLB.decl.h GreedyLB.def.h
        $(CHARMC) -c -I. GreedyLB.C
 
 GreedyCommLB.o: GreedyCommLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1345,10 +1386,11 @@ GreedyCommLB.o: GreedyCommLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h GreedyCommLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h \
-  manager.h GreedyCommLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h GreedyCommLB.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
+  GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h manager.h \
+  GreedyCommLB.def.h
        $(CHARMC) -c -I. GreedyCommLB.C
 
 GreedyAgentLB.o: GreedyAgentLB.C charm++.h charm.h converse.h \
@@ -1365,10 +1407,10 @@ GreedyAgentLB.o: GreedyAgentLB.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h LBSimulation.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h GreedyAgentLB.h GreedyAgentLB.decl.h LBAgent.h \
-  topology.h GreedyAgentLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h LBSimulation.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h GreedyAgentLB.h \
+  GreedyAgentLB.decl.h LBAgent.h topology.h GreedyAgentLB.def.h
        $(CHARMC) -c -I. GreedyAgentLB.C
 
 GridCommLB.o: GridCommLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1385,9 +1427,10 @@ GridCommLB.o: GridCommLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h GridCommLB.decl.h CentralLB.decl.h CentralLBMsg.h \
-  GridCommLB.h CentralLB.h BaseLB.h manager.h GridCommLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h GridCommLB.decl.h \
+  CentralLB.decl.h CentralLBMsg.h GridCommLB.h CentralLB.h BaseLB.h \
+  manager.h GridCommLB.def.h
        $(CHARMC) -c -I. GridCommLB.C
 
 Comm1LB.o: Comm1LB.C charm++.h charm.h converse.h conv-config.h \
@@ -1404,10 +1447,11 @@ Comm1LB.o: Comm1LB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h Comm1LB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h Comm1LB.decl.h CommLBHeap.h GreedyCommLB.h \
-  GreedyCommLB.decl.h elements.h Set.h heap.h Comm1LB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h Comm1LB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h Comm1LB.decl.h CommLBHeap.h \
+  GreedyCommLB.h GreedyCommLB.decl.h elements.h Set.h heap.h \
+  Comm1LB.def.h
        $(CHARMC) -c -I. Comm1LB.C
 
 OrbLB.o: OrbLB.C charm++.h charm.h converse.h conv-config.h conv-common.h \
@@ -1424,9 +1468,9 @@ OrbLB.o: OrbLB.C charm++.h charm.h converse.h conv-config.h conv-common.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h OrbLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h OrbLB.decl.h OrbLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h OrbLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h OrbLB.decl.h OrbLB.def.h
        $(CHARMC) -c -I. OrbLB.C
 
 RecBisectBfLB.o: RecBisectBfLB.C charm++.h charm.h converse.h \
@@ -1443,10 +1487,10 @@ RecBisectBfLB.o: RecBisectBfLB.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RecBisectBfLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h RecBisectBfLB.decl.h ObjGraph.h graph.h bitvecset.h \
-  RecBisectBfLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RecBisectBfLB.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
+  RecBisectBfLB.decl.h ObjGraph.h graph.h bitvecset.h RecBisectBfLB.def.h
        $(CHARMC) -c -I. RecBisectBfLB.C
 
 MetisLB.o: MetisLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1463,9 +1507,9 @@ MetisLB.o: MetisLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h MetisLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h MetisLB.decl.h MetisLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h MetisLB.h CentralLB.h \
+  BaseLB.h CentralLB.decl.h CentralLBMsg.h MetisLB.decl.h MetisLB.def.h
        $(CHARMC) -c -I. MetisLB.C
 
 PhasebyArrayLB.o: PhasebyArrayLB.C charm++.h charm.h converse.h \
@@ -1482,9 +1526,10 @@ PhasebyArrayLB.o: PhasebyArrayLB.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h PhasebyArrayLB.h CentralLB.h BaseLB.h CentralLB.decl.h \
-  CentralLBMsg.h PhasebyArrayLB.decl.h PhasebyArrayLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h PhasebyArrayLB.h \
+  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
+  PhasebyArrayLB.decl.h PhasebyArrayLB.def.h
        $(CHARMC) -c -I. PhasebyArrayLB.C
 
 RotateLB.o: RotateLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1501,9 +1546,10 @@ RotateLB.o: RotateLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h RotateLB.decl.h CentralLB.decl.h CentralLBMsg.h \
-  RotateLB.h CentralLB.h BaseLB.h RotateLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h RotateLB.decl.h \
+  CentralLB.decl.h CentralLBMsg.h RotateLB.h CentralLB.h BaseLB.h \
+  RotateLB.def.h
        $(CHARMC) -c -I. RotateLB.C
 
 NeighborLB.o: NeighborLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1520,10 +1566,10 @@ NeighborLB.o: NeighborLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h heap.h elements.h Set.h NeighborLB.h NborBaseLB.h \
-  BaseLB.h NborBaseLB.decl.h NeighborLBMsg.h topology.h NeighborLB.decl.h \
-  NeighborLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h heap.h elements.h Set.h \
+  NeighborLB.h NborBaseLB.h BaseLB.h NborBaseLB.decl.h NeighborLBMsg.h \
+  topology.h NeighborLB.decl.h NeighborLB.def.h
        $(CHARMC) -c -I. NeighborLB.C
 
 NeighborCommLB.o: NeighborCommLB.C charm++.h charm.h converse.h \
@@ -1540,10 +1586,10 @@ NeighborCommLB.o: NeighborCommLB.C charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h heap.h elements.h Set.h NeighborCommLB.h NborBaseLB.h \
-  BaseLB.h NborBaseLB.decl.h NeighborLBMsg.h topology.h \
-  NeighborCommLB.decl.h NeighborCommLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h heap.h elements.h Set.h \
+  NeighborCommLB.h NborBaseLB.h BaseLB.h NborBaseLB.decl.h \
+  NeighborLBMsg.h topology.h NeighborCommLB.decl.h NeighborCommLB.def.h
        $(CHARMC) -c -I. NeighborCommLB.C
 
 HybridLB.o: HybridLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1560,13 +1606,14 @@ HybridLB.o: HybridLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h HybridLB.h BaseLB.h CentralLB.h CentralLB.decl.h \
-  CentralLBMsg.h HybridBaseLB.h HybridBaseLB.decl.h NeighborLBMsg.h \
-  HybridLBMsg.h topology.h HybridLB.decl.h GreedyLB.h GreedyLB.decl.h \
-  GreedyCommLB.h GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h \
-  RefineCommLB.h RefinerComm.h Refiner.h RefineLB.h RefineLB.decl.h \
-  RefineCommLB.decl.h HybridLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h HybridLB.h BaseLB.h \
+  CentralLB.h CentralLB.decl.h CentralLBMsg.h HybridBaseLB.h \
+  HybridBaseLB.decl.h NeighborLBMsg.h HybridLBMsg.h topology.h \
+  HybridLB.decl.h GreedyLB.h GreedyLB.decl.h GreedyCommLB.h \
+  GreedyCommLB.decl.h elements.h Set.h heap.h CommLBHeap.h RefineCommLB.h \
+  RefinerComm.h Refiner.h RefineLB.h RefineLB.decl.h RefineCommLB.decl.h \
+  HybridLB.def.h
        $(CHARMC) -c -I. HybridLB.C
 
 TopoLB.o: TopoLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1583,9 +1630,10 @@ TopoLB.o: TopoLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
-  TopoLB.decl.h TopoLB.h topology.h TopoLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h TopoLB.decl.h TopoLB.h topology.h \
+  TopoLB.def.h
        $(CHARMC) -c -I. TopoLB.C
 
 RefineTopoLB.o: RefineTopoLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1602,10 +1650,10 @@ RefineTopoLB.o: RefineTopoLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
-  RefineTopoLB.decl.h RefineTopoLB.h TopoLB.h topology.h \
-  RefineTopoLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h RefineTopoLB.decl.h RefineTopoLB.h \
+  TopoLB.h topology.h RefineTopoLB.def.h
        $(CHARMC) -c -I. RefineTopoLB.C
 
 TopoCentLB.o: TopoCentLB.C charm++.h charm.h converse.h conv-config.h \
@@ -1622,9 +1670,10 @@ TopoCentLB.o: TopoCentLB.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h \
-  TopoCentLB.decl.h TopoCentLB.h topology.h TopoCentLB.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h TopoCentLB.decl.h TopoCentLB.h \
+  topology.h TopoCentLB.def.h
        $(CHARMC) -c -I. TopoCentLB.C
 
 manager.o: manager.C manager.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
@@ -1641,8 +1690,9 @@ manager.o: manager.C manager.h CentralLB.h BaseLB.h LBDatabase.h lbdb.h \
   ComlibArrayListener.h ComlibStrategy.h convcomlibstrategy.h \
   ComlibLearner.h envelope.h CkFutures.decl.h charisma.h charisma.decl.h \
   tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h \
-  CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
-  NullLB.decl.h BaseLB.decl.h CentralLB.decl.h CentralLBMsg.h
+  CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h NullLB.decl.h BaseLB.decl.h CentralLB.decl.h \
+  CentralLBMsg.h
        $(CHARMC) -c -I. manager.C
 
 blue.o: blue.C cklists.h pup.h converse.h conv-config.h conv-common.h \
@@ -1723,8 +1773,8 @@ blue_stub.o: blue_stub.C charm++.h charm.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h
        $(CHARMC) -c -I. blue_stub.C
 
 blue_standalone.o: blue_standalone.C
@@ -1751,16 +1801,17 @@ ComlibManager.o: ComlibManager.C ComlibManager.h charm++.h charm.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h EachToManyMulticastStrategy.h routerstrategy.h \
-  gridrouter.h petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h \
-  prefixrouter.h DirectMulticastStrategy.h ComlibSectionInfo.h \
-  StreamingStrategy.h DummyStrategy.h MPIStrategy.h NodeMulticast.h \
-  MsgPacker.h register.h pup_cmialloc.h RingMulticastStrategy.h \
-  MultiRingMulticast.h PipeBroadcastStrategy.h pipebroadcastconverse.h \
-  BroadcastStrategy.h MeshStreamingStrategy.h PrioStreaming.h qd.h \
-  CentralLB.h BaseLB.h CentralLB.decl.h CentralLBMsg.h comlib.def.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h \
+  EachToManyMulticastStrategy.h routerstrategy.h gridrouter.h petable.h \
+  graphrouter.h de.h treerouter.h 3dgridrouter.h prefixrouter.h \
+  DirectMulticastStrategy.h ComlibSectionInfo.h StreamingStrategy.h \
+  DummyStrategy.h MPIStrategy.h NodeMulticast.h MsgPacker.h register.h \
+  pup_cmialloc.h RingMulticastStrategy.h MultiRingMulticast.h \
+  PipeBroadcastStrategy.h pipebroadcastconverse.h BroadcastStrategy.h \
+  MeshStreamingStrategy.h PrioStreaming.h qd.h CentralLB.h BaseLB.h \
+  CentralLB.decl.h CentralLBMsg.h comlib.def.h
        $(CHARMC) -c -I. ComlibManager.C
 
 MPIStrategy.o: MPIStrategy.C MPIStrategy.h ComlibManager.h charm++.h \
@@ -1778,8 +1829,8 @@ MPIStrategy.o: MPIStrategy.C MPIStrategy.h ComlibManager.h charm++.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
-  convcomlibmanager.h ComlibStats.h comlib.decl.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h
        $(CHARMC) -c -I. MPIStrategy.C
 
 StreamingStrategy.o: StreamingStrategy.C StreamingStrategy.h \
@@ -1797,9 +1848,10 @@ StreamingStrategy.o: StreamingStrategy.C StreamingStrategy.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h MsgPacker.h register.h pup_cmialloc.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h MsgPacker.h register.h \
+  pup_cmialloc.h
        $(CHARMC) -c -I. StreamingStrategy.C
 
 DummyStrategy.o: DummyStrategy.C DummyStrategy.h ComlibManager.h \
@@ -1817,8 +1869,8 @@ DummyStrategy.o: DummyStrategy.C DummyStrategy.h ComlibManager.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
-  convcomlibmanager.h ComlibStats.h comlib.decl.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h
        $(CHARMC) -c -I. DummyStrategy.C
 
 NodeMulticast.o: NodeMulticast.C NodeMulticast.h ComlibManager.h \
@@ -1836,8 +1888,8 @@ NodeMulticast.o: NodeMulticast.C NodeMulticast.h ComlibManager.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
-  convcomlibmanager.h ComlibStats.h comlib.decl.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h
        $(CHARMC) -c -I. NodeMulticast.C
 
 EachToManyMulticastStrategy.o: EachToManyMulticastStrategy.C \
@@ -1855,11 +1907,11 @@ EachToManyMulticastStrategy.o: EachToManyMulticastStrategy.C \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h routerstrategy.h gridrouter.h petable.h graphrouter.h \
-  de.h treerouter.h 3dgridrouter.h prefixrouter.h AAPLearner.h \
-  AAMLearner.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h routerstrategy.h \
+  gridrouter.h petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h \
+  prefixrouter.h AAPLearner.h AAMLearner.h
        $(CHARMC) -c -I. EachToManyMulticastStrategy.C
 
 ComlibSectionInfo.o: ComlibSectionInfo.C ComlibManager.h charm++.h \
@@ -1877,8 +1929,9 @@ ComlibSectionInfo.o: ComlibSectionInfo.C ComlibManager.h charm++.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
-  convcomlibmanager.h ComlibStats.h comlib.decl.h ComlibSectionInfo.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h \
+  ComlibSectionInfo.h
        $(CHARMC) -c -I. ComlibSectionInfo.C
 
 AAPLearner.o: AAPLearner.C AAPLearner.h comlib.h converse.h conv-config.h \
@@ -1895,11 +1948,11 @@ AAPLearner.o: AAPLearner.C AAPLearner.h comlib.h converse.h conv-config.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h ComlibManager.h convcomlibmanager.h \
-  ComlibStats.h comlib.decl.h EachToManyMulticastStrategy.h \
-  routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
-  3dgridrouter.h prefixrouter.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h ComlibManager.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h \
+  EachToManyMulticastStrategy.h routerstrategy.h gridrouter.h petable.h \
+  graphrouter.h de.h treerouter.h 3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. AAPLearner.C
 
 ComlibStats.o: ComlibStats.C ComlibStats.h charm++.h charm.h converse.h \
@@ -1916,9 +1969,9 @@ ComlibStats.o: ComlibStats.C ComlibStats.h charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h convcomlibmanager.h comlib.h router.h ComlibManager.h \
-  comlib.decl.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h convcomlibmanager.h \
+  comlib.h router.h ComlibManager.h comlib.decl.h
        $(CHARMC) -c -I. ComlibStats.C
 
 AAMLearner.o: AAMLearner.C AAMLearner.h ComlibManager.h charm++.h charm.h \
@@ -1935,11 +1988,11 @@ AAMLearner.o: AAMLearner.C AAMLearner.h ComlibManager.h charm++.h charm.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h AAPLearner.h EachToManyMulticastStrategy.h \
-  routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
-  3dgridrouter.h prefixrouter.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h AAPLearner.h \
+  EachToManyMulticastStrategy.h routerstrategy.h gridrouter.h petable.h \
+  graphrouter.h de.h treerouter.h 3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. AAMLearner.C
 
 MsgPacker.o: MsgPacker.C MsgPacker.h charm++.h charm.h converse.h \
@@ -1956,9 +2009,10 @@ MsgPacker.o: MsgPacker.C MsgPacker.h charm++.h charm.h converse.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h ComlibManager.h comlib.h router.h convcomlibmanager.h \
-  ComlibStats.h comlib.decl.h register.h pup_cmialloc.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h ComlibManager.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h register.h \
+  pup_cmialloc.h
        $(CHARMC) -c -I. MsgPacker.C
 
 DirectMulticastStrategy.o: DirectMulticastStrategy.C \
@@ -1976,9 +2030,9 @@ DirectMulticastStrategy.o: DirectMulticastStrategy.C \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h ComlibSectionInfo.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h ComlibSectionInfo.h
        $(CHARMC) -c -I. DirectMulticastStrategy.C
 
 RingMulticastStrategy.o: RingMulticastStrategy.C RingMulticastStrategy.h \
@@ -1996,9 +2050,9 @@ RingMulticastStrategy.o: RingMulticastStrategy.C RingMulticastStrategy.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h ComlibSectionInfo.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h ComlibSectionInfo.h
        $(CHARMC) -c -I. RingMulticastStrategy.C
 
 MultiRingMulticast.o: MultiRingMulticast.C MultiRingMulticast.h \
@@ -2016,9 +2070,9 @@ MultiRingMulticast.o: MultiRingMulticast.C MultiRingMulticast.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h ComlibSectionInfo.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h ComlibSectionInfo.h
        $(CHARMC) -c -I. MultiRingMulticast.C
 
 PipeBroadcastStrategy.o: PipeBroadcastStrategy.C PipeBroadcastStrategy.h \
@@ -2036,9 +2090,9 @@ PipeBroadcastStrategy.o: PipeBroadcastStrategy.C PipeBroadcastStrategy.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h pipebroadcastconverse.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h pipebroadcastconverse.h
        $(CHARMC) -c -I. PipeBroadcastStrategy.C
 
 MeshStreamingStrategy.o: MeshStreamingStrategy.C MeshStreamingStrategy.h \
@@ -2056,9 +2110,10 @@ MeshStreamingStrategy.o: MeshStreamingStrategy.C MeshStreamingStrategy.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h pup_cmialloc.h MsgPacker.h register.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h pup_cmialloc.h \
+  MsgPacker.h register.h
        $(CHARMC) -c -I. MeshStreamingStrategy.C
 
 PrioStreaming.o: PrioStreaming.C PrioStreaming.h ComlibManager.h \
@@ -2076,9 +2131,9 @@ PrioStreaming.o: PrioStreaming.C PrioStreaming.h ComlibManager.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
-  convcomlibmanager.h ComlibStats.h comlib.decl.h StreamingStrategy.h \
-  MsgPacker.h register.h pup_cmialloc.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h \
+  router.h convcomlibmanager.h ComlibStats.h comlib.decl.h \
+  StreamingStrategy.h MsgPacker.h register.h pup_cmialloc.h
        $(CHARMC) -c -I. PrioStreaming.C
 
 BroadcastStrategy.o: BroadcastStrategy.C BroadcastStrategy.h \
@@ -2096,9 +2151,9 @@ BroadcastStrategy.o: BroadcastStrategy.C BroadcastStrategy.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h comlib.h router.h \
+  convcomlibmanager.h ComlibStats.h comlib.decl.h
        $(CHARMC) -c -I. BroadcastStrategy.C
 
 gridrouter.o: gridrouter.C gridrouter.h converse.h conv-config.h \
@@ -2115,8 +2170,8 @@ gridrouter.o: gridrouter.C gridrouter.h converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. gridrouter.C
 
 de.o: de.C de.h converse.h conv-config.h conv-common.h conv-mach.h \
@@ -2133,8 +2188,8 @@ de.o: de.C de.h converse.h conv-config.h conv-common.h conv-mach.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. de.C
 
 treerouter.o: treerouter.C treerouter.h converse.h conv-config.h \
@@ -2151,8 +2206,8 @@ treerouter.o: treerouter.C treerouter.h converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. treerouter.C
 
 petable.o: petable.C converse.h conv-config.h conv-common.h conv-mach.h \
@@ -2169,8 +2224,8 @@ petable.o: petable.C converse.h conv-config.h conv-common.h conv-mach.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. petable.C
 
 3dgridrouter.o: 3dgridrouter.C 3dgridrouter.h converse.h conv-config.h \
@@ -2187,8 +2242,8 @@ petable.o: petable.C converse.h conv-config.h conv-common.h conv-mach.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. 3dgridrouter.C
 
 graphrouter.o: graphrouter.C graphrouter.h converse.h conv-config.h \
@@ -2205,8 +2260,9 @@ graphrouter.o: graphrouter.C graphrouter.h converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h hypercubetopology.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h \
+  hypercubetopology.h
        $(CHARMC) -c -I. graphrouter.C
 
 hypercubetopology.o: hypercubetopology.C hypercubetopology.h \
@@ -2224,8 +2280,8 @@ hypercubetopology.o: hypercubetopology.C hypercubetopology.h \
   CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h petable.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h petable.h
        $(CHARMC) -c -I. hypercubetopology.C
 
 routerstrategy.o: routerstrategy.C routerstrategy.h convcomlibmanager.h \
@@ -2243,8 +2299,9 @@ routerstrategy.o: routerstrategy.C routerstrategy.h convcomlibmanager.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h gridrouter.h \
-  petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h prefixrouter.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h router.h \
+  gridrouter.h petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h \
+  prefixrouter.h
        $(CHARMC) -c -I. routerstrategy.C
 
 convcomlibmanager.o: convcomlibmanager.C convcomlibmanager.h converse.h \
@@ -2262,7 +2319,7 @@ convcomlibmanager.o: convcomlibmanager.C convcomlibmanager.h converse.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h \
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h router.h \
   routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
   3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. convcomlibmanager.C
@@ -2282,8 +2339,8 @@ pipebroadcastconverse.o: pipebroadcastconverse.C pipebroadcastconverse.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h convcomlibmanager.h \
-  comlib.h router.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
+  convcomlibmanager.h comlib.h router.h
        $(CHARMC) -c -I. pipebroadcastconverse.C
 
 pipelinestrategy.o: pipelinestrategy.C pipelinestrategy.h ckhashtable.h \
@@ -2301,8 +2358,8 @@ pipelinestrategy.o: pipelinestrategy.C pipelinestrategy.h ckhashtable.h \
   ComlibStrategy.h convcomlibstrategy.h ComlibLearner.h envelope.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
-  ckarrayreductionmgr.h trace.h trace-bluegene.h convcomlibmanager.h \
-  comlib.h router.h
+  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h \
+  convcomlibmanager.h comlib.h router.h
        $(CHARMC) -c -I. pipelinestrategy.C
 
 prefixrouter.o: prefixrouter.C prefixrouter.h converse.h conv-config.h \
@@ -2319,7 +2376,7 @@ prefixrouter.o: prefixrouter.C prefixrouter.h converse.h conv-config.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
-  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
-  trace-bluegene.h router.h
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h router.h
        $(CHARMC) -c -I. prefixrouter.C
 
index 75381b04c8377fb498d084f9a70774ed08802938..e1d9a6c3467dcf61f5cd1807d1c86a6d98abf1f3 100644 (file)
@@ -174,7 +174,7 @@ CKHEADERS=ck.h ckstream.h envelope.h init.h qd.h charm.h charm++.h \
          ckfutures.h \
          ckcallback.h CkCallback.decl.h ckcallback-ccs.h       \
          ckarrayreductionmgr.h cknodegroupreduction.h cksection.h \
-         ckarray.h cklocation.h ckreduction.h ckcheckpoint.h ckmemcheckpoint.h\
+         ckarray.h cklocation.h ckreduction.h ckcheckpoint.h ckmemcheckpoint.h ckevacuation.h\
          ckobjQ.h readonly.h charisma.h ComlibArrayListener.h ComlibStrategy.h\
          ComlibLearner.h $(UTILHEADERS) \
          tempo.h waitqd.h LBDatabase.h lbdb.h LBProfit.h LBDBManager.h \
@@ -539,7 +539,7 @@ LIBCK_CORE=trace-common.o tracef.o init.o register.o qd.o ck.o main.o       \
           msgalloc.o ckfutures.o debug-message.o debug-charm.o ckcallback.o \
           cklocation.o ckarray.o ckreduction.o ckarrayreductionmgr.o\
            tempo.o waitqd.o LBDatabase.o lbdb.o lbdbf.o charisma.o ckobjQ.o  \
-          LBAgent.o LBProfit.o ckcheckpoint.o ckmemcheckpoint.o\
+          LBAgent.o LBProfit.o ckcheckpoint.o ckmemcheckpoint.o ckevacuation.o\
            LBDBManager.o LBComm.o LBObj.o LBMachineUtil.o CentralPredictor.o \
           BaseLB.o CentralLB.o HybridBaseLB.o NborBaseLB.o WSLB.o \
            ObjGraph.o graph.o LButil.o Refiner.o RefinerApprox.o  \