fix for migration
[charm.git] / src / ck-core / cklocation.C
index b0211262f3e6e2ab1fab0e87d10738fac1e93631..97602046eedd00641cb48d8f35eac00810116856 100644 (file)
@@ -1050,6 +1050,7 @@ void CkMigratable::commonInit(void) {
        thisChareType=i.chareType;
        usesAtSync=CmiFalse;
        usesAutoMeasure=CmiTrue;
        thisChareType=i.chareType;
        usesAtSync=CmiFalse;
        usesAutoMeasure=CmiTrue;
+       usesChkpAtSync = CmiFalse;
        barrierRegistered=CmiFalse;
 
   local_state = OFF;
        barrierRegistered=CmiFalse;
 
   local_state = OFF;
@@ -1084,10 +1085,11 @@ void CkMigratable::pup(PUP::er &p) {
        p|thisIndexMax;
        p(usesAtSync);
   p(can_reset);
        p|thisIndexMax;
        p(usesAtSync);
   p(can_reset);
+    p(usesChkpAtSync);
        p(usesAutoMeasure);
 #if CMK_LBDB_ON 
        int readyMigrate;
        p(usesAutoMeasure);
 #if CMK_LBDB_ON 
        int readyMigrate;
-       if (p.isPacking()) readyMigrate = myRec->isReadyMigrate();
+       if (p.isPacking()||p.isCalChecking()) readyMigrate = myRec->isReadyMigrate();
        p|readyMigrate;
        if (p.isUnpacking()) myRec->ReadyMigrate(readyMigrate);
 #endif
        p|readyMigrate;
        if (p.isUnpacking()) myRec->ReadyMigrate(readyMigrate);
 #endif
@@ -1098,6 +1100,13 @@ void CkMigratable::pup(PUP::er &p) {
        p | asyncEvacuate;
        if(p.isUnpacking()){myRec->AsyncEvacuate(asyncEvacuate);}
        
        p | asyncEvacuate;
        if(p.isUnpacking()){myRec->AsyncEvacuate(asyncEvacuate);}
        
+       if(p.isUnpacking()){
+         atsync_chkp_iter = -1;
+       }
+       if(p.isUnpacking()){
+               resetForChkp();
+       }
+
        ckFinishConstruction();
 }
 
        ckFinishConstruction();
 }
 
@@ -1128,6 +1137,7 @@ CkMigratable::~CkMigratable() {
 #if CMK_LBDB_ON 
        if (barrierRegistered) {
          DEBL((AA"Removing barrier for element %s\n"AB,idx2str(thisIndexMax)));
 #if CMK_LBDB_ON 
        if (barrierRegistered) {
          DEBL((AA"Removing barrier for element %s\n"AB,idx2str(thisIndexMax)));
+         //CkPrintf("Removing barrier for element %s\n",idx2str(thisIndexMax));
          if (usesAtSync)
                myRec->getLBDB()->RemoveLocalBarrierClient(ldBarrierHandle);
          else
          if (usesAtSync)
                myRec->getLBDB()->RemoveLocalBarrierClient(ldBarrierHandle);
          else
@@ -1210,6 +1220,7 @@ void CkMigratable::ckFinishConstruction(void)
        myRec->setMeasure(usesAutoMeasure);
        if (barrierRegistered) return;
        DEBL((AA"Registering barrier client for %s\n"AB,idx2str(thisIndexMax)));
        myRec->setMeasure(usesAutoMeasure);
        if (barrierRegistered) return;
        DEBL((AA"Registering barrier client for %s\n"AB,idx2str(thisIndexMax)));
+       //CkPrintf("Registering barrier client for %s\n",idx2str(thisIndexMax));
         if (usesAtSync)
          ldBarrierHandle = myRec->getLBDB()->AddLocalBarrierClient(
                (LDBarrierFn)staticResumeFromSync,(void*)(this));
         if (usesAtSync)
          ldBarrierHandle = myRec->getLBDB()->AddLocalBarrierClient(
                (LDBarrierFn)staticResumeFromSync,(void*)(this));
@@ -1219,6 +1230,97 @@ void CkMigratable::ckFinishConstruction(void)
        barrierRegistered=CmiTrue;
 }
 
        barrierRegistered=CmiTrue;
 }
 
+
+void CkMigratable::setChkpResumeClient(CkCallback & _cb)
+{
+        //if(CmiMyPartition()==1){
+          CkPrintf("[%d] setChkpResumeClient\n",CkMyPe());
+        //}
+       chkp_cb = _cb;
+       nextChkpIter = -1;
+       nextChkpDecided = false;
+       atsync_chkp_iter = -1;
+       local_chkp_pause = false;
+       usesChkpAtSync = CmiTrue;
+}
+
+void CkMigratable::AtChkpSync()
+{
+       if(usesChkpAtSync){
+         if(CkMyPe()==0){
+//         CkPrintf("at chkp sync\n");
+         }
+               if(CmiNumPartition()==1){
+                       chkp_cb.send();
+                       return;
+               }
+               atsync_chkp_iter++;
+               myRec->getChkpMgr()->recvIter(atsync_chkp_iter);
+               //reduction to decide the current maximum
+               if(nextChkpDecided){
+                       if(atsync_chkp_iter<nextChkpIter){
+                               chkp_cb.send();
+                       }else if(atsync_chkp_iter == nextChkpIter){
+                               resetForChkp();
+                               myRec->getChkpMgr()->reachChkpIter();
+                       }else{
+                               CkAbort("Impossible state\n");
+                       }
+               }
+               else if(myRec->getChkpMgr()->localDecided){
+                       int localIter = myRec->getChkpMgr()->localMaxIter;
+                       if(atsync_chkp_iter==localIter){
+                               local_chkp_pause = true;
+                       }
+                       else if(atsync_chkp_iter<localIter){
+                               chkp_cb.send();
+                       }
+                       else{
+                               CkAbort("local Impossible state\n");
+                       }
+               }       
+               else{
+                       chkp_cb.send();
+               }
+       }
+}
+
+void CkMigratable::resetForChkp(){
+       nextChkpDecided = false;
+       local_chkp_pause = false;
+}
+
+void CkMigratable::ResumeFromChkp(){
+       
+       if(usesChkpAtSync&&!chkp_cb.isInvalid()){
+               chkp_cb.send();
+       }
+}
+
+void CkMigratable::recvChkpIter(void * _iter){
+       if(usesChkpAtSync){
+               int iter = *(int *)_iter;
+               nextChkpIter = iter;
+               nextChkpDecided = true;
+               //if(CkMyPe()==0){
+               //  CkPrintf("receive chkp iter %d %d\n",atsync_chkp_iter, nextChkpIter);
+               //}
+               if(atsync_chkp_iter>nextChkpIter){
+                       CkAbort("impossible state in notify\n");
+               }
+               else if(atsync_chkp_iter==nextChkpIter){
+                       resetForChkp();
+                       myRec->getChkpMgr()->reachChkpIter();
+               }
+               else{
+                       if(local_chkp_pause){
+                               if(!chkp_cb.isInvalid())
+                                       chkp_cb.send(); 
+                       }
+               }
+       }
+}
+
 void CkMigratable::AtSync(int waitForMigration)
 {
        if (!usesAtSync)
 void CkMigratable::AtSync(int waitForMigration)
 {
        if (!usesAtSync)
@@ -1229,7 +1331,9 @@ void CkMigratable::AtSync(int waitForMigration)
        myRec->AsyncMigrate(!waitForMigration);
        if (waitForMigration) ReadyMigrate(CmiTrue);
        ckFinishConstruction();
        myRec->AsyncMigrate(!waitForMigration);
        if (waitForMigration) ReadyMigrate(CmiTrue);
        ckFinishConstruction();
-  DEBL((AA"Element %s going to sync\n"AB,idx2str(thisIndexMax)));
+  //DEBL((AA"Element %s going to sync\n"AB,idx2str(thisIndexMax)));
+  //if(CmiMyPartition()==1)
+//     CkPrintf("Element %s going to sync\n",idx2str(thisIndexMax));
   // model-based load balancing, ask user to provide cpu load
   if (usesAutoMeasure == CmiFalse) UserSetLBLoad();
 
   // model-based load balancing, ask user to provide cpu load
   if (usesAutoMeasure == CmiFalse) UserSetLBLoad();
 
@@ -1417,6 +1521,7 @@ CkLocRec_local::CkLocRec_local(CkLocMgr *mgr,CmiBool fromMigration,
        bounced  = CmiFalse;
        the_lbdb=mgr->getLBDB();
        the_metalb=mgr->getMetaBalancer();
        bounced  = CmiFalse;
        the_lbdb=mgr->getLBDB();
        the_metalb=mgr->getMetaBalancer();
+       the_chkp = mgr->getChkpMgr();
        LDObjid ldid = idx2LDObjid(idx);
 #if CMK_GLOBAL_LOCATION_UPDATE
         ldid.locMgrGid = mgr->getGroupID().idx;
        LDObjid ldid = idx2LDObjid(idx);
 #if CMK_GLOBAL_LOCATION_UPDATE
         ldid.locMgrGid = mgr->getGroupID().idx;
@@ -1432,6 +1537,7 @@ CkLocRec_local::CkLocRec_local(CkLocMgr *mgr,CmiBool fromMigration,
                }
        }
 #endif
                }
        }
 #endif
+       the_chkp = mgr->getChkpMgr();
        /*
                FAULT_EVAC
        */
        /*
                FAULT_EVAC
        */
@@ -1982,6 +2088,7 @@ inline void CkLocMgr::springCleaning(void)
     if (rec->isObsolete(nSprings,idx)) {
       //This record is obsolete-- remove it from the table
       DEBK((AA"Cleaning out old record %s\n"AB,idx2str(idx)));
     if (rec->isObsolete(nSprings,idx)) {
       //This record is obsolete-- remove it from the table
       DEBK((AA"Cleaning out old record %s\n"AB,idx2str(idx)));
+      //CkPrintf("[%d][%d]Cleaning out old record %s\n",CmiMyPartition(),CkMyPe(),idx2str(idx));
       hash.remove(*(CkArrayIndex *)&idx);
       delete rec;
       it->seek(-1);//retry this hash slot
       hash.remove(*(CkArrayIndex *)&idx);
       delete rec;
       it->seek(-1);//retry this hash slot
@@ -1994,16 +2101,26 @@ void CkLocMgr::staticSpringCleaning(void *forWhom,double curWallTime) {
        DEBK((AA"Starting spring cleaning at %.2f\n"AB,CkWallTimer()));
        ((CkLocMgr *)forWhom)->springCleaning();
 }
        DEBK((AA"Starting spring cleaning at %.2f\n"AB,CkWallTimer()));
        ((CkLocMgr *)forWhom)->springCleaning();
 }
+static const char *rec2str[]={
+    "base (INVALID)",//Base class (invalid type)
+    "local",//Array element that lives on this Pe
+    "remote",//Array element that lives on some other Pe
+    "buffering",//Array element that was just created
+    "dead"//Deleted element (for debugging)
+};
+
 //doesn't delete if there is extra pe
 void CkLocMgr::flushLocalRecs(void)
 {
   void *objp;
   void *keyp;
 //doesn't delete if there is extra pe
 void CkLocMgr::flushLocalRecs(void)
 {
   void *objp;
   void *keyp;
-  CkHashtableIterator *it=hash.iterator();
+  CkHashtableIterator *it=localHash.iterator();
   CmiImmediateLock(hashImmLock);
   while (NULL!=(objp=it->next(&keyp))) {
     CkLocRec *rec=*(CkLocRec **)objp;
     CkArrayIndex &idx=*(CkArrayIndex *)keyp;
   CmiImmediateLock(hashImmLock);
   while (NULL!=(objp=it->next(&keyp))) {
     CkLocRec *rec=*(CkLocRec **)objp;
     CkArrayIndex &idx=*(CkArrayIndex *)keyp;
+    //if(CmiMyPartition()==1)
+      //CkPrintf("[%d][%d] flush home object %s %p %p\n",CmiMyPartition(),CkMyPe(),idx2str(idx),rec,hash.get(*(CkArrayIndex *)&idx));
     if (rec->type() == CkLocRec::local) {
         callMethod((CkLocRec_local*)rec, &CkMigratable::ckDestroy);
         it->seek(-1);//retry this hash slot
     if (rec->type() == CkLocRec::local) {
         callMethod((CkLocRec_local*)rec, &CkMigratable::ckDestroy);
         it->seek(-1);//retry this hash slot
@@ -2013,6 +2130,39 @@ void CkLocMgr::flushLocalRecs(void)
   CmiImmediateUnlock(hashImmLock);
 }
 
   CmiImmediateUnlock(hashImmLock);
 }
 
+void CkLocMgr::recvChkpIter(int iter)
+{
+  void *objp;
+  void *keyp;
+  CkHashtableIterator *it=localHash.iterator();
+  CmiImmediateLock(hashImmLock);
+  while (NULL!=(objp=it->next(&keyp))) {
+    CkLocRec *rec=*(CkLocRec **)objp;
+    CkArrayIndex &idx=*(CkArrayIndex *)keyp;
+    if (rec->type() == CkLocRec::local) {
+        callMethod((CkLocRec_local*)rec, &CkMigratable::recvChkpIter,&iter);
+    }
+  }
+  delete it;
+  CmiImmediateUnlock(hashImmLock);
+}
+
+void CkLocMgr::resumeFromChkp(){
+  void *objp;
+  void *keyp;
+  CkHashtableIterator *it=localHash.iterator();
+  CmiImmediateLock(hashImmLock);
+  while (NULL!=(objp=it->next(&keyp))) {
+    CkLocRec *rec=*(CkLocRec **)objp;
+    CkArrayIndex &idx=*(CkArrayIndex *)keyp;
+    if (rec->type() == CkLocRec::local) {
+        callMethod((CkLocRec_local*)rec, &CkMigratable::ResumeFromChkp);
+    }
+  }
+  delete it;
+  CmiImmediateUnlock(hashImmLock);
+}
+
 // clean all buffer'ed messages and also free local objects
 void CkLocMgr::flushAllRecs(void)
 {
 // clean all buffer'ed messages and also free local objects
 void CkLocMgr::flushAllRecs(void)
 {
@@ -2091,6 +2241,8 @@ CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkGroupID metalbID_,CkArra
   metalbID = metalbID_;
        initLB(lbdbID_, metalbID_);
        hashImmLock = CmiCreateImmediateLock();
   metalbID = metalbID_;
        initLB(lbdbID_, metalbID_);
        hashImmLock = CmiCreateImmediateLock();
+       chkpID = ckCheckPTGroupID;
+       the_chkp = (CkMemCheckPT *)CkLocalBranch(chkpID);
 }
 
 CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkArrayIndex& numInitial)
 }
 
 CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkArrayIndex& numInitial)
@@ -2125,6 +2277,7 @@ void CkLocMgr::pup(PUP::er &p){
        p|mapHandle;
        p|lbdbID;
   p|metalbID;
        p|mapHandle;
        p|lbdbID;
   p|metalbID;
+   p|chkpID;
        mapID = _defaultArrayMapID;
        if(p.isUnpacking()){
                thisProxy=thisgroup;
        mapID = _defaultArrayMapID;
        if(p.isUnpacking()){
                thisProxy=thisgroup;
@@ -2137,6 +2290,7 @@ void CkLocMgr::pup(PUP::er &p){
                map->registerArray(emptyIndex,thisgroup);
                // _lbdb is the fixed global groupID
                initLB(lbdbID, metalbID);
                map->registerArray(emptyIndex,thisgroup);
                // _lbdb is the fixed global groupID
                initLB(lbdbID, metalbID);
+               the_chkp = (CkMemCheckPT *)CkLocalBranch(chkpID);
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) ||CMK_MEM_CHECKPOINT    
         int count;
         p | count;
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) ||CMK_MEM_CHECKPOINT    
         int count;
         p | count;
@@ -2173,30 +2327,35 @@ void CkLocMgr::pup(PUP::er &p){
         void *keyp;
         CkVec<int> pe_list;
         CkVec<CkArrayIndex> idx_list;
         void *keyp;
         CkVec<int> pe_list;
         CkVec<CkArrayIndex> idx_list;
-        CkHashtableIterator *it = hash.iterator();
-      while (NULL!=(objp=it->next(&keyp))) {
+        //CkHashtableIterator *it = hash.iterator();
+        CkHashtableIterator *it = homeObjHash.iterator();
+      while (homeObjHash.numObjects()!=0&&NULL!=(objp=it->next(&keyp))) {
           CkLocRec *rec=*(CkLocRec **)objp;
           CkArrayIndex &idx=*(CkArrayIndex *)keyp;
           CkLocRec *rec=*(CkLocRec **)objp;
           CkArrayIndex &idx=*(CkArrayIndex *)keyp;
-            if(rec->type() != CkLocRec::local){
-                if(homePe(idx) == CmiMyPe()){
-                  int pe;
-                  CkArrayIndex max = idx;
-                  pe = rec->lookupProcessor();
-                  idx_list.push_back(max);
-                  pe_list.push_back(pe);
-                    count++;
-                }
-            }
-        }
-        p | count;
 
 
-               // releasing iterator memory
-               delete it;
+          //if(CmiMyPartition()==0&&CkMyPe()==0)
+            //CkPrintf("[%d][%d] pup home object %s rec %s\n",CmiMyPartition(),CkMyPe(),idx2str(idx),rec2str[rec->type()]);
+            //CkPrintf("[%d][%d] pup home object %s rec %p\n",CmiMyPartition(),CkMyPe(),idx2str(idx),rec);
+          if(rec!=NULL&&rec->type() != CkLocRec::local){
+              if(homePe(idx) == CmiMyPe()){
+                int pe;
+                CkArrayIndex max = idx;
+                pe = rec->lookupProcessor();
+                idx_list.push_back(max);
+                pe_list.push_back(pe);
+                  count++;
+              }
+          }
+      }
+      p | count;
 
 
-      for(int i=0;i<pe_list.length();i++){
-        CkArrayIndex max = idx_list[i];
-        max.pup(p);
-        p|pe_list[i];
+              // releasing iterator memory
+              delete it;
+
+    for(int i=0;i<pe_list.length();i++){
+      CkArrayIndex max = idx_list[i];
+      max.pup(p);
+      p|pe_list[i];
       }
     /*    it = hash.iterator();
       while (NULL!=(objp=it->next(&keyp))) {
       }
     /*    it = hash.iterator();
       while (NULL!=(objp=it->next(&keyp))) {
@@ -2325,12 +2484,12 @@ CkLocRec_local *CkLocMgr::createLocal(const CkArrayIndex &idx,
                CmiBool forMigration, CmiBool ignoreArrival,
                CmiBool notifyHome)
 {
                CmiBool forMigration, CmiBool ignoreArrival,
                CmiBool notifyHome)
 {
+       //CkPrintf("Adding new record for element %s\n",idx2str(idx));
        int localIdx=nextFree();
        DEBC((AA"Adding new record for element %s at local index %d\n"AB,idx2str(idx),localIdx));
        int localIdx=nextFree();
        DEBC((AA"Adding new record for element %s at local index %d\n"AB,idx2str(idx),localIdx));
+       //CkPrintf("Adding new record for element %s at local index %d\n",idx2str(idx),localIdx);
        CkLocRec_local *rec=new CkLocRec_local(this,forMigration,ignoreArrival,idx,localIdx);
        insertRec(rec,idx); //Add to global hashtable
        CkLocRec_local *rec=new CkLocRec_local(this,forMigration,ignoreArrival,idx,localIdx);
        insertRec(rec,idx); //Add to global hashtable
-
-
        if (notifyHome) informHome(idx,CkMyPe());
        return rec;
 }
        if (notifyHome) informHome(idx,CkMyPe());
        return rec;
 }
@@ -2353,7 +2512,6 @@ CmiBool CkLocMgr::addElement(CkArrayID id,const CkArrayIndex &idx,
                   thisProxy.updateLocation(idx, CkMyPe());  
                 }
 #endif
                   thisProxy.updateLocation(idx, CkMyPe());  
                 }
 #endif
-                
        } else 
        { //rec is *already* local-- must not be the first insertion    
                rec=((CkLocRec_local *)oldRec);
        } else 
        { //rec is *already* local-- must not be the first insertion    
                rec=((CkLocRec_local *)oldRec);
@@ -2448,6 +2606,10 @@ void CkLocMgr::removeFromTable(const CkArrayIndex &idx) {
                CkAbort("CkLocMgr::removeFromTable called on invalid index!");
 #endif
         CmiImmediateLock(hashImmLock);
                CkAbort("CkLocMgr::removeFromTable called on invalid index!");
 #endif
         CmiImmediateLock(hashImmLock);
+       CkLocRec *old=elementNrec(idx);
+        if(old->type()==CkLocRec::local){
+          localHash.remove(*(CkArrayIndex *)&idx);
+        }
        hash.remove(*(CkArrayIndex *)&idx);
         CmiImmediateUnlock(hashImmLock);
 #if CMK_ERROR_CHECKING
        hash.remove(*(CkArrayIndex *)&idx);
         CmiImmediateUnlock(hashImmLock);
 #if CMK_ERROR_CHECKING
@@ -2711,7 +2873,46 @@ void CkLocMgr::iterate(CkLocIterator &dest) {
   delete it;
 }
 
   delete it;
 }
 
+void CkLocMgr::iterateLocal(CkLocIterator &dest) {
+  //Poke through the hash table for local ArrayRecs.
+  void *objp;
+  CkHashtableIterator *it=localHash.iterator();
+  CmiImmediateLock(hashImmLock);
+
+  while (NULL!=(objp=it->next())) {
+    CkLocRec *rec=*(CkLocRec **)objp;
+    if (rec->type()==CkLocRec::local) {
+      CkLocation loc(this,(CkLocRec_local *)rec);
+      dest.addLocation(loc);
+    }
+  }
+  CmiImmediateUnlock(hashImmLock);
+  delete it;
+}
+
+void CkLocMgr::iterateChkpSync(CkLocIterator &dest) {
+  //Poke through the hash table for local ArrayRecs.
+  void *objp;
+  CkHashtableIterator *it=localHash.iterator();
+  CmiImmediateLock(hashImmLock);
 
 
+  while (NULL!=(objp=it->next())) {
+    CkLocRec *rec=*(CkLocRec **)objp;
+    if (rec->type()==CkLocRec::local) {
+      CkLocation loc(this,(CkLocRec_local *)rec);
+         int localIdx=((CkLocRec_local *)rec)->getLocalIndex();
+         for (ManagerRec *m=firstManager;m!=NULL;m=m->next) {
+               CkMigratable *el=m->element(localIdx);
+       if(el->getChkpSync()==CmiTrue){
+                       dest.addLocation(loc);
+                       break;
+               }
+         }
+    }
+  }
+  CmiImmediateUnlock(hashImmLock);
+  delete it;
+}
 
 
 /************************** LocMgr: MIGRATION *************************/
 
 
 /************************** LocMgr: MIGRATION *************************/
@@ -2723,7 +2924,6 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
     register ManagerRec *m;
     int localIdx=rec->getLocalIndex();
     CkVec<CkMigratable *> dummyElts;
     register ManagerRec *m;
     int localIdx=rec->getLocalIndex();
     CkVec<CkMigratable *> dummyElts;
-
     for (m=firstManager;m!=NULL;m=m->next) {
         int elCType;
         if (!p.isUnpacking())
     for (m=firstManager;m!=NULL;m=m->next) {
         int elCType;
         if (!p.isUnpacking())
@@ -2736,8 +2936,8 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
         if (p.isUnpacking() && elCType!=-1) {
             CkMigratable *elt=m->mgr->allocateMigrated(elCType,rec->getIndex(),type);
             int migCtorIdx=_chareTable[elCType]->getMigCtor();
         if (p.isUnpacking() && elCType!=-1) {
             CkMigratable *elt=m->mgr->allocateMigrated(elCType,rec->getIndex(),type);
             int migCtorIdx=_chareTable[elCType]->getMigCtor();
-                if(!dummy){
-                       if(create)
+                       if(!dummy){
+                               if(create)
                                if (!addElementToRec(rec,m,elt,migCtorIdx,NULL)) return;
                                }else{
                     CkMigratable_initInfo &i=CkpvAccess(mig_initInfo);
                                if (!addElementToRec(rec,m,elt,migCtorIdx,NULL)) return;
                                }else{
                     CkMigratable_initInfo &i=CkpvAccess(mig_initInfo);
@@ -2747,7 +2947,7 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
                     if (!rec->invokeEntry(elt,NULL,migCtorIdx,CmiTrue)) return ;
                 }
         }
                     if (!rec->invokeEntry(elt,NULL,migCtorIdx,CmiTrue)) return ;
                 }
         }
-    }
+       }
     if(!dummy){
         for (m=firstManager;m!=NULL;m=m->next) {
             CkMigratable *elt=m->element(localIdx);
     if(!dummy){
         for (m=firstManager;m!=NULL;m=m->next) {
             CkMigratable *elt=m->element(localIdx);
@@ -2773,7 +2973,7 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
 }
 #else
 void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
 }
 #else
 void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
-               CkElementCreation_t type,CmiBool rebuild)
+               CkElementCreation_t type,CmiBool rebuild,CmiBool create)
 {
        p.comment("-------- Array Location --------");
        register ManagerRec *m;
 {
        p.comment("-------- Array Location --------");
        register ManagerRec *m;
@@ -2789,26 +2989,30 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
                        if (elt) elCType=elt->ckGetChareType();
                        else elCType=-1; //Element hasn't been created
                }
                        if (elt) elCType=elt->ckGetChareType();
                        else elCType=-1; //Element hasn't been created
                }
+
                p(elCType);
                if (p.isUnpacking() && elCType!=-1) {
                        //Create the element
                        CkMigratable *elt=m->mgr->allocateMigrated(elCType,rec->getIndex(),type);
                        int migCtorIdx=_chareTable[elCType]->getMigCtor();
                        //Insert into our tables and call migration constructor
                p(elCType);
                if (p.isUnpacking() && elCType!=-1) {
                        //Create the element
                        CkMigratable *elt=m->mgr->allocateMigrated(elCType,rec->getIndex(),type);
                        int migCtorIdx=_chareTable[elCType]->getMigCtor();
                        //Insert into our tables and call migration constructor
-                       if (!addElementToRec(rec,m,elt,migCtorIdx,NULL)) return;
+                       if(create)
+                               if (!addElementToRec(rec,m,elt,migCtorIdx,NULL)) return;
                }
        }
                }
        }
+
        //Next pup the element data
        for (m=firstManager;m!=NULL;m=m->next) {
                CkMigratable *elt=m->element(localIdx);
                if (elt!=NULL)
        //Next pup the element data
        for (m=firstManager;m!=NULL;m=m->next) {
                CkMigratable *elt=m->element(localIdx);
                if (elt!=NULL)
-                {
-                        elt->pup(p);
+               {
+                       elt->pup(p);
 #if CMK_ERROR_CHECKING
 #if CMK_ERROR_CHECKING
-                        if (p.isUnpacking()) elt->sanitycheck();
+                       if (p.isUnpacking()) elt->sanitycheck();
 #endif
 #endif
-                }
+               }
        }
        }
+
 #if CMK_MEM_CHECKPOINT
        if(rebuild){
          ArrayElement *elt;
 #if CMK_MEM_CHECKPOINT
        if(rebuild){
          ArrayElement *elt;
@@ -2824,7 +3028,6 @@ void CkLocMgr::pupElementsFor(PUP::er &p,CkLocRec_local *rec,
                  if (c) c->redNo = 0;
                }
          }
                  if (c) c->redNo = 0;
                }
          }
-               
        }
 #endif
 }
        }
 #endif
 }
@@ -3089,14 +3292,25 @@ void CkLocMgr::resume(const CkArrayIndex &idx, PUP::er &p, CmiBool create, int d
     }
 }
 #else
     }
 }
 #else
-void CkLocMgr::resume(const CkArrayIndex &idx, PUP::er &p, CmiBool notify,CmiBool rebuild)
+void CkLocMgr::resume(const CkArrayIndex &idx, PUP::er &p, CmiBool notify,CmiBool rebuild,CmiBool create)
 {
 {
-       CkLocRec_local *rec=createLocal(idx,CmiFalse,CmiFalse,notify /* home doesn't know yet */ );
-
-       //Create the new elements as we unpack the message
-       pupElementsFor(p,rec,CkElementCreation_resume,rebuild);
+       CkLocRec_local *rec;
+       CkLocRec *recGlobal;    
 
 
-       callMethod(rec,&CkMigratable::ckJustMigrated);
+       if(create){
+               //Create the new elements as we unpack the message
+               rec=createLocal(idx,CmiFalse,CmiFalse,notify /* home doesn't know yet */ );
+       }else{
+               recGlobal = elementNrec(idx);
+               if(recGlobal == NULL) 
+                       CmiAbort("Local object not found");
+               if(recGlobal->type() != CkLocRec::local)
+                       CmiAbort("Local object not local, :P");
+               rec = (CkLocRec_local *)recGlobal;
+       }
+       pupElementsFor(p,rec,CkElementCreation_resume,rebuild,create);
+       if(!p.isChecking())
+               callMethod(rec,&CkMigratable::ckJustMigrated);
 }
 #endif
 
 }
 #endif
 
@@ -3151,13 +3365,6 @@ bool CkLocMgr::isRemote(const CkArrayIndex &idx,int *onPe) const
        }
 }
 
        }
 }
 
-static const char *rec2str[]={
-    "base (INVALID)",//Base class (invalid type)
-    "local",//Array element that lives on this Pe
-    "remote",//Array element that lives on some other Pe
-    "buffering",//Array element that was just created
-    "dead"//Deleted element (for debugging)
-};
 
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 void CkLocMgr::setDuringMigration(CmiBool _duringMigration){
 
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 void CkLocMgr::setDuringMigration(CmiBool _duringMigration){
@@ -3172,6 +3379,8 @@ void CkLocMgr::insertRec(CkLocRec *rec,const CkArrayIndex &idx) {
        insertRecN(rec,idx);
        if (old!=NULL) {
                DEBC((AA"  replaces old rec(%s) for %s\n"AB,rec2str[old->type()],idx2str(idx)));
        insertRecN(rec,idx);
        if (old!=NULL) {
                DEBC((AA"  replaces old rec(%s) for %s\n"AB,rec2str[old->type()],idx2str(idx)));
+       //      if(CmiMyPartition()==1)
+        //          CkPrintf("[%d][%d]replaces old rec(%s) for %s %p new type %s\n",CmiMyPartition(),CkMyPe(),rec2str[old->type()],idx2str(idx),old,rec2str[rec->type()]);
                //There was an old element at this location
                if (old->type()==CkLocRec::local && rec->type()==CkLocRec::local) {
                    if (!CkInRestarting()) {    // ok if it is restarting
                //There was an old element at this location
                if (old->type()==CkLocRec::local && rec->type()==CkLocRec::local) {
                    if (!CkInRestarting()) {    // ok if it is restarting
@@ -3180,6 +3389,10 @@ void CkLocMgr::insertRec(CkLocRec *rec,const CkArrayIndex &idx) {
                    }
                }
                old->beenReplaced();
                    }
                }
                old->beenReplaced();
+                //need to remove it from homeObjHash
+               if (old->type()!=CkLocRec::local && rec->type()==CkLocRec::local) {
+                  homeObjHash.remove(*(CkArrayIndex *)&idx);
+                }
                delete old;
        }
 }
                delete old;
        }
 }
@@ -3189,6 +3402,14 @@ void CkLocMgr::insertRecN(CkLocRec *rec,const CkArrayIndex &idx) {
        DEBC((AA"  adding new rec(%s) for %s\n"AB,rec2str[rec->type()],idx2str(idx)));
         CmiImmediateLock(hashImmLock);
        hash.put(*(CkArrayIndex *)&idx)=rec;
        DEBC((AA"  adding new rec(%s) for %s\n"AB,rec2str[rec->type()],idx2str(idx)));
         CmiImmediateLock(hashImmLock);
        hash.put(*(CkArrayIndex *)&idx)=rec;
+        if(rec->type()==CkLocRec::local){
+          //if(CmiMyPartition()==1)
+            //CkPrintf("[%d][%d] put home object %s type %s %p \n",CmiMyPartition(),CkMyPe(),idx2str(idx),rec2str[rec->type()],rec);
+          localHash.put(*(CkArrayIndex *)&idx)=rec;
+        }
+        if(rec->type()!=CkLocRec::local&&homePe(idx)==CkMyPe()){
+          homeObjHash.put(*(CkArrayIndex *)&idx)=rec;
+        }
         CmiImmediateUnlock(hashImmLock);
 }
 
         CmiImmediateUnlock(hashImmLock);
 }