fix for migration
[charm.git] / src / libs / ck-libs / tcharm / tcharm.C
index c51c8c136e01974567e4a47f09db79f616489bd9..aca121847d7ffba30c68799312af9a1afe2551f3 100644 (file)
@@ -161,7 +161,6 @@ TCharm::TCharm(TCharmInitMsg *initMsg_)
   initMsg=initMsg_;
   initMsg->opts.sanityCheck();
   timeOffset=0.0;
-  threadGlobals=CtgCreate();
   if (tcharm_nothreads)
   { //Don't even make a new thread-- just use main thread
     tid=CthSelf();
@@ -178,6 +177,7 @@ TCharm::TCharm(TCharmInitMsg *initMsg_)
     BgUnsetStartOutOfCore();
 #endif
   }
+  threadGlobals=CtgCreate(tid);
   CtvAccessOther(tid,_curTCharm)=this;
   isStopped=true;
   resumeAfterMigration=false;
@@ -196,6 +196,8 @@ TCharm::TCharm(TCharmInitMsg *initMsg_)
   nUd=0;
   usesAtSync=CmiTrue;
   run();
+  CkCallback cb(CkIndex_TCharm::ResumeFromChkpSync(),thisProxy(thisIndex));
+  setChkpResumeClient(cb);
 }
 
 TCharm::TCharm(CkMigrateMessage *msg)
@@ -222,13 +224,16 @@ void checkPupMismatch(PUP::er &p,int expected,const char *where)
 void TCharm::pup(PUP::er &p) {
 //Pup superclass
   ArrayElement1D::pup(p);
-
   //BIGSIM_OOC DEBUGGING
   //if(!p.isUnpacking()){
   //  CmiPrintf("TCharm[%d] packing: ", thisIndex);
   //  CthPrintThdStack(tid);
   //}
-
+  //if(p.isUnpacking()&&CkInRestarting()){
+  if(p.isUnpacking()){
+    CkCallback cb(CkIndex_TCharm::ResumeFromChkpSync(),thisProxy(thisIndex));
+    setChkpResumeClient(cb);
+  }
   checkPupMismatch(p,5134,"before TCHARM");
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
     if(!isStopped){
@@ -315,8 +320,9 @@ void TCharm::pup(PUP::er &p) {
 // Pup our thread and related data
 void TCharm::pupThread(PUP::er &pc) {
     pup_er p=(pup_er)&pc;
-    CthThread savedtid = tid;
     checkPupMismatch(pc,5138,"before TCHARM thread");
+    if (1 || CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
+      CmiIsomallocBlockListPup(p,&heapBlocks,tid);
     tid = CthPup(p, tid);
     if (pc.isUnpacking()) {
       CtvAccessOther(tid,_curTCharm)=this;
@@ -324,8 +330,6 @@ void TCharm::pupThread(PUP::er &pc) {
       BgAttach(tid);
 #endif
     }
-    if (1 || CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
-      CmiIsomallocBlockListPup(p,&heapBlocks,savedtid);
     threadGlobals=CtgPup(p,threadGlobals);
     checkPupMismatch(pc,5139,"after TCHARM thread");
 }
@@ -367,7 +371,7 @@ TCharm::~TCharm()
   //BIGSIM_OOC DEBUGGING
   //CmiPrintf("TCharm destructor called with heapBlocks=%p!\n", heapBlocks);
   
-#if !USE_MEMPOOL_ISOMALLOC
+#if !CMK_USE_MEMPOOL_ISOMALLOC
   if (heapBlocks) CmiIsomallocBlockListDelete(heapBlocks);
 #endif
   CthFree(tid);
@@ -499,6 +503,7 @@ void TCharm::start(void)
 #else
   CthAwaken(tid);
 #endif
+  DBG("thread resuming soon");
 }
 
 //Block our thread, schedule, and come back:
@@ -521,6 +526,16 @@ void TCharm::migrate(void)
 }
 
 
+void TCharm::chkpsync(void)
+{
+#if CMK_LBDB_ON
+  DBG("going to sync");
+  AtChkpSync();
+  stop();
+#else
+  DBG("skipping sync, because there is no load balancer");
+#endif
+}
 
 void TCharm::evacuate(){
        /*
@@ -528,7 +543,6 @@ void TCharm::evacuate(){
        */
        //CkClearAllArrayElementsCPP();
        if(CkpvAccess(startedEvac)){
-               int nextPE = getNextPE(CkArrayIndex1D(thisIndex));
 //             resumeAfterMigration=true;
                CcdCallFnAfter((CcdVoidFn)CkEmmigrateElement, (void *)myRec, 1);
                suspend();
@@ -576,8 +590,13 @@ void TCharm::ResumeFromSync(void)
   //if(isSelfDone) return;
   //if (exitWhenDone) return; //for bigsim ooc execution
   if (!skipResume) start();
+  CkPrintf("thread ResumeFromSync\n");
 }
 
+void TCharm::ResumeFromChkpSync(void)
+{
+  start();
+}
 
 /****** TcharmClient ******/
 void TCharmClient1D::ckJustMigrated(void) {
@@ -747,8 +766,6 @@ CkGroupID CkCreatePropMap(void);
 
 static CProxy_TCharm TCHARM_Build_threads(TCharmInitMsg *msg)
 {
-  char *tmp;
-  char **argv=CkGetArgv();
   CkArrayOptions opts(msg->numElements);
   CkAssert(CkpvAccess(mapCreated)==1);
 
@@ -778,7 +795,6 @@ static CProxy_TCharm TCHARM_Build_threads(TCharmInitMsg *msg)
     mapID = CkCreatePropMap();
   }
   opts.setMap(mapID);
-  int nElem=msg->numElements; //<- save it because msg will be deleted.
   return CProxy_TCharm::ckNew(msg,opts);
 }
 
@@ -892,7 +908,20 @@ CDECL void TCHARM_Migrate(void)
         }
        TCharm::get()->migrate();
 }
+
+CDECL void TCHARM_ChkpSync(void)
+{
+       TCHARMAPI("TCHARM_ChkpSync");
+       if (CthMigratable() == 0) {
+         if(CkMyPe() == 0)
+           CkPrintf("Warning> thread migration is not supported!\n");
+          return;
+        }
+       TCharm::get()->chkpsync();
+}
+
 FORTRAN_AS_C(TCHARM_MIGRATE,TCHARM_Migrate,tcharm_migrate,(void),())
+FORTRAN_AS_C(TCHARM_CHKPSYNC,TCHARM_ChkpSync,tcharm_chkpsync,(void),())
 
 CDECL void TCHARM_Async_Migrate(void)
 {
@@ -1007,7 +1036,7 @@ FDECL void FTN_NAME(TCHARM_INIT,tcharm_init)(void)
 */
 /// Find this semaphore, or insert if there isn't one:
 TCharm::TCharmSemaphore *TCharm::findSema(int id) {
-       for (int s=0;s<sema.size();s++)
+       for (unsigned int s=0;s<sema.size();s++)
                if (sema[s].id==id) 
                        return &sema[s];
        sema.push_back(TCharmSemaphore(id));
@@ -1016,7 +1045,7 @@ TCharm::TCharmSemaphore *TCharm::findSema(int id) {
 /// Remove this semaphore from the list
 void TCharm::freeSema(TCharmSemaphore *doomed) {
        int id=doomed->id;
-       for (int s=0;s<sema.size();s++)
+       for (unsigned int s=0;s<sema.size();s++)
                if (sema[s].id==id) {
                        sema[s]=sema[sema.length()-1];
                        sema.length()--;