fix for migration
[charm.git] / src / libs / ck-libs / tcharm / tcharm.C
index 5e1dfe2a385ca19709ecc8bf69d1ad3b1a5216ac..aca121847d7ffba30c68799312af9a1afe2551f3 100644 (file)
@@ -5,11 +5,12 @@ Orion Sky Lawlor, olawlor@acm.org, 11/19/2001
  */
 #include "tcharm_impl.h"
 #include "tcharm.h"
+#include "ckevacuation.h"
 #include <ctype.h>
 
-#if 0
+#if 0 
     /*Many debugging statements:*/
-#    define DBG(x) ckout<<"["<<thisIndex<<"] TCHARM> "<<x<<endl;
+#    define DBG(x) ckout<<"["<<thisIndex<<","<<CkMyPe()<<"] TCHARM> "<<x<<endl;
 #    define DBGX(x) ckout<<"PE("<<CkMyPe()<<") TCHARM> "<<x<<endl;
 #else
     /*No debugging statements*/
@@ -55,6 +56,9 @@ static TCharmTraceLibList tcharm_tracelibs;
 static int tcharm_nomig=0, tcharm_nothreads=0;
 static int tcharm_stacksize=1*1024*1024; /*Default stack size is 1MB*/
 static int tcharm_initted=0;
+CkpvDeclare(int, mapCreated);
+static CkGroupID mapID;
+static char* mapping = NULL;
 
 void TCharm::nodeInit(void)
 {
@@ -65,6 +69,10 @@ void TCharm::procInit(void)
   CtvInitialize(TCharm *,_curTCharm);
   CtvAccess(_curTCharm)=NULL;
   tcharm_initted=1;
+  CtgInit();
+
+  CkpvInitialize(int, mapCreated);
+  CkpvAccess(mapCreated) = 0;
 
   // called on every pe to eat these arguments
   char **argv=CkGetArgv();
@@ -74,7 +82,23 @@ void TCharm::procInit(void)
   char *traceLibName=NULL;
   while (CmiGetArgStringDesc(argv,"+tcharm_trace",&traceLibName,"Print each call to this library"))
       tcharm_tracelibs.addTracing(traceLibName);
-  CmiGetArgIntDesc(argv,"+tcharm_stacksize",&tcharm_stacksize,"Set the thread stack size (default 1MB)");
+  // CmiGetArgIntDesc(argv,"+tcharm_stacksize",&tcharm_stacksize,"Set the thread stack size (default 1MB)");
+  char *str;
+  if (CmiGetArgStringDesc(argv,"+tcharm_stacksize",&str,"Set the thread stack size (default 1MB)"))  {
+    if (strpbrk(str,"M")) {
+      sscanf(str, "%dM", &tcharm_stacksize);
+      tcharm_stacksize *= 1024*1024;
+    }
+    else if (strpbrk(str,"K")) {
+      sscanf(str, "%dK", &tcharm_stacksize);
+      tcharm_stacksize *= 1024;
+    }
+    else {
+      sscanf(str, "%d", &tcharm_stacksize);
+    }
+    if (CkMyPe() == 0)
+      CkPrintf("TCharm> stack size is set to %d.\n", tcharm_stacksize);
+  }
   if (CkMyPe()!=0) { //Processor 0 eats "+vp<N>" and "-vp<N>" later:
        int ignored;
        while (CmiGetArgIntDesc(argv,"-vp",&ignored,NULL)) {}
@@ -84,6 +108,11 @@ void TCharm::procInit(void)
     if (tcharm_nomig) CmiPrintf("TCHARM> Disabling migration support, for debugging\n");
     if (tcharm_nothreads) CmiPrintf("TCHARM> Disabling thread support, for debugging\n");
   }
+  if (CkpvAccess(mapCreated)==0) {
+    if (0!=CmiGetArgString(argv, "+mapping", &mapping)){
+    }
+    CkpvAccess(mapCreated)=1;
+  }
 }
 
 void TCHARM_Api_trace(const char *routineName,const char *libraryName)
@@ -98,13 +127,32 @@ void TCHARM_Api_trace(const char *routineName,const char *libraryName)
        CmiPrintf("\n");
 }
 
+// register thread start functions to get a function handler
+// this is portable across heterogeneous platforms, or on machines with
+// random stack/function pointer
+
+static CkVec<TCHARM_Thread_data_start_fn> threadFnTable;
+
+int TCHARM_Register_thread_function(TCHARM_Thread_data_start_fn fn)
+{
+  int idx = threadFnTable.size();
+  threadFnTable.push_back(fn);
+  return idx+1;                     // make 0 invalid number
+}
+
+TCHARM_Thread_data_start_fn getTCharmThreadFunction(int idx)
+{
+  CmiAssert(idx > 0);
+  return threadFnTable[idx-1];
+}
+
 static void startTCharmThread(TCharmInitMsg *msg)
 {
        DBGX("thread started");
-       CtvAccess(_curTCharm)->activateHeap();
-       typedef void (*threadFn_t)(void *);
-       ((threadFn_t)msg->threadFn)(msg->data);
-       CmiIsomallocBlockListActivate(NULL); //Turn off migratable memory
+       TCharm::activateThread();
+       TCHARM_Thread_data_start_fn threadFn = getTCharmThreadFunction(msg->threadFn);
+       threadFn(msg->data);
+       TCharm::deactivateThread();
        CtvAccess(_curTCharm)->done();
 }
 
@@ -124,21 +172,32 @@ TCharm::TCharm(TCharmInitMsg *initMsg_)
     } else {
       tid=CthCreateMigratable((CthVoidFn)startTCharmThread,initMsg,initMsg->opts.stackSize);
     }
-#if CMK_BLUEGENE_CHARM
+#if CMK_BIGSIM_CHARM
     BgAttach(tid);
+    BgUnsetStartOutOfCore();
 #endif
   }
+  threadGlobals=CtgCreate(tid);
   CtvAccessOther(tid,_curTCharm)=this;
   isStopped=true;
   resumeAfterMigration=false;
+       /* FAULT_EVAC*/
+       AsyncEvacuate(CmiTrue);
+  skipResume=false;
   exitWhenDone=initMsg->opts.exitWhenDone;
+  isSelfDone = false;
   threadInfo.tProxy=CProxy_TCharm(thisArrayID);
   threadInfo.thisElement=thisIndex;
   threadInfo.numElements=initMsg->numElements;
-  heapBlocks=CmiIsomallocBlockListNew();
+  if (1 || CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC)) {
+       heapBlocks=CmiIsomallocBlockListNew(tid);
+  } else
+       heapBlocks=0;
   nUd=0;
   usesAtSync=CmiTrue;
   run();
+  CkCallback cb(CkIndex_TCharm::ResumeFromChkpSync(),thisProxy(thisIndex));
+  setChkpResumeClient(cb);
 }
 
 TCharm::TCharm(CkMigrateMessage *msg)
@@ -146,94 +205,189 @@ TCharm::TCharm(CkMigrateMessage *msg)
 {
   initMsg=NULL;
   tid=NULL;
+  threadGlobals=NULL;
   threadInfo.tProxy=CProxy_TCharm(thisArrayID);
+       AsyncEvacuate(CmiTrue);
+  heapBlocks=0;
+}
+
+void checkPupMismatch(PUP::er &p,int expected,const char *where)
+{
+       int v=expected;
+       p|v;
+       if (v!=expected) {
+               CkError("FATAL ERROR> Mismatch %s pup routine\n",where);
+               CkAbort("FATAL ERROR: Pup direction mismatch");
+       }
 }
 
 void TCharm::pup(PUP::er &p) {
 //Pup superclass
   ArrayElement1D::pup(p);
-
-  p(isStopped); p(resumeAfterMigration); p(exitWhenDone);
+  //BIGSIM_OOC DEBUGGING
+  //if(!p.isUnpacking()){
+  //  CmiPrintf("TCharm[%d] packing: ", thisIndex);
+  //  CthPrintThdStack(tid);
+  //}
+  //if(p.isUnpacking()&&CkInRestarting()){
+  if(p.isUnpacking()){
+    CkCallback cb(CkIndex_TCharm::ResumeFromChkpSync(),thisProxy(thisIndex));
+    setChkpResumeClient(cb);
+  }
+  checkPupMismatch(p,5134,"before TCHARM");
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
+    if(!isStopped){
+//      resumeAfterMigration = true;
+    }
+    isStopped = true;
+#endif
+  p(isStopped); p(resumeAfterMigration); p(exitWhenDone); p(isSelfDone); p(skipResume);
   p(threadInfo.thisElement);
   p(threadInfo.numElements);
   
-  if (sema.size()>0) 
+  if (sema.size()>0){
        CkAbort("TCharm::pup> Cannot migrate with unconsumed semaphores!\n");
+  }
 
 #ifndef CMK_OPTIMIZE
   DBG("Packing thread");
-  if (!isStopped)
-    CkAbort("Cannot pup a running thread.  You must suspend before migrating.\n");
+  if (!isStopped && !CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC)){
+    if(_BgOutOfCoreFlag==0) //not doing out-of-core scheduling
+       CkAbort("Cannot pup a running thread.  You must suspend before migrating.\n");
+  }    
   if (tcharm_nomig) CkAbort("Cannot migrate with the +tcharm_nomig option!\n");
 #endif
 
-//Pup thread (EVIL & UGLY):
   //This seekBlock allows us to reorder the packing/unpacking--
   // This is needed because the userData depends on the thread's stack
   // and heap data both at pack and unpack time.
   PUP::seekBlock s(p,2);
+  
   if (p.isUnpacking())
   {//In this case, unpack the thread & heap before the user data
     s.seek(1);
-    tid = CthPup((pup_er) &p, tid);
-    CtvAccessOther(tid,_curTCharm)=this;
-    CmiIsomallocBlockListPup((pup_er) &p,&heapBlocks);
+    pupThread(p);
     //Restart our clock: set it up so packTime==CkWallTimer+timeOffset
     double packTime;
     p(packTime);
     timeOffset=packTime-CkWallTimer();
   }
+  
+//Pack all user data
+  // Set up TCHARM context for use during user's pup routines:
+  CtvAccess(_curTCharm)=this;
+  activateThread();
 
-  //Pack all user data
   s.seek(0);
+  checkPupMismatch(p,5135,"before TCHARM user data");
   p(nUd);
-  for(int i=0;i<nUd;i++)
+  for(int i=0;i<nUd;i++) {
+    if (p.isUnpacking()) ud[i].update(tid);
     ud[i].pup(p);
-  p|sud;
+  }
+  checkPupMismatch(p,5137,"after TCHARM_Register user data");
 
+  if (CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
+    deactivateThread();
+  p|sud;           //  sud vector block can not be in isomalloc
+  checkPupMismatch(p,5138,"after TCHARM_Global user data");
+  
+  // Tear down TCHARM context after calling user pup routines
+  if (!CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
+    deactivateThread();
+  CtvAccess(_curTCharm)=NULL;
+  
   if (!p.isUnpacking())
   {//In this case, pack the thread & heap after the user data
     s.seek(1);
-    tid = CthPup((pup_er) &p, tid);
-    CmiIsomallocBlockListPup((pup_er) &p,&heapBlocks);
+    pupThread(p);
     //Stop our clock:
     double packTime=CkWallTimer()+timeOffset;
     p(packTime);
   }
+  
   s.endBlock(); //End of seeking block
+  checkPupMismatch(p,5140,"after TCHARM");
+  
+  //BIGSIM_OOC DEBUGGING
+  //if(p.isUnpacking()){
+  //  CmiPrintf("TCharm[%d] unpacking: ", thisIndex);
+  //  CthPrintThdStack(tid);
+  //}
+
+}
+
+// Pup our thread and related data
+void TCharm::pupThread(PUP::er &pc) {
+    pup_er p=(pup_er)&pc;
+    checkPupMismatch(pc,5138,"before TCHARM thread");
+    if (1 || CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
+      CmiIsomallocBlockListPup(p,&heapBlocks,tid);
+    tid = CthPup(p, tid);
+    if (pc.isUnpacking()) {
+      CtvAccessOther(tid,_curTCharm)=this;
+#if CMK_BIGSIM_CHARM
+      BgAttach(tid);
+#endif
+    }
+    threadGlobals=CtgPup(p,threadGlobals);
+    checkPupMismatch(pc,5139,"after TCHARM thread");
 }
 
 //Pup one group of user data
 void TCharm::UserData::pup(PUP::er &p)
 {
   pup_er pext=(pup_er)(&p);
-  p(isC);
-  //Save address of userdata-- assumes user data is on the stack
-  p((void*)&data,sizeof(data));
-  if (isC) { //C version
-    //FIXME: function pointers may not be valid across processors
-    p((void*)&cfn, sizeof(TCpupUserDataC));
-    if (cfn) cfn(pext,data);
-  }
-  else { //Fortran version
-    //FIXME: function pointers may not be valid across processors
-    p((void*)&ffn, sizeof(TCpupUserDataF));
-    if (ffn) ffn(pext,data);
-  }
+  p(mode);
+  switch(mode) {
+  case 'c': { /* C mode: userdata is on the stack, so keep address */
+//     p((char*)&data,sizeof(data));
+     p(pos);
+     //FIXME: function pointers may not be valid across processors
+     p((char*)&cfn, sizeof(TCHARM_Pup_fn));
+     char *data = CthPointer(t, pos);
+     if (cfn) cfn(pext,data);
+     } break;
+  case 'g': { /* Global mode: zero out userdata on arrival */
+     if (CmiMemoryIs(CMI_MEMORY_IS_ISOMALLOC))
+     {
+        // keep the pointer value if using isomalloc, no need to use pup
+       p(pos);
+     }
+     else if (p.isUnpacking())      //  zero out userdata on arrival
+       pos=0;
+
+       //FIXME: function pointers may not be valid across processors
+     p((char*)&gfn, sizeof(TCHARM_Pup_global_fn));
+     if (gfn) gfn(pext);
+     } break;
+  default:
+     break;
+  };
 }
 
 TCharm::~TCharm()
 {
-  CmiIsomallocBlockListDelete(heapBlocks);
+  //BIGSIM_OOC DEBUGGING
+  //CmiPrintf("TCharm destructor called with heapBlocks=%p!\n", heapBlocks);
+  
+#if !CMK_USE_MEMPOOL_ISOMALLOC
+  if (heapBlocks) CmiIsomallocBlockListDelete(heapBlocks);
+#endif
   CthFree(tid);
+  CtgFree(threadGlobals);
   delete initMsg;
 }
 
 void TCharm::migrateTo(int destPE) {
        if (destPE==CkMyPe()) return;
+       if (CthMigratable() == 0) {
+           CkPrintf("Warning> thread migration is not supported!\n");
+            return;
+        }
        // Make sure migrateMe gets called *after* we suspend:
        thisProxy[thisIndex].migrateDelayed(destPE);
-       resumeAfterMigration=true;
+//     resumeAfterMigration=true;
        suspend();
 }
 void TCharm::migrateDelayed(int destPE) {
@@ -241,16 +395,40 @@ void TCharm::migrateDelayed(int destPE) {
 }
 void TCharm::ckJustMigrated(void) {
        ArrayElement::ckJustMigrated();
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
+//  resumeAfterMigration = true;
+#endif
        if (resumeAfterMigration) {
-               resumeAfterMigration=false;
+               resumeAfterMigration=false;
                resume(); //Start the thread running
        }
 }
 
+void TCharm::ckJustRestored(void) {
+       //CkPrintf("call just restored from TCharm[%d]\n", thisIndex);
+       ArrayElement::ckJustRestored();
+       //if (resumeAfterMigration) {
+       //      resumeAfterMigration=false;
+       //      resume(); //Start the thread running
+       //}
+}
+
+/*
+       FAULT_EVAC
+
+       If a Tcharm object is about to migrate it should be suspended first
+*/
+void TCharm::ckAboutToMigrate(void){
+       ArrayElement::ckAboutToMigrate();
+       resumeAfterMigration = true;
+       isStopped = true;
+//     suspend();
+}
+
 // clear the data before restarting from disk
 void TCharm::clear()
 {
-  CmiIsomallocBlockListDelete(heapBlocks);
+  if (heapBlocks) CmiIsomallocBlockListDelete(heapBlocks);
   CthFree(tid);
   delete initMsg;
 }
@@ -293,24 +471,39 @@ void TCharm::stop(void)
   stopTiming();
   isStopped=true;
   DBG("thread suspended");
+
   CthSuspend();
-  DBG("thread resumed");
+//     DBG("thread resumed");
   /*SUBTLE: We have to do the get() because "this" may have changed
     during a migration-suspend.  If you access *any* members
     from this point onward, you'll cause heap corruption if
     we're resuming from migration!  (OSL 2003/9/23)
    */
   TCharm *dis=TCharm::get();
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) 
+/*  CpvAccess(_currentObj) = dis;
+ *      printf("[%d] _currentObject set to TCharm index %d %p\n",CkMyPe(),dis->thisIndex,dis);*/
+#endif
   dis->isStopped=false;
   dis->startTiming();
+  //CkPrintf("[%d] Thread resumed  for tid %p\n",dis->thisIndex,dis->tid);
 }
 
 //Resume the waiting thread
 void TCharm::start(void)
 {
+  //  since this thread is scheduled, it is not a good idea to migrate 
   isStopped=false;
   DBG("thread resuming soon");
+  //CkPrintf("TCharm[%d]::start()\n", thisIndex);
+  //CmiPrintStackTrace(0);
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
+//CthAwakenPrio(tid, CQS_QUEUEING_BFIFO, 1, &prio);
   CthAwaken(tid);
+#else
+  CthAwaken(tid);
+#endif
+  DBG("thread resuming soon");
 }
 
 //Block our thread, schedule, and come back:
@@ -332,12 +525,78 @@ void TCharm::migrate(void)
 #endif
 }
 
+
+void TCharm::chkpsync(void)
+{
+#if CMK_LBDB_ON
+  DBG("going to sync");
+  AtChkpSync();
+  stop();
+#else
+  DBG("skipping sync, because there is no load balancer");
+#endif
+}
+
+void TCharm::evacuate(){
+       /*
+               FAULT_EVAC
+       */
+       //CkClearAllArrayElementsCPP();
+       if(CkpvAccess(startedEvac)){
+//             resumeAfterMigration=true;
+               CcdCallFnAfter((CcdVoidFn)CkEmmigrateElement, (void *)myRec, 1);
+               suspend();
+               return;
+       }
+       return;
+
+}
+
+//calls atsync with async mode
+void TCharm::async_migrate(void)
+{
+#if CMK_LBDB_ON
+  DBG("going to sync at async mode");
+  skipResume = true;           // we resume immediately
+  ReadyMigrate(false);
+  AtSync(0);
+  schedule();
+//  allow_migrate();
+#else
+  DBG("skipping sync, because there is no load balancer");
+#endif
+}
+
+/*
+Note:
+ thread can only migrate at the point when this is called
+*/
+void TCharm::allow_migrate(void)
+{
+#if CMK_LBDB_ON
+//  ReadyMigrate(true);
+  int nextPe = MigrateToPe();
+  if (nextPe != -1) {
+    migrateTo(nextPe);
+  }
+#else
+  DBG("skipping sync, because there is no load balancer");
+#endif
+}
+
 //Resume from sync: start the thread again
 void TCharm::ResumeFromSync(void)
 {
-  start();
+  //if(isSelfDone) return;
+  //if (exitWhenDone) return; //for bigsim ooc execution
+  if (!skipResume) start();
+  CkPrintf("thread ResumeFromSync\n");
 }
 
+void TCharm::ResumeFromChkpSync(void)
+{
+  start();
+}
 
 /****** TcharmClient ******/
 void TCharmClient1D::ckJustMigrated(void) {
@@ -359,14 +618,22 @@ CkArrayID TCHARM_Get_threads(void) {
 /************* Startup/Shutdown Coordination Support ************/
 
 // Useless values to reduce over:
-int vals[2]={0,1};
+int _vals[2]={0,1};
 
 //Called when we want to go to a barrier
 void TCharm::barrier(void) {
        //Contribute to a synchronizing reduction
        CkCallback cb(index_t::atBarrier(0), thisProxy[0]);
-       contribute(sizeof(vals),&vals,CkReduction::sum_int,cb);
+       contribute(sizeof(_vals),&_vals,CkReduction::sum_int,cb);
+#if CMK_BIGSIM_CHARM
+        void *curLog;          // store current log in timeline
+        _TRACE_BG_TLINE_END(&curLog);
+       TRACE_BG_AMPI_BREAK(NULL, "TCharm_Barrier_START", NULL, 0, 1);
+#endif
        stop();
+#if CMK_BIGSIM_CHARM
+        _TRACE_BG_SET_INFO(NULL, "TCHARM_Barrier_END",  &curLog, 1);
+#endif
 }
 
 //Called when we've reached the barrier
@@ -378,22 +645,25 @@ void TCharm::atBarrier(CkReductionMsg *m) {
 
 //Called when the thread is done running
 void TCharm::done(void) {
+       //CmiPrintStackTrace(0);
        DBG("TCharm thread "<<thisIndex<<" done")
        if (exitWhenDone) {
                //Contribute to a synchronizing reduction
                CkCallback cb(index_t::atExit(0), thisProxy[0]);
-               contribute(sizeof(vals),&vals,CkReduction::sum_int,cb);
+               contribute(sizeof(_vals),&_vals,CkReduction::sum_int,cb);
        }
+       isSelfDone = true;
        stop();
 }
 //Called when all threads are done running
 void TCharm::atExit(CkReductionMsg *m) {
-       DBGX("TCharm::atExit> exiting");
+       DBGX("TCharm::atExit1> exiting");
        delete m;
+       //thisProxy.unsetFlags();
        CkExit();
+       //CkPrintf("After CkExit()!!!!!!!\n");
 }
 
-
 /************* Setup **************/
 
 //Globals used to control setup process
@@ -458,26 +728,26 @@ CDECL void TCHARM_Set_exit(void) { g_tcharmOptions.exitWhenDone=1; }
 
 /*Create a new array of threads, which will be bound to by subsequent libraries*/
 CDECL void TCHARM_Create(int nThreads,
-                       TCHARM_Thread_start_fn threadFn)
+                       int threadFn)
 {
        TCHARMAPI("TCHARM_Create");
        TCHARM_Create_data(nThreads,
-                        (TCHARM_Thread_data_start_fn)threadFn,NULL,0);
+                        threadFn,NULL,0);
 }
 FDECL void FTN_NAME(TCHARM_CREATE,tcharm_create)
-       (int *nThreads,TCHARM_Thread_start_fn threadFn)
+       (int *nThreads,int threadFn)
 { TCHARM_Create(*nThreads,threadFn); }
 
 static CProxy_TCharm TCHARM_Build_threads(TCharmInitMsg *msg);
 
 /*As above, but pass along (arbitrary) data to threads*/
 CDECL void TCHARM_Create_data(int nThreads,
-                 TCHARM_Thread_data_start_fn threadFn,
+                 int threadFn,
                  void *threadData,int threadDataLen)
 {
        TCHARMAPI("TCHARM_Create_data");
        TCharmInitMsg *msg=new (threadDataLen,0) TCharmInitMsg(
-               (CthVoidFn)threadFn,g_tcharmOptions);
+               threadFn,g_tcharmOptions);
        msg->numElements=nThreads;
        memcpy(msg->data,threadData,threadDataLen);
        TCHARM_Build_threads(msg);
@@ -488,24 +758,44 @@ CDECL void TCHARM_Create_data(int nThreads,
 
 FDECL void FTN_NAME(TCHARM_CREATE_DATA,tcharm_create_data)
        (int *nThreads,
-                 TCHARM_Thread_data_start_fn threadFn,
+                 int threadFn,
                  void *threadData,int *threadDataLen)
 { TCHARM_Create_data(*nThreads,threadFn,threadData,*threadDataLen); }
 
-static int propMapCreated=0;
-static CkGroupID propMapID;
 CkGroupID CkCreatePropMap(void);
 
 static CProxy_TCharm TCHARM_Build_threads(TCharmInitMsg *msg)
 {
-       CkArrayOptions opts(msg->numElements);
-       if (!propMapCreated) {
-               propMapCreated=1;
-               propMapID=CkCreatePropMap();
-       }
-       opts.setMap(propMapID);
-       int nElem=msg->numElements; //<- save it because msg will be deleted.
-       return CProxy_TCharm::ckNew(msg,opts);
+  CkArrayOptions opts(msg->numElements);
+  CkAssert(CkpvAccess(mapCreated)==1);
+
+  if(haveConfigurableRRMap()){
+    CkPrintf("USING ConfigurableRRMap\n");
+    mapID=CProxy_ConfigurableRRMap::ckNew();
+  } else if(mapping==NULL){
+#if CMK_BIGSIM_CHARM
+    mapID=CProxy_BlockMap::ckNew();
+#else
+#if __FAULT__
+       mapID=CProxy_RRMap::ckNew();
+#else
+    mapID=CkCreatePropMap();
+#endif
+#endif
+  } else if(0 == strcmp(mapping,"BLOCK_MAP")) {
+    CkPrintf("USING BLOCK_MAP\n");
+    mapID = CProxy_BlockMap::ckNew();
+  } else if(0 == strcmp(mapping,"RR_MAP")) {
+    CkPrintf("USING RR_MAP\n");
+    mapID = CProxy_RRMap::ckNew();
+  } else if(0 == strcmp(mapping,"MAPFILE")) {
+    CkPrintf("Reading map from file\n");
+    mapID = CProxy_ReadFileMap::ckNew();
+  } else {  // "PROP_MAP" or anything else
+    mapID = CkCreatePropMap();
+  }
+  opts.setMap(mapID);
+  return CProxy_TCharm::ckNew(msg,opts);
 }
 
 // Helper used when creating a new array bound to the TCHARM threads:
@@ -515,6 +805,9 @@ CkArrayOptions TCHARM_Attach_start(CkArrayID *retTCharmArray,int *retNumElts)
        if (!tc)
                CkAbort("You must call TCHARM initialization routines from a TCHARM thread!");
        int nElts=tc->getNumElements();
+      
+        //CmiPrintf("TCHARM Elements = %d\n", nElts);  
+      
        if (retNumElts!=NULL) *retNumElts=nElts;
        *retTCharmArray=tc->getProxy();
        CkArrayOptions opts(nElts);
@@ -550,8 +843,15 @@ FDECL int FTN_NAME(TCHARM_NUM_ELEMENTS,tcharm_num_elements)(void)
 static void checkAddress(void *data)
 {
        if (tcharm_nomig||tcharm_nothreads) return; //Stack is not isomalloc'd
-       if (!CmiIsomallocInRange(data))
+       if (CmiThreadIs(CMI_THREAD_IS_ALIAS)||CmiThreadIs(CMI_THREAD_IS_STACKCOPY)) return; // memory alias thread
+       if (CmiIsomallocEnabled()) {
+          if (!CmiIsomallocInRange(data))
            CkAbort("The UserData you register must be allocated on the stack!\n");
+        }
+        else {
+         if(CkMyPe() == 0)
+           CkPrintf("Warning> checkAddress failed because isomalloc not supported.\n");
+       }
 }
 
 /* Old "register"-based userdata: */
@@ -559,15 +859,14 @@ CDECL int TCHARM_Register(void *data,TCHARM_Pup_fn pfn)
 { 
        TCHARMAPI("TCHARM_Register");
        checkAddress(data);
-       return TCharm::get()->add(TCharm::UserData(pfn,data));
+       return TCharm::get()->add(TCharm::UserData(pfn,TCharm::get()->getThread(),data));
 }
 FDECL int FTN_NAME(TCHARM_REGISTER,tcharm_register)
-       (void *data,TCpupUserDataF pfn)
+       (void *data,TCHARM_Pup_fn pfn)
 { 
        TCHARMAPI("TCHARM_Register");
        checkAddress(data);
-       return TCharm::get()->add(TCharm::UserData(
-               pfn,data,TCharm::UserData::isFortran()));
+       return TCharm::get()->add(TCharm::UserData(pfn,TCharm::get()->getThread(),data));
 }
 
 CDECL void *TCHARM_Get_userdata(int id)
@@ -588,7 +887,7 @@ CDECL void TCHARM_Set_global(int globalID,void *new_value,TCHARM_Pup_global_fn p
                int newLen=2*globalID;
                tc->sud.resize(newLen);
        }
-       tc->sud[globalID]=TCharm::UserData((TCHARM_Pup_fn) pup_or_NULL,new_value);
+       tc->sud[globalID]=TCharm::UserData(pup_or_NULL,tc->getThread(),new_value);
 }
 CDECL void *TCHARM_Get_global(int globalID)
 {
@@ -602,15 +901,54 @@ CDECL void *TCHARM_Get_global(int globalID)
 CDECL void TCHARM_Migrate(void)
 {
        TCHARMAPI("TCHARM_Migrate");
+       if (CthMigratable() == 0) {
+         if(CkMyPe() == 0)
+           CkPrintf("Warning> thread migration is not supported!\n");
+          return;
+        }
        TCharm::get()->migrate();
 }
+
+CDECL void TCHARM_ChkpSync(void)
+{
+       TCHARMAPI("TCHARM_ChkpSync");
+       if (CthMigratable() == 0) {
+         if(CkMyPe() == 0)
+           CkPrintf("Warning> thread migration is not supported!\n");
+          return;
+        }
+       TCharm::get()->chkpsync();
+}
+
 FORTRAN_AS_C(TCHARM_MIGRATE,TCHARM_Migrate,tcharm_migrate,(void),())
+FORTRAN_AS_C(TCHARM_CHKPSYNC,TCHARM_ChkpSync,tcharm_chkpsync,(void),())
+
+CDECL void TCHARM_Async_Migrate(void)
+{
+       TCHARMAPI("TCHARM_Async_Migrate");
+       TCharm::get()->async_migrate();
+}
+FORTRAN_AS_C(TCHARM_ASYNC_MIGRATE,TCHARM_Async_Migrate,tcharm_async_migrate,(void),())
+
+CDECL void TCHARM_Allow_Migrate(void)
+{
+       TCHARMAPI("TCHARM_Allow_Migrate");
+       TCharm::get()->allow_migrate();
+}
+FORTRAN_AS_C(TCHARM_ALLOW_MIGRATE,TCHARM_Allow_Migrate,tcharm_allow_migrate,(void),())
 
 CDECL void TCHARM_Migrate_to(int destPE)
 {
        TCHARMAPI("TCHARM_Migrate_to");
        TCharm::get()->migrateTo(destPE);
 }
+
+CDECL void TCHARM_Evacuate()
+{
+       TCHARMAPI("TCHARM_Migrate_to");
+       TCharm::get()->evacuate();
+}
+
 FORTRAN_AS_C(TCHARM_MIGRATE_TO,TCHARM_Migrate_to,tcharm_migrate_to,
        (int *destPE),(*destPE))
 
@@ -683,8 +1021,8 @@ CDECL void TCHARM_Init(int *argc,char ***argv) {
 FDECL void FTN_NAME(TCHARM_INIT,tcharm_init)(void)
 {
        int argc=1;
-       char *argv_sto[2]={"foo",NULL};
-       char **argv=argv_sto;
+       const char *argv_sto[2]={"foo",NULL};
+       char **argv=(char **)argv_sto;
        TCHARM_Init(&argc,&argv);
 }
 
@@ -698,7 +1036,7 @@ FDECL void FTN_NAME(TCHARM_INIT,tcharm_init)(void)
 */
 /// Find this semaphore, or insert if there isn't one:
 TCharm::TCharmSemaphore *TCharm::findSema(int id) {
-       for (int s=0;s<sema.size();s++)
+       for (unsigned int s=0;s<sema.size();s++)
                if (sema[s].id==id) 
                        return &sema[s];
        sema.push_back(TCharmSemaphore(id));
@@ -707,7 +1045,7 @@ TCharm::TCharmSemaphore *TCharm::findSema(int id) {
 /// Remove this semaphore from the list
 void TCharm::freeSema(TCharmSemaphore *doomed) {
        int id=doomed->id;
-       for (int s=0;s<sema.size();s++)
+       for (unsigned int s=0;s<sema.size();s++)
                if (sema[s].id==id) {
                        sema[s]=sema[sema.length()-1];
                        sema.length()--;
@@ -768,4 +1106,39 @@ void *TCharm::semaPeek(int id) {
        return s->data;
 }
 
+/****** System Call support ******/
+/*
+TCHARM_System exists to work around a bug where Linux ia32
+glibc2.2.x with pthreads crashes at the fork() site when 
+called from a user-levelthread. 
+
+The fix is to call system from the main thread, by 
+passing the request out of the thread to our array element 
+before calling system().
+*/
+
+CDECL int 
+TCHARM_System(const char *shell_command)
+{
+       return TCharm::get()->system(shell_command);
+}
+int TCharm::system(const char *cmd)
+{
+       int ret=-1778;
+       callSystemStruct s;
+       s.cmd=cmd;
+       s.ret=&ret;
+       thisProxy[thisIndex].callSystem(s);
+       suspend();
+       return ret;
+}
+
+void TCharm::callSystem(const callSystemStruct &s)
+{
+       *s.ret = ::system(s.cmd);
+       resume();
+}
+
+
+
 #include "tcharm.def.h"