TCharm: manages a CthThread's creation, load balacing,
authorOrion Lawlor <olawlor@acm.org>
Thu, 6 Dec 2001 23:21:09 +0000 (23:21 +0000)
committerOrion Lawlor <olawlor@acm.org>
Thu, 6 Dec 2001 23:21:09 +0000 (23:21 +0000)
migration, and shutdown.  Used as a "framework framework"
by FEM, AMPI, and MBlock.

src/libs/ck-libs/tcharm/Makefile [new file with mode: 0644]
src/libs/ck-libs/tcharm/compat_funs.c [new file with mode: 0644]
src/libs/ck-libs/tcharm/compat_fus.c [new file with mode: 0644]
src/libs/ck-libs/tcharm/compat_uns.c [new file with mode: 0644]
src/libs/ck-libs/tcharm/compat_us.c [new file with mode: 0644]
src/libs/ck-libs/tcharm/tcharm.C [new file with mode: 0644]
src/libs/ck-libs/tcharm/tcharm.ci [new file with mode: 0644]
src/libs/ck-libs/tcharm/tcharm.h [new file with mode: 0644]
src/libs/ck-libs/tcharm/tcharmc.h [new file with mode: 0644]
src/libs/ck-libs/tcharm/tcharmf.h [new file with mode: 0644]

diff --git a/src/libs/ck-libs/tcharm/Makefile b/src/libs/ck-libs/tcharm/Makefile
new file mode 100644 (file)
index 0000000..2d4b561
--- /dev/null
@@ -0,0 +1,33 @@
+CDIR=../../../..
+CHARMC=$(CDIR)/bin/charmc $(OPTS)
+
+HEADERS=tcharm.h tcharmc.h tcharmf.h tcharm.decl.h
+OBJS=tcharm.o
+COMPAT=compat_uns.o compat_us.o compat_funs.o compat_fus.o
+DEST=$(CDIR)/lib/libmoduletcharm.a
+
+all: $(DEST)
+
+$(DEST): $(OBJS) $(COMPAT) headers
+       $(CHARMC) $(OBJS) $(COMPAT) -o $@
+
+headers: $(HEADERS)
+       cp $(HEADERS) $(CDIR)/include/
+       touch headers
+
+$(COMPAT):
+       @for o in $(COMPAT); \
+       do \
+               file=`echo $$o | sed -e "s/\.o/.c/g"`; \
+               echo "$(CHARMC) -c $$file"; \
+               $(CHARMC) -c $$file || exit 1; \
+       done
+
+tcharm.o: tcharm.C $(HEADERS)
+       $(CHARMC) $(CFLAGS) -c $<
+
+tcharm.decl.h: tcharm.ci
+       $(CHARMC) tcharm.ci
+
+clean: 
+       -rm -fr *.o *~ *.decl.h *.def.h gmon.out $(DEST) conv-host charmrun
diff --git a/src/libs/ck-libs/tcharm/compat_funs.c b/src/libs/ck-libs/tcharm/compat_funs.c
new file mode 100644 (file)
index 0000000..9f6c69d
--- /dev/null
@@ -0,0 +1,3 @@
+#include "charm-api.h"
+#include "tcharmc.h"
+FDECL void FTN_NAME(TCHARM_USER_NODE_SETUP,tcharm_user_node_setup)(void) {}
diff --git a/src/libs/ck-libs/tcharm/compat_fus.c b/src/libs/ck-libs/tcharm/compat_fus.c
new file mode 100644 (file)
index 0000000..7f74d27
--- /dev/null
@@ -0,0 +1,5 @@
+#include "charm-api.h"
+#include "tcharmc.h"
+FDECL void FTN_NAME(TCHARM_USER_SETUP,tcharm_user_setup)(void) {
+       TCharmInDefaultSetup();
+}
diff --git a/src/libs/ck-libs/tcharm/compat_uns.c b/src/libs/ck-libs/tcharm/compat_uns.c
new file mode 100644 (file)
index 0000000..88a8dc4
--- /dev/null
@@ -0,0 +1,3 @@
+#include "charm-api.h"
+#include "tcharmc.h"
+CDECL void TCharmUserNodeSetup(void) {}
diff --git a/src/libs/ck-libs/tcharm/compat_us.c b/src/libs/ck-libs/tcharm/compat_us.c
new file mode 100644 (file)
index 0000000..6dde318
--- /dev/null
@@ -0,0 +1,5 @@
+#include "charm-api.h"
+#include "tcharmc.h"
+CDECL void TCharmUserSetup(void) {
+       TCharmInDefaultSetup();
+}
diff --git a/src/libs/ck-libs/tcharm/tcharm.C b/src/libs/ck-libs/tcharm/tcharm.C
new file mode 100644 (file)
index 0000000..f7bcbad
--- /dev/null
@@ -0,0 +1,505 @@
+/*
+Threaded Charm++ "Framework Framework"
+
+Orion Sky Lawlor, olawlor@acm.org, 11/19/2001
+ */
+#include "tcharm.h"
+
+#if 0
+    /*Many debugging statements:*/
+#    define DBG(x) ckout<<"["<<thisIndex<<"] TCHARM> "<<x<<endl;
+#    define DBGX(x) ckout<<"PE("<<CkMyPe()<<") TCHARM> "<<x<<endl;
+#else
+    /*No debugging statements*/
+#    define DBG(x) /*empty*/
+#    define DBGX(x) /*empty*/
+#endif
+
+CtvDeclare(TCharm *,_curTCharm);
+CpvDeclare(inState,_stateTCharm);
+
+void TCharm::nodeInit(void)
+{
+  CtvInitialize(TCharm *,_curTCharm);
+  CpvInitialize(inState,_stateTCharm);
+  TCharmUserNodeSetup();
+  FTN_NAME(TCHARM_USER_NODE_SETUP,tcharm_user_node_setup)();
+}
+
+static void startTCharmThread(TCharmInitMsg *msg)
+{
+       TCharm::setState(inDriver);
+       typedef void (*threadFn_t)(void *);
+       ((threadFn_t)msg->threadFn)(msg->data);
+       CtvAccess(_curTCharm)->done();
+}
+
+TCharm::TCharm(TCharmInitMsg *initMsg_)
+{
+  initMsg=initMsg_;
+  
+tid=CthCreateMigratable((CthVoidFn)startTCharmThread,initMsg,initMsg->stackSize);
+  CtvAccessOther(tid,_curTCharm)=this;
+  TCharm::setState(inInit);
+  isStopped=true;
+  threadInfo.tProxy=CProxy_TCharm(thisArrayID);
+  threadInfo.thisElement=thisIndex;
+  threadInfo.numElements=initMsg->numElements;
+  nUd=0;
+  usesAtSync=CmiTrue;
+  ready();
+}
+
+TCharm::TCharm(CkMigrateMessage *msg)
+       :ArrayElement1D(msg)
+{
+  initMsg=NULL;
+  tid=NULL;
+  threadInfo.tProxy=CProxy_TCharm(thisArrayID);  
+}
+
+void TCharm::pup(PUP::er &p) {
+//Pup superclass
+  ArrayElement1D::pup(p);  
+
+  p(isStopped);
+  p(threadInfo.thisElement);
+  p(threadInfo.numElements);
+
+#ifndef CMK_OPTIMIZE
+  DBG("Packing thread");
+  if (!isStopped)
+    CkAbort("Cannot pup a running thread.  You must suspend before migrating.\n");
+#endif
+
+//Pup thread (EVIL & UGLY):
+  //This seekBlock allows us to reorder the packing/unpacking--
+  // This is needed because the userData depends on the thread's stack
+  // both at pack and unpack time.
+  PUP::seekBlock s(p,2);
+  if (p.isUnpacking()) 
+  {//In this case, unpack the thread before the user data
+    s.seek(1);
+    tid = CthPup((pup_er) &p, tid);
+    CtvAccessOther(tid,_curTCharm)=this;
+  }
+  
+  //Pack all user data
+  TCharm::setState(inPup);
+  s.seek(0);
+  p(nUd);
+  for(int i=0;i<nUd;i++) 
+    ud[i].pup(p);
+  TCharm::setState(inFramework);
+
+  if (!p.isUnpacking()) 
+  {//In this case, pack the thread after the user data
+    s.seek(1);
+    tid = CthPup((pup_er) &p, tid);
+  }
+  s.endBlock(); //End of seeking block
+}
+
+//Pup one group of user data
+void TCharm::UserData::pup(PUP::er &p)
+{
+  pup_er pext=(pup_er)(&p);
+  p(isC);
+  //Save address of userdata-- assumes user data is on the stack
+  p((void*)&data,sizeof(data));
+  if (isC) { //C version
+    //FIXME: function pointers may not be valid across processors
+    p((void*)&cfn, sizeof(TCpupUserDataC));
+    cfn(pext,data);
+  } 
+  else { //Fortran version
+    //FIXME: function pointers may not be valid across processors
+    p((void*)&ffn, sizeof(TCpupUserDataF));        
+    ffn(pext,data);
+  }
+}
+
+TCharm::~TCharm() 
+{
+  CthFree(tid);
+  delete initMsg;
+}
+
+//Register user data to be packed with the thread
+int TCharm::add(const TCharm::UserData &d)
+{
+  if (nUd>=maxUserData)
+    CkAbort("TCharm: Registered too many user data fields!\n");
+  int nu=nUd++;
+  ud[nu]=d;
+  return nu;
+}
+void *TCharm::lookupUserData(int i) {
+       if (i<0 || i>=nUd)
+               CkAbort("Bad user data index passed to TCharmGetUserdata!\n");
+       return ud[i].getData();
+}
+
+//Start the thread running
+void TCharm::run(void)
+{
+  DBG("TCharm::run()");
+  start();
+}
+
+//Block the thread until start()ed again.
+void TCharm::stop(void)
+{
+  if (isStopped) return; //Nothing to do
+#ifndef CMK_OPTIMIZE
+  DBG("suspending thread");
+  if (tid != CthSelf())
+    CkAbort("Called TCharm::stop from outside TCharm thread!\n");
+#endif
+  isStopped=true;
+  stopTiming();
+  TCharm::setState(inFramework);
+  CthSuspend();
+  TCharm::setState(inDriver);
+  /*We have to do the get() because "this" may have changed
+    during a migration-suspend.*/
+  TCharm::get()->startTiming();
+}
+
+//Resume the waiting thread
+void TCharm::start(void)
+{
+  if (!isStopped) return; //Already started
+  isStopped=false;
+  TCharm::setState(inDriver);
+  DBG("awakening thread");
+  CthAwaken(tid);
+}
+
+//Go to sync, block, possibly migrate, and then resume
+void TCharm::migrate(void)
+{
+#if CMK_LBDB_ON
+  DBG("going to sync");  
+  AtSync();
+  stop();
+#else
+  DBG("skipping sync, because there is no load balancer");
+#endif
+}
+
+//Resume from sync: start the thread again
+void TCharm::ResumeFromSync(void)
+{
+  start();
+}
+
+#ifndef CMK_OPTIMIZE
+//Make sure we're actually in driver
+void TCharm::check(void)
+{
+       if (getState()!=inDriver)
+               CkAbort("TCharm> Can only use that routine from within driver!\n");
+}
+#endif
+
+static int propMapCreated=0;
+static CkGroupID propMapID;
+CkGroupID CkCreatePropMap(void);
+
+static void TCharmBuildThreads(TCharmInitMsg *msg,TCharmSetupCookie &cook)
+{
+       CkArrayOptions opts(msg->numElements);
+       if (!propMapCreated) {
+               propMapCreated=1;
+               propMapID=CkCreatePropMap();
+       }
+       opts.setMap(propMapID);
+       CkArrayID id=CProxy_TCharm::ckNew(msg,opts);
+       cook.setThreads(id,msg->numElements);
+}
+
+/************* Startup/Shutdown Coordination Support ************/
+
+enum {TC_READY=23, TC_DONE=42};
+
+//Called when a client is ready to run
+void TCharm::ready(void) {
+       DBG("TCharm thread "<<thisIndex<<" ready")
+       int vals[2]={0,1};
+       if (thisIndex==0) vals[0]=TC_READY;
+       //Contribute to a synchronizing reduction
+       contribute(sizeof(vals),&vals,CkReduction::sum_int);
+}
+
+//Called when the thread is done running
+void TCharm::done(void) {
+       DBG("TCharm thread "<<thisIndex<<" done")
+       int vals[2]={0,1};
+       if (thisIndex==0) vals[0]=TC_DONE;
+       //Contribute to a synchronizing reduction
+       contribute(sizeof(vals),&vals,CkReduction::sum_int);
+       stop();
+}
+
+//Called when an array reduction is complete
+static void coordinatorReduction(void *coord_,int dataLen,void *reductionData)
+{
+       TCharmCoordinator *coord=(TCharmCoordinator *)coord_;
+       int *vals=(int *)reductionData;
+       if (dataLen!=2*sizeof(int))
+               CkAbort("Unexpected length in TCharm array reduction!\n");
+       DBGX("Finished coordinator reduction: "<<vals[0]<<", "<<vals[1]);
+       switch (vals[0]) {
+       case TC_READY: coord->clientReady(); break;
+       case TC_DONE: coord->clientDone(); break;
+       default:
+               CkAbort("Unexpected value from TCharm array reduction!\n");
+       };
+}
+
+int TCharmCoordinator::nArrays=0; //Total number of running thread arrays
+TCharmCoordinator *TCharmCoordinator::head=NULL; //List of coordinators
+
+
+TCharmCoordinator::TCharmCoordinator(CkArrayID threads_,int nThreads_)
+       :threads(threads_), nThreads(nThreads_), nClients(0), nReady(0)
+{
+       nArrays++;
+       //Link into the coordinator list
+       next=head;
+       head=this;
+
+       threads.setReductionClient(coordinatorReduction,this);
+       nClients=1; //Thread array itself is a client
+}
+TCharmCoordinator::~TCharmCoordinator()
+{
+       //Coordinators never get deleted
+}
+void TCharmCoordinator::addClient(const CkArrayID &client)
+{
+       nClients++;
+}
+void TCharmCoordinator::clientReady(void)
+{
+       DBGX("client "<<nReady+1<<" of "<<nClients<<" ready");
+       nReady++;
+       if (nReady>=nClients) { //All client arrays are ready-- start threads
+               DBGX("starting threads");
+               threads.run();
+       }
+}
+void TCharmCoordinator::clientDone(void)
+{
+       DBGX("clientDone");     
+       nArrays--;
+       if (nArrays<=0) { //All arrays have exited
+               DBGX("done with computation");
+               CkExit();
+       }
+}
+
+/************* Setup **************/
+
+//Cookie used during setup
+TCharmSetupCookie *TCharmSetupCookie::theCookie;
+
+//Globals used to control setup process
+static int g_numDefaultSetups=0;
+static TCharmFallbackSetupFn g_fallbackSetup=NULL;
+void TCharmSetFallbackSetup(TCharmFallbackSetupFn f)
+{
+       g_fallbackSetup=f;
+}
+CDECL void TCharmInDefaultSetup(void) {
+       g_numDefaultSetups++;
+}
+
+//Tiny simple main chare
+class TCharmMain : public Chare {
+public:
+  TCharmMain(CkArgMsg *msg) {
+    TCharmSetupCookie cookie(msg->argv);
+    TCharmSetupCookie::theCookie=&cookie;
+    g_numDefaultSetups=0;
+    
+    /*Call user-overridable C setup*/
+    TCharmUserSetup();
+    /*Call user-overridable Fortran setup*/
+    FTN_NAME(TCHARM_USER_SETUP,tcharm_user_setup)();
+    
+    if (g_numDefaultSetups==2) 
+    { //User didn't override either setup routine
+           if (g_fallbackSetup)
+                   (g_fallbackSetup)();
+           else
+                   CmiAbort("You need to override TCharmUserSetup to start your computation, or else link in a framework module\n");
+    }      
+    
+    delete msg;
+    
+    if (0==TCharmCoordinator::getTotal())
+           CkAbort("You didn't create any TCharm arrays in TCharmUserSetup!\n");
+  }
+};
+
+#ifndef CMK_OPTIMIZE
+/*The setup cookie, used to store global initialization state*/
+TCharmSetupCookie &TCharmSetupCookie::check(void)
+{
+       if (magic!=correctMagic)
+               CkAbort("TCharm setup cookie is damaged!\n");
+       return *this;
+}
+#endif
+
+void TCharmSetupCookie::setThreads(const CkArrayID &aid,int nel)
+{
+       coord=new TCharmCoordinator(aid,nel);
+       tc=aid; numElements=nel;
+}
+
+TCharmSetupCookie::TCharmSetupCookie(char **argv_)
+{
+       magic=correctMagic;
+       stackSize=0;
+       argv=argv_;
+       coord=NULL;
+}
+
+
+/************** User API ***************/
+
+#define cookie (*TCharmSetupCookie::get())
+
+/**********************************
+Callable from UserSetup: 
+*/
+
+/*Set the size of the thread stack*/
+CDECL void TCharmSetStackSize(int newStackSize)
+{
+       if (TCharm::getState()!=inInit)
+               CkAbort("TCharm> Can only set stack size from in init!\n");
+       cookie.setStackSize(newStackSize);
+}
+FDECL void FTN_NAME(TCHARM_SET_STACK_SIZE,tcharm_set_stack_size)
+       (int *newSize)
+{ TCharmSetStackSize(*newSize); }
+
+
+/*Create a new array of threads, which will be bound to by subsequent libraries*/
+CDECL void TCharmCreate(int nThreads,
+                       TCharmThreadStartFn threadFn)
+{
+       TCharmCreateData(nThreads,
+                        (TCharmThreadDataStartFn)threadFn,NULL,0);
+}
+FDECL void FTN_NAME(TCHARM_CREATE,tcharm_create)
+       (int *nThreads,TCharmThreadStartFn threadFn)
+{ TCharmCreate(*nThreads,threadFn); }
+
+
+/*As above, but pass along (arbitrary) data to threads*/
+CDECL void TCharmCreateData(int nThreads,
+                 TCharmThreadDataStartFn threadFn,
+                 void *threadData,int threadDataLen)
+{
+       if (TCharm::getState()!=inInit)
+               CkAbort("TCharm> Can only create threads from in init!\n");
+       TCharmSetupCookie &cook=cookie;
+       TCharmInitMsg *msg=new (threadDataLen,0) TCharmInitMsg(
+               (CthVoidFn)threadFn,cook.getStackSize());
+       msg->numElements=nThreads;
+       memcpy(msg->data,threadData,threadDataLen);
+       TCharmBuildThreads(msg,cook);
+}
+
+FDECL void FTN_NAME(TCHARM_CREATE_DATA,tcharm_create_data)
+       (int *nThreads,
+                 TCharmThreadDataStartFn threadFn,
+                 void *threadData,int *threadDataLen)
+{ TCharmCreateData(*nThreads,threadFn,threadData,*threadDataLen); }
+
+
+/*Get the unconsumed command-line arguments*/
+CDECL char **TCharmArgv(void)
+{
+       if (TCharm::getState()!=inInit)
+               CkAbort("TCharm> Can only get arguments from in init!\n");
+       return cookie.getArgv();
+}
+CDECL int TCharmArgc(void)
+{
+       if (TCharm::getState()!=inInit)
+               CkAbort("TCharm> Can only get arguments from in init!\n");
+       return CmiGetArgc(cookie.getArgv());
+}
+
+CDECL int TCharmGetNumChunks(void)
+{
+       int nChunks=CkNumPes();
+       char **argv=TCharmArgv();
+       CmiGetArgInt(argv,"-vp",&nChunks);
+       CmiGetArgInt(argv,"+vp",&nChunks);
+       return nChunks;
+}
+FDECL int FTN_NAME(TCHARM_GET_NUM_CHUNKS,tcharm_get_num_chunks)(void)
+{
+       return TCharmGetNumChunks();
+}
+
+
+/***********************************
+Callable from worker thread
+*/
+CDECL int TCharmElement(void)
+{ return TCharm::get()->getElement();}
+CDECL int TCharmNumElements(void)
+{ return TCharm::get()->getNumElements();}
+
+FDECL int FTN_NAME(TCHARM_ELEMENT,tcharm_element)(void) 
+{ return TCharmElement();}
+FDECL int FTN_NAME(TCHARM_NUM_ELEMENTS,tcharm_num_elements)(void) 
+{ return TCharmNumElements();}
+
+CDECL int TCharmRegister(void *data,TCharmPupFn pfn)
+{ 
+       return TCharm::get()->add(TCharm::UserData(pfn,data));
+}
+FDECL int FTN_NAME(TCHARM_REGISTER,tcharm_register)
+       (void *data,TCpupUserDataF pfn)
+{ 
+       return TCharm::get()->add(TCharm::UserData(
+               pfn,data,TCharm::UserData::isFortran()));
+}
+
+CDECL void *TCharmGetUserdata(int id)
+{
+       return TCharm::get()->lookupUserData(id);
+}
+FDECL void *FTN_NAME(TCHARM_GET_USERDATA,tcharm_get_userdata)(int *id)
+{ return TCharmGetUserdata(*id); }
+
+CDECL void TCharmMigrate(void)
+{
+       TCharm::get()->migrate();
+}
+FDECL void FTN_NAME(TCHARM_MIGRATE,tcharm_migrate)(void)
+{
+       TCharm::get()->migrate();
+}
+
+CDECL void TCharmDone(void)
+{
+       TCharm::get()->done();
+}
+FDECL void FTN_NAME(TCHARM_DONE,tcharm_done)(void)
+{
+       TCharmDone();
+}
+
+
+
+#include "tcharm.def.h"
diff --git a/src/libs/ck-libs/tcharm/tcharm.ci b/src/libs/ck-libs/tcharm/tcharm.ci
new file mode 100644 (file)
index 0000000..dc7589e
--- /dev/null
@@ -0,0 +1,13 @@
+module tcharm {
+  message TCharmInitMsg {
+       char data[];
+  };
+  array [1D] TCharm {
+    entry TCharm(TCharmInitMsg *initMsg);
+    entry void run(void);
+    initcall void nodeInit(void);
+  };
+  mainchare TCharmMain {
+    entry TCharmMain();
+  };
+};
diff --git a/src/libs/ck-libs/tcharm/tcharm.h b/src/libs/ck-libs/tcharm/tcharm.h
new file mode 100644 (file)
index 0000000..4ceca05
--- /dev/null
@@ -0,0 +1,218 @@
+/*
+Threaded Charm++ "Framework Framework"
+
+Implements an array of migratable threads.
+Provides utility routines for registering user
+data, stopping, starting, and migrating threads.
+
+Orion Sky Lawlor, olawlor@acm.org, 11/19/2001
+*/
+#ifndef __CHARM_TCHARM_H
+#define __CHARM_TCHARM_H
+
+#include "tcharm.decl.h"
+#include "pup_c.h"
+#include "charm-api.h"
+
+class TCharm;
+
+class TCharmInitMsg : public CMessage_TCharmInitMsg {
+ public:
+       //Function to start thread with:
+       CthVoidFn threadFn;
+       //Initial stack size, in bytes:
+       int stackSize;
+       //Array size (number of elements)
+       int numElements;
+       //Data to pass to thread:
+       char *data;
+       
+       TCharmInitMsg(CthVoidFn threadFn_,int stackSize_)
+               :threadFn(threadFn_), stackSize(stackSize_) {}
+};
+
+//Current computation location
+typedef enum {inInit,inDriver,inFramework,inPup} inState;
+
+//Thread-local variables:
+CtvExtern(TCharm *,_curTCharm);
+CpvExtern(inState,_stateTCharm);
+
+CDECL {typedef void (*TCpupUserDataC)(pup_er p,void *data);};
+FDECL {typedef void (*TCpupUserDataF)(pup_er p,void *data);};
+
+class TCharm: public ArrayElement1D
+{
+ public:
+       
+       //User's heap-allocated/global data:
+       class UserData {
+               void *data; //user data pointer
+               bool isC;
+               TCpupUserDataC cfn;
+               TCpupUserDataF ffn;
+       public:
+               UserData() {data=NULL; cfn=NULL; ffn=NULL;}
+               UserData(TCpupUserDataC cfn_,void *data_)
+                       {cfn=cfn_; data=data_; isC=true;}
+               class isFortran{};
+               UserData(TCpupUserDataF ffn_,void *data_,isFortran tag)
+                       {ffn=ffn_; data=data_; isC=false;}
+               void *getData(void) {return data;}
+               void pup(PUP::er &p);
+       };
+
+       //One-time initialization
+       static void nodeInit(void);
+ private:
+       //Informational data about the current thread:
+       class ThreadInfo {
+       public:
+               CProxy_TCharm tProxy; //Our proxy
+               int thisElement; //Index of current element
+               int numElements; //Number of array elements
+       };
+       
+       TCharmInitMsg *initMsg; //Thread initialization data
+       CthThread tid; //Our migratable thread
+       bool isStopped;
+       ThreadInfo threadInfo;
+
+       enum {maxUserData=16};
+       int nUd;
+       UserData ud[maxUserData];
+
+       void ResumeFromSync(void);
+
+#ifdef CMK_OPTIMIZE
+       static inline void check(void) {}
+#else
+       static void check(void);
+#endif
+
+ public:
+       TCharm(TCharmInitMsg *initMsg);
+       TCharm(CkMigrateMessage *);
+       ~TCharm();
+       
+       //Pup routine packs the user data and migrates the thread
+       virtual void pup(PUP::er &p);
+
+       //Start running the thread for the first time
+       void run(void);
+
+//Client-callable routines:
+       //One client is ready to run
+       void ready(void);
+
+       //Thread finished running
+       void done(void);
+
+       //Register user data to be packed with the thread
+       int add(const UserData &d);
+       void *lookupUserData(int ud);
+       
+       inline static TCharm *get(void) {check(); return CtvAccess(_curTCharm);}
+       inline static inState getState(void) {return CpvAccess(_stateTCharm);}
+       inline static void setState(inState to) {CpvAccess(_stateTCharm)=to;}
+       inline CthThread getThread(void) {return tid;}
+       inline const CProxy_TCharm &getProxy(void) const {return threadInfo.tProxy;}
+       inline int getElement(void) const {return threadInfo.thisElement;}
+       inline int getNumElements(void) const {return threadInfo.numElements;}
+
+       //Start/stop load balancer measurements
+       inline void stopTiming(void) {ckStopTiming();}
+       inline void startTiming(void) {ckStartTiming();}
+
+       //Block our thread, run the scheduler, and come back
+       inline void schedule(void) {
+               stopTiming();
+               CthYield();
+               startTiming();
+       }
+
+       //As above, but start/stop the thread itself, too.
+       void stop(void); //Blocks; will not return until "start" called.
+       void start(void);
+       //Aliases:
+       inline void suspend(void) {stop();}
+       inline void resume(void) {start();}
+
+       //Go to sync, block, possibly migrate, and then resume
+       void migrate(void);
+};
+
+//Controls array startup, ready, run and shutdown
+class TCharmCoordinator {
+       static int nArrays; //Total number of running thread arrays
+       static TCharmCoordinator *head; //List of coordinators
+
+       TCharmCoordinator *next; //Next coordinator in list
+       CProxy_TCharm threads;//The threads I coordinate
+       int nThreads;//Number of threads (array elements)
+       int nClients; //Number of bound client arrays
+       int nReady; //Number of ready clients
+public:
+       TCharmCoordinator(CkArrayID threads,int nThreads);
+       ~TCharmCoordinator();
+       void addClient(const CkArrayID &client);
+       void clientReady(void);
+       void clientDone(void);
+       
+       static int getTotal(void) {
+               return nArrays;
+       }
+};
+
+//Controls initial setup (main::main & init routines)
+class TCharmSetupCookie {
+       enum {correctMagic=0x5432abcd};
+       int magic; //To make sure this is actually a cookie
+       
+       int stackSize; //Thread stack size, in bytes
+       char **argv; //Command-line arguments
+       
+       CkArrayID tc; //Handle to last-created TCharm array
+       int numElements; //Number of elements in last-created TCharm
+       TCharmCoordinator *coord; 
+
+       //Points to the active cookie
+       static TCharmSetupCookie *theCookie;
+       friend class TCharmMain;
+ public:
+       TCharmSetupCookie(char **argv_);
+       
+#ifdef CMK_OPTIMIZE //Skip check, for speed
+       inline TCharmSetupCookie &check(void) {return *this;}
+#else
+       TCharmSetupCookie &check(void);
+#endif
+       void setStackSize(int sz) {stackSize=sz;}
+       int getStackSize(void) const {return stackSize;}
+       char **getArgv(void) {return argv;}
+       
+       bool hasThreads(void) const {return 0!=coord;}
+       const CkArrayID &getThreads(void) const {return tc;}
+       TCharmCoordinator *getCoordinator(void) {return coord;}
+       int getNumElements(void) const {return numElements;}
+
+       void addClient(const CkArrayID &client) {coord->addClient(client);}
+       
+       void setThreads(const CkArrayID &aid,int nel);
+
+       static TCharmSetupCookie *get(void) {return theCookie;}
+};
+
+#include "tcharmc.h"
+
+//Node setup callbacks: called at startup on each node
+FDECL void FTN_NAME(TCHARM_USER_NODE_SETUP,tcharm_user_node_setup)(void);
+FDECL void FTN_NAME(TCHARM_USER_SETUP,tcharm_user_setup)(void);
+
+//Library fallback setup routine:
+typedef void (*TCharmFallbackSetupFn)(void);
+void TCharmSetFallbackSetup(TCharmFallbackSetupFn f);
+
+#endif
+
+
diff --git a/src/libs/ck-libs/tcharm/tcharmc.h b/src/libs/ck-libs/tcharm/tcharmc.h
new file mode 100644 (file)
index 0000000..6cdb167
--- /dev/null
@@ -0,0 +1,59 @@
+/*
+User-callable C API for TCharm library
+Orion Sky Lawlor, olawlor@acm.org, 11/20/2001
+*/
+#ifndef __TCHARM_H
+#define __TCHARM_H
+
+#include "pup_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*User callbacks: you define these functions*/
+void TCharmUserNodeSetup(void);
+void TCharmUserSetup(void);
+
+
+/*Routines you can call from UserSetup:*/
+
+/*Set the size of the thread stack*/
+void TCharmSetStackSize(int newStackSize);
+
+/*Create a new array of threads, which will be bound to by subsequent libraries*/
+typedef void (*TCharmThreadStartFn)(void);
+void TCharmCreate(int nThreads,TCharmThreadStartFn threadFn);
+
+/*As above, but pass along (arbitrary) data to thread*/
+typedef void (*TCharmThreadDataStartFn)(void *threadData);
+void TCharmCreateData(int nThreads,TCharmThreadDataStartFn threadFn,
+                 void *threadData,int threadDataLen);
+
+/*Get the unconsumed command-line arguments (C only; no Fortran)*/
+char **TCharmArgv(void);
+int TCharmArgc(void);
+
+/*Get the number of chunks we expect based on the command line*/
+int TCharmGetNumChunks(void);
+
+/*Implementation routines*/
+void TCharmInDefaultSetup(void);
+
+
+/*Routines you can call from a thread*/
+int TCharmElement(void);
+int TCharmNumElements(void);
+
+typedef void (*TCharmPupFn)(pup_er p,void *data);
+int TCharmRegister(void *data,TCharmPupFn pfn);
+void *TCharmGetUserdata(int id);
+void TCharmMigrate(void);
+void TCharmDone(void);
+
+#ifdef __cplusplus
+};
+#endif
+#endif /*def(thisHeader)*/
+
diff --git a/src/libs/ck-libs/tcharm/tcharmf.h b/src/libs/ck-libs/tcharm/tcharmf.h
new file mode 100644 (file)
index 0000000..2b9443e
--- /dev/null
@@ -0,0 +1,11 @@
+       external tcharm_set_stack_size
+       external tcharm_create
+       external tcharm_create_data
+       
+       integer, external :: tcharm_get_num_chunks
+       integer, external :: tcharm_element
+       integer, external :: tcharm_num_elements
+       
+       external tcharm_register
+       external tcharm_migrate
+       external tcharm_done