Modifications for TAU tracing in the Charm++ framework.
authorChee Wai Lee <cheelee@illinois.edu>
Sat, 22 Nov 2008 03:58:25 +0000 (03:58 +0000)
committerChee Wai Lee <cheelee@illinois.edu>
Sat, 22 Nov 2008 03:58:25 +0000 (03:58 +0000)
16 files changed:
README
build
src/arch/cell/cell_lib/spert_ppu.cpp
src/arch/origin-pthreads/machine.c
src/arch/util/machine-smp.c
src/ck-perf/trace-Tau.C [new file with mode: 0644]
src/ck-perf/trace-Tau.ci [new file with mode: 0644]
src/ck-perf/trace-Tau.h [new file with mode: 0644]
src/ck-perf/trace-TauBOC.h [new file with mode: 0644]
src/ck-perf/trace-common.C
src/ck-perf/trace-common.h
src/conv-core/threads.c
src/scripts/Make.depends
src/scripts/Makefile
src/scripts/charmc
tests/charm++/pingpong/Makefile

diff --git a/README b/README
index e4a30e7843ccf34a9cf8c0b22ab1e57aa30db92d..3d03d05258e70d45c753d58889b4189e5e39a72b 100644 (file)
--- a/README
+++ b/README
@@ -226,6 +226,7 @@ and run
        "charm++"  The basic Charm++ language.
        "AMPI"     An implementation of MPI on top of Charm++
        "FEM"      A Finite-Element framework on top of Charm++
        "charm++"  The basic Charm++ language.
        "AMPI"     An implementation of MPI on top of Charm++
        "FEM"      A Finite-Element framework on top of Charm++
+       "Tau"      TAU's performance profiling/tracing
 
 <version> is described above
 
 
 <version> is described above
 
diff --git a/build b/build
index a0263f318c4bf09fb1e55649366addfbc5a24aec..a8c1e0b816fbf0c5a9be3440d21dcd1a35406e61 100755 (executable)
--- a/build
+++ b/build
@@ -22,6 +22,7 @@ syntax() {
   echo '  pose            build POSE parallel discrete event simulator'
   echo '  jade            build Jade compiler (auto-builds charm++, msa)'
   echo '  msa             build Multiphase Shared Arrays(MSA) library'
   echo '  pose            build POSE parallel discrete event simulator'
   echo '  jade            build Jade compiler (auto-builds charm++, msa)'
   echo '  msa             build Multiphase Shared Arrays(MSA) library'
+  echo '  Tau             build the TAU tracing library for generating TAU performance data' 
   echo '  ChaNGa          compile Charm++ core and necessary libraries for ChaNGa'
   echo ''
   fi
   echo '  ChaNGa          compile Charm++ core and necessary libraries for ChaNGa'
   echo ''
   fi
@@ -89,6 +90,7 @@ syntax() {
   echo "  --with-romio       build AMPI with ROMIO library"
   echo "  --destination=DIR  build Charm++ inside DIR, by default the destination is <version>"
   echo "  --suffix=DIR       append DIR to the destination directory of the Charm++ build"
   echo "  --with-romio       build AMPI with ROMIO library"
   echo "  --destination=DIR  build Charm++ inside DIR, by default the destination is <version>"
   echo "  --suffix=DIR       append DIR to the destination directory of the Charm++ build"
+  echo "  --tau-makefile=FILE Specify which TAU stub makefile to use"
   fi
   echo ''
   echo '<charmc-options>: normal compiler options e.g. -g -O -save -verbose'
   fi
   echo ''
   echo '<charmc-options>: normal compiler options e.g. -g -O -save -verbose'
@@ -113,6 +115,7 @@ syntax() {
   echo '  ./build charm++ mpi-linux --incdir /usr/local/mpich/include --libdir /usr/local/mpich/lib -O'
   echo '   or in short,'
   echo '  ./build charm++ mpi-linux --basedir /usr/local/mpich -O'
   echo '  ./build charm++ mpi-linux --incdir /usr/local/mpich/include --libdir /usr/local/mpich/lib -O'
   echo '   or in short,'
   echo '  ./build charm++ mpi-linux --basedir /usr/local/mpich -O'
+  echo '  ./build Tau --tau-makefile=/usr/local/packages/TAU/x86_64/lib/Makefile.tau-mpi'
   echo ''
   echo 'Note: This script:'
   echo ' 1. Creates directories <destination> and <destination>/tmp';
   echo ''
   echo 'Note: This script:'
   echo ' 1. Creates directories <destination> and <destination>/tmp';
@@ -305,6 +308,14 @@ do
                DESTINATION_SUFFIX="`echo $1 | awk -F= '{print $2}'`"
                shift
                ;;
                DESTINATION_SUFFIX="`echo $1 | awk -F= '{print $2}'`"
                shift
                ;;
+    --tau-makefile)
+               TAU_MAKEFILE=$1
+               shift
+               ;;
+    --tau-makefile=*)
+               TAU_MAKEFILE=`echo $1 | awk -F= '{print $2}'`
+               shift
+               ;;
     -j*)
                 PMAKENUM=`echo $1 | awk -Fj '{print $2}'`
                MAKEOPTS="$MAKEOPTS -j $PMAKENUM"
     -j*)
                 PMAKENUM=`echo $1 | awk -Fj '{print $2}'`
                MAKEOPTS="$MAKEOPTS -j $PMAKENUM"
@@ -343,6 +354,34 @@ do
        ;;
   esac
 done
        ;;
   esac
 done
+if [ "$PROGRAM" = "Tau" ]
+then
+       echo "TAU>>>> makefile config option: $TAU_MAKEFILE"
+       if [ -f $TAU_MAKEFILE ]
+       then
+       #Setting up TAU trace library:
+       echo "TAU>>>> configuring with this TAU makefile: $TAU_MAKEFILE"
+
+       SED_CHARMC="s@TAU_MAKEFILE=\(.*\)@TAU_MAKEFILE=$TAU_MAKEFILE@"
+
+       echo $SED_CHARMC
+       sed -i -e $SED_CHARMC src/scripts/Makefile 
+       echo $SED_CHARMC
+       sed -i -e $SED_CHARMC src/scripts/charmc  
+       else
+       echo "TAU>>>> ERROR could not find Makefile: $TAU_MAKEFILE"
+       fi
+else
+    # Still need to set the variables in Makefile and charmc to ""
+    # even in the non-Tau case.
+    TAU_MAKEFILE=""
+    SED_CHARMC="s@TAU_MAKEFILE=\(.*\)@TAU_MAKEFILE=$TAU_MAKEFILE@"
+
+    echo $SED_CHARMC
+    sed -i -e $SED_CHARMC src/scripts/Makefile
+    echo $SED_CHARMC
+    sed -i -e $SED_CHARMC src/scripts/charmc
+fi
 
 [ "x$VERSION" = "x" ] && syntax && exit 1
 
 
 [ "x$VERSION" = "x" ] && syntax && exit 1
 
index 8a0efe548cbed5a5dc6efafe578167cfd1cba6e8..a9753c117d0859681bca83eb2834bb1dddae0d53 100644 (file)
@@ -2563,8 +2563,8 @@ void* pthread_func(void* arg) {
     fprintf(stderr, "OffloadAPI :: ERROR : Unable to destroy SPE Context\n");
     exit(EXIT_FAILURE);
   }
     fprintf(stderr, "OffloadAPI :: ERROR : Unable to destroy SPE Context\n");
     exit(EXIT_FAILURE);
   }
-
-  pthread_exit(NULL);
+  printf("called tau pthread exit");
+  tau_pthread_exit(NULL);
   return NULL;
 }
 
   return NULL;
 }
 
@@ -2685,7 +2685,8 @@ SPEThread* createSPEThread(SPEData *speData) {
   }
 
   // Create the pthread for the SPE Thread
   }
 
   // Create the pthread for the SPE Thread
-  int rtnCode = pthread_create(&(speThread->pThread), NULL, pthread_func, speThread);
+       printf("calling tau_pthread_create");
+  int rtnCode = tau_pthread_create(&(speThread->pThread), NULL, pthread_func, speThread);
   if (rtnCode != 0) {
     fprintf(stderr, "OffloadAPI :: ERROR : Unable to create pthread (rtnCode = %d)\n", rtnCode);
     exit(EXIT_FAILURE);
   if (rtnCode != 0) {
     fprintf(stderr, "OffloadAPI :: ERROR : Unable to create pthread (rtnCode = %d)\n", rtnCode);
     exit(EXIT_FAILURE);
@@ -2836,7 +2837,8 @@ SPEThread** createSPEThreads(SPEThread **speThreads, int numThreads) {
     }
 
     // Create the pthread for the SPE Thread
     }
 
     // Create the pthread for the SPE Thread
-    int rtnCode = pthread_create(&(speThreads[i]->pThread), NULL, pthread_func, speThreads[i]);
+         printf("calling tau_pthread_create");
+    int rtnCode = tau_pthread_create(&(speThreads[i]->pThread), NULL, pthread_func, speThreads[i]);
     if (rtnCode != 0) {
       fprintf(stderr, "OffloadAPI :: ERROR : Unable to create pthread (rtnCode = %d)\n", rtnCode);
       exit(EXIT_FAILURE);
     if (rtnCode != 0) {
       fprintf(stderr, "OffloadAPI :: ERROR : Unable to create pthread (rtnCode = %d)\n", rtnCode);
       exit(EXIT_FAILURE);
index c9b1008c9b089ce1a2b6affe074a3eb29426a89d..d715547737a7f42b2897a412196a26793e65914e 100644 (file)
@@ -183,7 +183,8 @@ void ConverseInit(int argc, char **argv, CmiStartFn fn, int usched, int initret)
     usrparam->argv = CmiCopyArgs(argv);
     usrparam->mype = i;
 
     usrparam->argv = CmiCopyArgs(argv);
     usrparam->mype = i;
 
-    pthread_create(&aThread[i],(pthread_attr_t *)0,threadInit,(void *)usrparam);
+         printf("calling tau_pthread_create");
+    tau_pthread_create(&aThread[i],(pthread_attr_t *)0,threadInit,(void *)usrparam);
   }
   usrparam = (USER_PARAMETERS *) CmiAlloc(sizeof(USER_PARAMETERS));
   usrparam->argv = CmiCopyArgs(argv);
   }
   usrparam = (USER_PARAMETERS *) CmiAlloc(sizeof(USER_PARAMETERS));
   usrparam->argv = CmiCopyArgs(argv);
index b91e548357f4752e0127a82d20b6e795c2d9a7a3..d56d1291625c9a61c8c993f411033b5326200aff 100644 (file)
@@ -428,7 +428,8 @@ static void CmiStartThreads(char **argv)
   for (i=1; i<=tocreate; i++) {
     pthread_attr_init(&attr);
     pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
   for (i=1; i<=tocreate; i++) {
     pthread_attr_init(&attr);
     pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
-    ok = pthread_create(&pid, &attr, call_startfn, (void *)i);
+         printf("calling tau_pthread_create");
+    ok = tau_pthread_create(&pid, &attr, call_startfn, (void *)i);
     if (ok<0) PerrorExit("pthread_create"); 
     pthread_attr_destroy(&attr);
   }
     if (ok<0) PerrorExit("pthread_create"); 
     pthread_attr_destroy(&attr);
   }
diff --git a/src/ck-perf/trace-Tau.C b/src/ck-perf/trace-Tau.C
new file mode 100644 (file)
index 0000000..f9aa94d
--- /dev/null
@@ -0,0 +1,430 @@
+#include <stdlib.h>
+#include "charm++.h"
+#include "trace-Tau.h"
+#include "trace-TauBOC.h"
+#include "trace-common.h"
+#include "TAU.h"
+//#include "tau_selective.cpp"
+#include "map"
+#include "stack"
+#include <string>
+using namespace std;
+
+#if defined(_WIN32) && !defined(__CYGWIN__)
+#include <direct.h>
+#define CHDIR _chdir
+#define GETCWD _getcwd
+#define PATHSEP '\\'
+#define PATHSEPSTR "\\"
+#else
+#include <unistd.h>
+#define CHDIR chdir
+#define GETCWD getcwd
+#define PATHSEP '/'
+#define PATHSEPSTR "/"
+#endif
+
+/*#ifndef PROFILING_ON
+void TAU_PROFILER_CREATE(void *p, char *n, char *s, taugroup_t t) {
+dprintf("---> tau
+create profiler: %s \n", s); }
+
+void TAU_PROFILER_STOP(void *p) { dprintf("---> tau
+stop profiler"); }
+
+void TAU_PROFILER_START(void *p) { dprintf("---> tau
+start profiler"); }
+
+void TAU_PROFILE_SET_NODE(int i) { dprintf("---> tau
+set node"); }
+#endif
+*/
+
+#ifdef DEBUG_PROF
+#define dprintf printf
+#else // DEBUG_PROF 
+#define dprintf if (0) printf
+#endif
+
+extern bool processFileForInstrumentation(const string& file_name);
+extern void printExcludeList();
+extern bool instrumentEntity(const string& function_name);
+extern int processInstrumentationRequests(char *fname);
+// Charm++ "processor"(user thread)-private global variable
+CkpvStaticDeclare(TraceTau*, _trace);
+
+// This global variable is required for any post-execution 
+// parallel analysis or activities the trace module might wish to perform.
+CkGroupID traceTauGID;
+
+/**
+  For each TraceFoo module, _createTraceFoo() must be defined.
+  This function is called in _createTraces() generated in moduleInit.C
+*/
+
+void *idle, *comp;
+//char *name = "default";
+bool profile = true, snapshotProfiling = false;
+
+map<const int, void*> events;
+stack<void*> eventStack;
+void *EXCLUDED = NULL;
+void startEntryEvent(int id)
+{
+  dprintf("---------> starting Entry Event with id: %d\n", id);
+
+  if (events[id] == NULL)
+       {
+         dprintf("-------> create event with id: %d\n", id);
+               //sprintf(name, "Event %d", id);
+               if (id == -1)
+               { /*
+                       char *name = "dummy_thread_ep";
+                       dprintf(" ------> creating event: %s\n", name);
+                       TAU_PROFILER_CREATE(events[id], name, "", TAU_DEFAULT);
+                       dprintf("timer created.\n");
+                       eventStack.push(events[id]);
+                       dprintf(" ------> starting event: %s\n", (char*) name);
+                       TAU_PROFILER_START(eventStack.top());*/
+      //exclude dummy event
+                       dprintf("------> excluding dummy function");
+                       events[id] = EXCLUDED;
+                       eventStack.push(events[id]);
+               }
+               else
+               {
+                       //string check("doFFT(RSFFTMsg* impl_msg)");
+                       //string name_s(_entryTable[id]->name);
+                       //printf("checking name4: %s", _entryTable[id]->name);
+                       //if (check.compare(name_s) != 0)
+                       //{
+                               char name [500];
+                               sprintf(name, "%s::%s::%d", _chareTable[_entryTable[id]->chareIdx]->name,
+                               _entryTable[id]->name, id);
+                               //should this fuction be excluded from instrumentation?
+                               if (!instrumentEntity(name))
+                               {
+                                 //exclude function.
+                                       dprintf("------> excluding function %s\n", name);
+                                       events[id] = EXCLUDED;
+                                       eventStack.push(events[id]);
+                               }
+                               else
+                               {
+                                       dprintf(" ------> creating event: %s\n", name);
+                                       TAU_PROFILER_CREATE(events[id], name, "", TAU_DEFAULT);
+                                       dprintf("timer created.\n");
+                                       eventStack.push(events[id]);
+                                       dprintf("starting event\n");
+                                       dprintf(" ------> starting event: %s\n", (char*) name);
+                                       TAU_PROFILER_START(eventStack.top());
+                         }
+                       dprintf("done.\n");
+               }
+       }
+       else
+       {
+               eventStack.push(events[id]);
+         if (events[id] != EXCLUDED)
+               {
+                       TAU_PROFILER_START(eventStack.top());
+               }
+       }
+}
+
+void stopEntryEvent()
+{
+  dprintf("stop timer...\n");
+  if (eventStack.top() != EXCLUDED)
+       {
+    TAU_PROFILER_STOP(eventStack.top());
+       }
+       eventStack.pop();
+}
+
+
+void _createTraceTau(char **argv)
+{
+  //TAU_INIT(1, argv);
+  dprintf("arguments:\n");
+  dprintf("[0] = %s, ", argv[0]);
+  dprintf("[1] = %s, ", argv[1]);
+  dprintf("[2] = %s, ", argv[2]);
+       dprintf("\n");
+       string disable = "disable-profiling";
+  if (argv[1] == NULL) { profile = true; }
+       else if (argv[1] == disable) { profile = false; }
+  if (not CkpvAccess(traceOn)) { 
+         dprintf("traceoff selected using snapshot profiling.\n");
+         snapshotProfiling = true; 
+       }
+
+  CkpvInitialize(TraceTau*, _trace);
+  CkpvAccess(_trace) = new TraceTau(argv);
+  CkpvAccess(_traces)->addTrace(CkpvAccess(_trace));
+}
+
+TraceTau::TraceTau(char **argv)
+{
+  if (CkpvAccess(traceOnPe) == 0) return;
+
+  // Process runtime arguments intended for the module
+  CmiGetArgIntDesc(argv,"+TauPar0", &par0, "Fake integer parameter 0");
+  CmiGetArgDoubleDesc(argv,"+TauPar1", &par1, "Fake double parameter 1");
+  //TAU_REGISTER_THREAD();
+       if (profile)
+       {
+               if (strcmp(CkpvAccess(selective), ""))
+               {
+                 //printf("select file: %s\n", CkpvAccess(selective));
+                       //processFileForInstrumentation(CkpvAccess(selective));
+                       processInstrumentationRequests(CkpvAccess(selective));
+                       printExcludeList();
+                       if (!instrumentEntity("Main::done(void)::99"))
+                       {
+                         dprintf("selective file working...\n");
+                       }
+                       else
+                         dprintf("selective flile not working...\n");
+               }
+        
+               TAU_PROFILER_CREATE(idle, "Idle", "", TAU_DEFAULT);
+               //TAU_PROFILER_CREATE(entry,name,"", TAU_DEFAULT);
+               dprintf("before %p\n", comp);  
+         TAU_PROFILER_CREATE(comp, "Main", "", TAU_DEFAULT);
+               dprintf("after %p\n", comp);  
+
+               //Need to add an entry timer to the top of the stack because
+               //traceTauExitFunction calls CkExit() which calls endExecute
+         eventStack.push(EXCLUDED);
+       }
+       else 
+       {
+         dprintf("--> [TAU] creating timers...\n");
+       }
+}
+
+void TraceTau::userEvent(int eventID) 
+{
+  dprintf("[%d] User Point Event id %d encountered\n", CkMyPe(), eventID);
+}
+
+void TraceTau::userBracketEvent(int eventID, double bt, double et) {
+  dprintf("[%d] User Bracket Event id %d encountered\n", CkMyPe(), eventID);
+}
+
+void TraceTau::creation(envelope *, int epIdx, int num) {
+  dprintf("[%d] Point-to-Point Message for Entry Method id %d sent\n",
+          CkMyPe(), epIdx);
+}
+
+void TraceTau::creationMulticast(envelope *, int epIdx, int num, 
+                                   int *pelist) {
+  dprintf("[%d] Multicast Message for Entry Method id %d sent to %d pes\n",
+          CkMyPe(), epIdx, num);
+}
+
+void TraceTau::creationDone(int num) {
+  dprintf("[%d] Last initiated send completes\n", CkMyPe());
+}
+  
+void TraceTau::messageRecv(char *env, int pe) {
+  dprintf("[%d] Message from pe %d received by scheduler\n", 
+          CkMyPe(), pe);
+}
+  
+void TraceTau::beginExecute(CmiObjId *tid)
+{
+  // CmiObjId is a 4-integer tuple uniquely identifying a migratable
+  //   Charm++ object. Note that there are other non-migratable Charm++
+  //   objects that CmiObjId will not identify.
+  dprintf("[%d] Entry Method invoked using object id\n", CkMyPe());
+               if (profile) {
+                 startEntryEvent(-1);
+               }
+               else
+               {
+           dprintf("--> [TAU] starting entry timer...\n");
+               }
+}
+
+void TraceTau::beginExecute(envelope *e)
+{
+  // no message means thread execution
+  if (e == NULL) {
+    dprintf("[%d] Entry Method invoked via thread id %d\n", CkMyPe(),
+            _threadEP);
+               if (profile) {
+                 startEntryEvent(-1);
+               }
+               else
+               {
+           dprintf("--> [TAU] starting entry timer...\n");
+               }
+    // Below is what is found in trace-summary.
+    // beginExecute(-1,-1,_threadEP,-1);
+  } else {
+    dprintf("[%d] Entry Method %d invoked via message envelope\n", 
+            CkMyPe(), e->getEpIdx());
+               if (profile) {
+                 startEntryEvent(e->getEpIdx());
+               }
+               else
+               {
+           dprintf("--> [TAU] starting entry timer...\n");
+               }
+    // Below is what is found in trace-summary.
+    // beginExecute(-1,-1,e->getEpIdx(),-1);
+  }  
+}
+
+void TraceTau::beginExecute(int event,int msgType,int ep,int srcPe, 
+                              int mlen, CmiObjId *idx)
+{
+  dprintf("[%d] Entry Method %d invoked by parameters\n", CkMyPe(),
+          ep);
+               if (profile) {
+                 startEntryEvent(ep);
+               }
+               else
+               {
+           dprintf("--> [TAU] starting entry timer...\n");
+               }
+}
+
+void TraceTau::endExecute(void)
+{
+               if (profile) {
+                 stopEntryEvent();
+               }
+               else
+               {
+           dprintf("--> [TAU] stoping entry timer...\n");
+               }
+  dprintf("[%d] Previously executing Entry Method completes\n", CkMyPe());
+}
+
+void TraceTau::beginIdle(double curWallTime) {
+  dprintf("[%d] Scheduler has no useful user-work\n", CkMyPe());
+               if (profile) {
+           TAU_PROFILER_START(idle);
+               }
+               else
+               {
+           dprintf("--> [TAU] starting idle timer...\n");
+               }
+}
+
+void TraceTau::endIdle(double curWallTime) {
+               if (profile) {
+           TAU_PROFILER_STOP(idle);
+               }
+               else
+               {
+           dprintf("--> [TAU] stopping idle timer...\n");
+               }
+  dprintf("[%d] Scheduler now has useful user-work\n", CkMyPe());
+}
+  
+void TraceTau::beginComputation(void)
+{
+  dprintf("[%d] Computation Begins\n", CkMyPe());
+  // Code Below shows what trace-summary would do.
+  // initialze arrays because now the number of entries is known.
+  // _logPool->initMem();
+}
+
+void TraceTau::endComputation(void)
+{
+  dprintf("[%d] Computation Ends\n", CkMyPe());
+}
+
+void TraceTau::traceBegin(void)
+{
+               dprintf("[%d] Tracing Begins\n", CkMyPe());
+               if (profile) {
+                       dprintf("ptr: %p\n", comp);  
+                       TAU_DB_PURGE();
+      TAU_ENABLE_ALL_GROUPS();
+           TAU_PROFILER_START(comp);
+               }
+               else
+               {
+           dprintf("--> [TAU] starting computation timer...\n");
+               }
+}
+void TraceTau::traceEnd(void)
+{
+  dprintf("[%d] Tracing Ends\n", CkMyPe());
+               if (profile){
+                       dprintf("ptr: %p\n", comp);  
+                       //TAU_PROFILER_STOP(comp);
+           TAU_PROFILE_EXIT("tracing complete.");
+      TAU_DISABLE_ALL_GROUPS();
+               }
+               else
+               {
+           dprintf("--> [TAU] stopping computation timer and writing profiles\n");
+               }
+  dprintf("[%d] Computation Ends\n", CkMyPe());
+}
+
+void TraceTau::malloc(void *where, int size, void **stack, int stackSize)
+{
+  dprintf("[%d] Memory allocation of size %d occurred\n", CkMyPe(), size);
+}
+
+void TraceTau::free(void *where, int size) {
+  dprintf("[%d] %d-byte Memory block freed\n", CkMyPe(), size);
+}
+
+void TraceTau::traceClose(void)
+{
+       dprintf("traceClose called.\n");
+  CkpvAccess(_trace)->endComputation();
+  CkpvAccess(_trace)->traceEnd();
+       //TAU_PROFILE_EXIT("closing trace...");
+       //dprintf(" [%d] Exit called \n", CkMyPe());
+       //TAU_PROFILE_EXIT("exiting...");
+  // remove myself from traceArray so that no tracing will be called.
+  CkpvAccess(_traces)->removeTrace(this);
+}
+
+extern "C" void traceTauExitFunction() {
+       dprintf("traceTauExitFunction called.\n");
+  // The exit function of any Charm++ module must call CkExit() or
+  // the entire exit process will hang if multiple modules are linked.
+  // FIXME: This is NOT a feature. Something needs to be done about this.
+       //TAU_PROFILE_EXIT("exiting...");
+       //TAU_PROFILE_EXIT("done");
+       //eventStack.push(NULL);
+  CkExit();
+}
+
+// Initialization of the parallel trace module.
+void initTraceTauBOC() {
+  //void *main;
+  dprintf("tracetauboc setting node %d\n", CmiMyPe());
+       if (profile) {
+    TAU_PROFILE_SET_NODE(CmiMyPe());
+       }
+       else
+       {
+         dprintf("---> [TAU] settting node.\n");
+       }
+       //TAU_PROFILER_CREATE(main, "main", "", TAU_DEFAULT);
+       //TAU_PROFILER_START(main);
+#ifdef __BLUEGENE__
+  if (BgNodeRank()==0) {
+#else
+    if (CkMyRank() == 0) {
+#endif
+      registerExitFn(traceTauExitFunction);
+    }
+}
+
+#include "TraceTau.def.h"
+
+
+/*@}*/
diff --git a/src/ck-perf/trace-Tau.ci b/src/ck-perf/trace-Tau.ci
new file mode 100644 (file)
index 0000000..562f84e
--- /dev/null
@@ -0,0 +1,15 @@
+module TraceTau {
+
+  mainchare TraceTauInit {
+    entry TraceTauInit(CkArgMsg *m);
+  };
+
+  initnode void initTraceTauBOC();
+  readonly CkGroupID traceTauGID;
+
+  group [migratable] TraceTauBOC {
+    entry TraceTauBOC(void);
+  };
+
+};
+
diff --git a/src/ck-perf/trace-Tau.h b/src/ck-perf/trace-Tau.h
new file mode 100644 (file)
index 0000000..3cc8184
--- /dev/null
@@ -0,0 +1,74 @@
+#ifndef _VERBOSE_H
+#define _VERBOSE_H
+
+#include <stdio.h>
+#include <errno.h>
+
+#include "trace.h"
+#include "envelope.h"
+#include "register.h"
+#include "trace-common.h"
+
+/**
+ *  TraceTau is a template for an instrumentation module making use of
+ *    the tracing framework hooks provided in Charm++.
+ *
+ *  Only the more common hooks are listened to in this module.
+ */
+class TraceTau : public Trace {
+ public:
+  int par0;
+  double par1;
+ public:
+  TraceTau(char **argv);
+  
+  // a user event has just occured
+  void userEvent(int eventID);
+  // a pair of begin/end user event has just occured
+  void userBracketEvent(int eventID, double bt, double et);
+  
+  // "creation" of message(s) - message Sends
+  void creation(envelope *, int epIdx, int num=1);
+  void creationMulticast(envelope *, int epIdx, int num=1, int *pelist=NULL);
+  void creationDone(int num=1);
+  
+  void messageRecv(char *env, int pe);
+  
+  // **************************************************************
+  // begin/end execution of a Charm++ entry point
+  // NOTE: begin/endPack and begin/endUnpack can be called in between
+  //       a beginExecute and its corresponding endExecute.
+  void beginExecute(envelope *);
+  void beginExecute(CmiObjId *tid);
+  void beginExecute(
+                   int event,   // event type defined in trace-common.h
+                   int msgType, // message type
+                   int ep,      // Charm++ entry point id
+                   int srcPe,   // Which PE originated the call
+                   int ml,      // message size
+                   CmiObjId* idx);    // index
+  void endExecute(void);
+  
+  // begin/end idle time for this pe
+  void beginIdle(double curWallTime);
+  void endIdle(double curWallTime);
+  
+  // begin/end of execution
+  void beginComputation(void);
+  void endComputation(void);
+
+       //begin/end tracing
+       void traceBegin(void);
+       void traceEnd(void);
+  
+  /* Memory tracing */
+  void malloc(void *where, int size, void **stack, int stackSize);
+  void free(void *where, int size);
+  
+  // do any clean-up necessary for tracing
+  void traceClose();
+};
+
+#endif
+
+/*@}*/
diff --git a/src/ck-perf/trace-TauBOC.h b/src/ck-perf/trace-TauBOC.h
new file mode 100644 (file)
index 0000000..2f60976
--- /dev/null
@@ -0,0 +1,31 @@
+#include "TraceTau.decl.h"
+
+extern CkGroupID traceTauGID;
+
+// We typically declare parallel object classes here for the purposes of
+// performing parallel operations for the trace module after the main
+// application has completed execution (and calls CkExit()).
+//
+// TraceTauInit is an initialization class.
+//
+// TraceTauBOC is a one-per-processor object (defined in the .ci file as
+//    a "group" instead of a "chare") which hosts the methods for the
+//    parallel operations. In this case, there are no methods defined.
+//    Otherwise, one may write any Charm++ code here.
+
+class TraceTauInit : public Chare {
+  public:
+  TraceTauInit(CkArgMsg*) {
+    traceTauGID = CProxy_TraceTauBOC::ckNew();
+    CProxy_TraceTauBOC tauProxy(traceTauGID);
+  }
+  TraceTauInit(CkMigrateMessage *m):Chare(m) {}
+};
+
+class TraceTauBOC : public CBase_TraceTauBOC {
+public:
+  TraceTauBOC(void) {};
+  TraceTauBOC(CkMigrateMessage *m) {};
+};
+
+
index 8958cb8ea05ee7747001e36a1d49c5043b6acbeb..8d7567de05839fa7dd60f53b52e9fd4f2ee8c781 100644 (file)
@@ -59,6 +59,7 @@ CkpvDeclare(double, traceInitCpuTime);
 CpvDeclare(int, traceOn);
 CkpvDeclare(int, traceOnPe);
 CkpvDeclare(char*, traceRoot);
 CpvDeclare(int, traceOn);
 CkpvDeclare(int, traceOnPe);
 CkpvDeclare(char*, traceRoot);
+CkpvDeclare(char*, selective);
 CkpvDeclare(bool, verbose);
 
 typedef void (*mTFP)();                   // function pointer for
 CkpvDeclare(bool, verbose);
 
 typedef void (*mTFP)();                   // function pointer for
@@ -126,6 +127,34 @@ static void traceCommonInit(char **argv)
     CkpvAccess(traceRoot) = (char *) malloc(strlen(argv[0])+1);
     _MEMCHECK(CkpvAccess(traceRoot));
     strcpy(CkpvAccess(traceRoot), argv[0]);
     CkpvAccess(traceRoot) = (char *) malloc(strlen(argv[0])+1);
     _MEMCHECK(CkpvAccess(traceRoot));
     strcpy(CkpvAccess(traceRoot), argv[0]);
+  }
+       /* added for TAU trace module. */
+       char *cwd;
+  CkpvInitialize(char*, selective);
+  if (CmiGetArgStringDesc(argv, "+selective", &temproot, "TAU's selective instrumentation file")) {
+    int i;
+    // Trying to decide if the traceroot path is absolute or not. If it is not
+    // then create an absolute pathname for it.
+    if (temproot[0] != PATHSEP) {
+      cwd = GETCWD(NULL,0);
+      root = (char *)malloc(strlen(cwd)+strlen(temproot)+2);
+      strcpy(root, cwd);
+      strcat(root, PATHSEPSTR);
+      strcat(root, temproot);
+    } else {
+      root = (char *)malloc(strlen(temproot)+1);
+      strcpy(root,temproot);
+    }
+    CkpvAccess(selective) = (char *) malloc(strlen(root)+1);
+    _MEMCHECK(CkpvAccess(selective));
+    strcpy(CkpvAccess(selective), root);
+    if (CkMyPe() == 0) 
+      CmiPrintf("Trace: selective: %s\n", CkpvAccess(selective));
+  }
+  else {
+    CkpvAccess(selective) = (char *) malloc(3);
+    _MEMCHECK(CkpvAccess(selective));
+    strcpy(CkpvAccess(selective), "");
   }
   
 #ifdef __BLUEGENE__
   }
   
 #ifdef __BLUEGENE__
index 88876db98ba7b3aff6612fbd3ae97d84e248ad57..56774dda9f201e6cf8c5e075c0119a23f5091304 100644 (file)
@@ -53,6 +53,7 @@
 
 CkpvExtern(int, CtrLogBufSize);
 CkpvExtern(char*, traceRoot);
 
 CkpvExtern(int, CtrLogBufSize);
 CkpvExtern(char*, traceRoot);
+CkpvExtern(char*, selective);
 CkpvExtern(bool, verbose);
 CkpvExtern(double, traceInitTime);
 CkpvExtern(double, traceInitCpuTime);
 CkpvExtern(bool, verbose);
 CkpvExtern(double, traceInitTime);
 CkpvExtern(double, traceInitCpuTime);
index 0388cb408b3a796615ff271962c0c82aeb765243..5e5a6864651aa11f28afb756d4eafdf148306d3f 100644 (file)
@@ -1220,7 +1220,9 @@ void CthResume(CthThread t)
   pthread_cond_signal(&(t->cond)); /* wake up the next thread */
   if (tc->base.exiting) {
     pthread_mutex_unlock(&CthCpvAccess(sched_mutex));
   pthread_cond_signal(&(t->cond)); /* wake up the next thread */
   if (tc->base.exiting) {
     pthread_mutex_unlock(&CthCpvAccess(sched_mutex));
-    pthread_exit(0);
+
+       printf("calling tau_pthread_exit");
+    tau_pthread_exit(0);
   } else {
     /* pthread_cond_wait might (with low probability) return when the 
       condition variable has not been signaled, guarded with 
   } else {
     /* pthread_cond_wait might (with low probability) return when the 
       condition variable has not been signaled, guarded with 
@@ -1273,8 +1275,9 @@ CthThread CthCreate(CthVoidFn fn, void *arg, int size)
         reported = 1;
       }
   }
         reported = 1;
       }
   }
-
-  r = pthread_create(&(result->self), &attr, CthOnly, (void*) result);
+       
+       printf("calling tau_pthread_create");
+  r = tau_pthread_create(&(result->self), &attr, CthOnly, (void*) result);
   if (0 != r) {
     CmiPrintf("pthread_create failed with %d\n", r);
     CmiAbort("CthCreate failed to created a new pthread\n");
   if (0 != r) {
     CmiPrintf("pthread_create failed with %d\n", r);
     CmiAbort("CthCreate failed to created a new pthread\n");
index da8084a8f51f9f04a11bfb43621323b8dbe4cab9..d51db227bd8986c3c2b279209f78d0aeb3662dc9 100644 (file)
@@ -262,15 +262,15 @@ trace-simple.o: trace-simple.C charm++.h charm.h converse.h conv-config.h \
 trace-controlPoints.o: trace-controlPoints.C charm++.h charm.h converse.h \
   conv-config.h conv-autoconfig.h conv-common.h conv-mach.h \
   conv-mach-opt.h pup_c.h conv-cpm.h conv-cpath.h conv-qd.h conv-random.h \
 trace-controlPoints.o: trace-controlPoints.C charm++.h charm.h converse.h \
   conv-config.h conv-autoconfig.h conv-common.h conv-mach.h \
   conv-mach-opt.h pup_c.h conv-cpm.h conv-cpath.h conv-qd.h conv-random.h \
-  conv-lists.h conv-trace.h persistent.h conv-onesided.h debug-conv.h \
-  pup.h middle.h middle-conv.h cklists.h ckbitvector.h ckstream.h init.h \
-  ckhashtable.h debug-charm.h CkMarshall.decl.h cksection.h ckcallback.h \
-  conv-ccs.h sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h \
-  CkReduction.decl.h cknodegroupreduction.h CkArrayReductionMgr.decl.h \
-  ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
-  cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
-  LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
-  CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
+  conv-lists.h conv-trace.h persistent.h debug-conv.h pup.h middle.h \
+  middle-conv.h cklists.h ckbitvector.h ckstream.h init.h ckhashtable.h \
+  debug-charm.h CkMarshall.decl.h cksection.h ckcallback.h conv-ccs.h \
+  sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h CkReduction.decl.h \
+  cknodegroupreduction.h CkArrayReductionMgr.decl.h ckmemcheckpoint.h \
+  CkMemCheckpoint.decl.h readonly.h ckarray.h cklocation.h LBDatabase.h \
+  lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h \
+  LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h CkLocation.decl.h \
+  CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
   sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
   sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
index cb6c7b28d36911ede686e982d598e7c5bd322ffd..d449d0333983faf1c8483abe3886198f7e80f13e 100644 (file)
@@ -103,6 +103,15 @@ jadeall: jade
 #      make the jade.jar file
        cd ../java/charm/jade && $(MAKE)
 
 #      make the jade.jar file
        cd ../java/charm/jade && $(MAKE)
 
+#include TAU defs
+TAU_MAKEFILE=/expand/home/cheelee/work/tau-charm/tau-2.17.3b2/x86_64/lib/Makefile.tau
+include        $(TAU_MAKEFILE) #stub-makefile
+
+#builds TAU's trace library
+Tau: 
+       echo "buliding charm trace library with support for TAU."
+       $(MAKE) $(L)/libtrace-Tau.a $(L)/libtrace-all.a TAU_TRACE_OBJ=libtrace-Tau.o
+
 LIBS: CONVLIBS CHARMLIBS
        cd libs; $(MAKE) otherlibs
 
 LIBS: CONVLIBS CHARMLIBS
        cd libs; $(MAKE) otherlibs
 
@@ -213,7 +222,8 @@ CKHEADERS=ck.h ckstream.h envelope.h init.h qd.h charm.h charm++.h \
           NeighborCommLB.decl.h HbmLB.decl.h \
           HybridBaseLB.decl.h HybridLB.decl.h WSLB.decl.h EveryLB.decl.h \
           charisma.decl.h TraceSummary.decl.h TraceProjections.decl.h \
           NeighborCommLB.decl.h HbmLB.decl.h \
           HybridBaseLB.decl.h HybridLB.decl.h WSLB.decl.h EveryLB.decl.h \
           charisma.decl.h TraceSummary.decl.h TraceProjections.decl.h \
-          TraceSimple.decl.h TraceControlPoints.decl.h BlueGene.decl.h \
+          TraceSimple.decl.h TraceControlPoints.decl.h TraceTau.decl.h \
+         BlueGene.decl.h \
          comlib.decl.h ComlibManager.h DummyStrategy.h \
          DirectMulticastStrategy.h RingMulticastStrategy.h     \
          MultiRingMulticast.h \
          comlib.decl.h ComlibManager.h DummyStrategy.h \
          DirectMulticastStrategy.h RingMulticastStrategy.h     \
          MultiRingMulticast.h \
@@ -393,7 +403,7 @@ CONVLIBS: converse
        cd libs; $(MAKE) convlibs OPTS='$(OPTS)'
 
 $(L)/libconv-core.a: $(LIBCONV_CORE)
        cd libs; $(MAKE) convlibs OPTS='$(OPTS)'
 
 $(L)/libconv-core.a: $(LIBCONV_CORE)
-       $(CHARMC) -o $@ $(LIBCONV_CORE)
+       $(CHARMC) $(TAU_LIBS) -o $@ $(LIBCONV_CORE) 
 
 $(L)/libconv-cplus-y.a: $(L)/libconv-cplus-n.a
 
 
 $(L)/libconv-cplus-y.a: $(L)/libconv-cplus-n.a
 
@@ -672,6 +682,10 @@ TraceSimple.def.h: TraceSimple.decl.h
 TraceSimple.decl.h: trace-simple.ci charmxi
        $(CHARMXI) trace-simple.ci
 
 TraceSimple.decl.h: trace-simple.ci charmxi
        $(CHARMXI) trace-simple.ci
 
+TraceTau.def.h: TraceTau.decl.h
+
+TraceTau.decl.h: trace-Tau.ci charmxi
+       $(CHARMXI) trace-Tau.ci
 
 TraceControlPoints.def.h: TraceControlPoints.decl.h
 
 
 TraceControlPoints.def.h: TraceControlPoints.decl.h
 
@@ -769,6 +783,31 @@ LIBTRACE_SIMPLE=trace-simple.o
 $(L)/libtrace-simple.a: $(LIBTRACE_SIMPLE)
        $(CHARMC) -o $@ $(LIBTRACE_SIMPLE)
 
 $(L)/libtrace-simple.a: $(LIBTRACE_SIMPLE)
        $(CHARMC) -o $@ $(LIBTRACE_SIMPLE)
 
+libtrace-Tau.o: trace-Tau.C charm++.h charm.h converse.h conv-config.h \
+  conv-autoconfig.h conv-common.h conv-mach.h conv-mach-opt.h \
+  conv-mach-ifort.h pup_c.h conv-cpm.h conv-cpath.h conv-qd.h \
+  conv-random.h conv-lists.h conv-trace.h persistent.h debug-conv.h pup.h \
+  middle.h middle-conv.h cklists.h ckbitvector.h ckstream.h init.h \
+  ckhashtable.h debug-charm.h CkMarshall.decl.h cksection.h ckcallback.h \
+  conv-ccs.h sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h \
+  CkReduction.decl.h cknodegroupreduction.h CkArrayReductionMgr.decl.h \
+  ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
+  cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
+  LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
+  CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
+  convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
+  charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h \
+  ckarrayreductionmgr.h trace.h trace-bluegene.h trace-simple.h \
+  register.h trace-common.h trace-simpleBOC.h TraceSimple.decl.h \
+  TraceSimple.def.h
+       $(CHARMC) -c -I. $(TAU_INCLUDE) $(TAU_DEFS) trace-Tau.C -o libtrace-Tau.o
+
+tau_selective.o: $(TAUROOT)/utils/tau_selective.cpp 
+       $(CHARMC) -I$(TAUROOT)/utils -DTAU_DYNINST $(TAUROOT)/utils/tau_selective.cpp
+
+$(L)/libtrace-Tau.a: $(TAU_TRACE_OBJ) tau_selective.o
+       $(CHARMC) $(TAU_LIBS) -o $@ $(TAU_TRACE_OBJ) tau_selective.o $(TAU_TRACE_OBJ)
 
 LIBTRACE_CONTROLPOINTS=trace-controlPoints.o
 $(L)/libtrace-controlPoints.a: $(LIBTRACE_CONTROLPOINTS)
 
 LIBTRACE_CONTROLPOINTS=trace-controlPoints.o
 $(L)/libtrace-controlPoints.a: $(LIBTRACE_CONTROLPOINTS)
@@ -790,7 +829,9 @@ LIBTRACE_MEMORY=trace-memory.o
 $(L)/libtrace-memory.a: $(LIBTRACE_MEMORY)
        $(CHARMC) -o $@ $(LIBTRACE_MEMORY)
 
 $(L)/libtrace-memory.a: $(LIBTRACE_MEMORY)
        $(CHARMC) -o $@ $(LIBTRACE_MEMORY)
 
-LIBTRACE_ALL=trace-all.o trace-projections.o trace-summary.o trace-simple.o trace-recordreplay.o trace-projector.o traceCore.o traceCoreCommon.o charmProjections.o converseProjections.o machineProjections.o trace-memory.o trace-controlPoints.o
+LIBTRACE_ALL=trace-all.o trace-projections.o trace-summary.o trace-simple.o \
+$(TAU_TRACE_OBJ) trace-recordreplay.o trace-projector.o traceCore.o traceCoreCommon.o charmProjections.o converseProjections.o machineProjections.o trace-memory.o trace-controlPoints.o
+
 $(L)/libtrace-all.a: $(LIBTRACE_ALL)
        $(CHARMC) -o $@ $(LIBTRACE_ALL)
 
 $(L)/libtrace-all.a: $(LIBTRACE_ALL)
        $(CHARMC) -o $@ $(LIBTRACE_ALL)
 
index 8a3ee8a10528b1b19e2ba6279ee8532bd47e18a0..a63faa1560c005ca8a5a23d2ec69bc9d208e3c42 100755 (executable)
@@ -83,6 +83,8 @@ CHARM_SHARED="0"
 BUILD_SHARE="1"
 SWAPGLOBALS="0"
 
 BUILD_SHARE="1"
 SWAPGLOBALS="0"
 
+TAU_MAKEFILE=/expand/home/cheelee/work/tau-charm/tau-2.17.3b2/x86_64/lib/Makefile.tau
+TAU_TRACE_MPI=1
 ####################################################################
 #
 #  Utility routines used below
 ####################################################################
 #
 #  Utility routines used below
@@ -108,7 +110,7 @@ printUsage() {
     echo "        -cpp-option <opt> -ldro-option <opt> -cc-option <opt>"
     echo "        -c++-option <opt> -ld-option <opt> -ld++-option <opt>"
     echo "        -fmoddir <f90mod path>"
     echo "        -cpp-option <opt> -ldro-option <opt> -cc-option <opt>"
     echo "        -c++-option <opt> -ld-option <opt> -ld++-option <opt>"
     echo "        -fmoddir <f90mod path>"
-    echo "        -pg <opt> -build-shared -charm-shared"
+    echo "        -pg <opt> -build-shared -charm-shared -no-trace-mpi"
     echo
     echo "  Charmc compiles C, C++, f77, f90, Converse, Converse++, Charm, "
     echo "and Charm++ programs.  The flags generally match those of cc or f77."
     echo
     echo "  Charmc compiles C, C++, f77, f90, Converse, Converse++, Charm, "
     echo "and Charm++ programs.  The flags generally match those of cc or f77."
@@ -453,6 +455,9 @@ do
                fi
                shift
                ;;
                fi
                shift
                ;;
+       "-no-trace-mpi")
+               TAU_TRACE_MPI=0
+               ;;
        
        "-swapglobal"|"-swapglobals")
                SWAPGLOBALS="1"
        
        "-swapglobal"|"-swapglobals")
                SWAPGLOBALS="1"
@@ -1457,6 +1462,9 @@ for trace in $TRACEMODE; do
     elif test $trace = "simple"
     then
       echo "  extern void _registerTraceSimple();" >> $modInitSrc
     elif test $trace = "simple"
     then
       echo "  extern void _registerTraceSimple();" >> $modInitSrc
+    elif test $trace = "Tau"
+    then
+      echo "  extern void _registerTraceTau();" >> $modInitSrc
     elif test $trace = "controlPoints"
     then
       echo "  extern void _registerTraceControlPoints();" >> $modInitSrc
     elif test $trace = "controlPoints"
     then
       echo "  extern void _registerTraceControlPoints();" >> $modInitSrc
@@ -1482,6 +1490,10 @@ for trace in $TRACEMODE; do
     elif test $trace = "projections"
     then
       echo "  _registerTraceProjections();" >> $modInitSrc
     elif test $trace = "projections"
     then
       echo "  _registerTraceProjections();" >> $modInitSrc
+    elif test $trace = "Tau"
+    then
+                 TRACE_WITH_TAU=1
+      echo "  _registerTraceTau();" >> $modInitSrc
     elif test $trace = "simple"
     then
       echo "  _registerTraceSimple();" >> $modInitSrc
     elif test $trace = "simple"
     then
       echo "  _registerTraceSimple();" >> $modInitSrc
@@ -1528,11 +1540,37 @@ MAKE_LD="0"
 
 CORE_LIBS="-lconv-core -lconv-util $TRACE_OBJ "
 
 
 CORE_LIBS="-lconv-core -lconv-util $TRACE_OBJ "
 
+# get TAU stub makefile variables
+echo "include $TAU_MAKEFILE" > _make.tau.tmp
+echo "print-%:; @echo \$(\$*)" >> _make.tau.tmp
+TAU_LIBS=`make --no-print-directory -f _make.tau.tmp print-TAU_LIBS`
+TAU_MPI_LIBS=`make --no-print-directory -f _make.tau.tmp print-TAU_MPI_LIBS` 
+TAU_MPI_FLIBS=`make --no-print-directory -f _make.tau.tmp print-TAU_MPI_FLIBS`
+#rm _make.tau.tmp
+# done getting TAU variables
+
 case "$LANGUAGE" in
 case "$LANGUAGE" in
+"f90charm")
+       MAKE_LDXX="1"
+       MIDDLE_LIBS="-lck -lconv-cplus-y $CORE_LIBS "
+        CMK_SYSLIBS="$CMK_SYSLIBS -lm"
+       if [ "$TAU_TRACE_MPI" = "1" ]
+       then
+               TAU_LIBS="$TAU_MPI_FLIBS $TAU_LIBS"
+       else
+               TAU_LIBS="$TAU_LIBS"
+       fi
+       ;;
 "charm"|"charm++"|"f90charm"|"ckbluegene"|"bgcharm++")
        MAKE_LDXX="1"
        MIDDLE_LIBS="-lck -lconv-cplus-y $CORE_LIBS "
         CMK_SYSLIBS="$CMK_SYSLIBS -lm"
 "charm"|"charm++"|"f90charm"|"ckbluegene"|"bgcharm++")
        MAKE_LDXX="1"
        MIDDLE_LIBS="-lck -lconv-cplus-y $CORE_LIBS "
         CMK_SYSLIBS="$CMK_SYSLIBS -lm"
+       if [ "$TAU_TRACE_MPI" = "1" ]
+       then
+               TAU_LIBS="$TAU_MPI_LIBS $TAU_LIBS"
+       else
+               TAU_LIBS="$TAU_LIBS"
+       fi
        ;;
 "converse")
        MAKE_LD="1"
        ;;
 "converse")
        MAKE_LD="1"
@@ -1568,7 +1606,14 @@ then
 else
   LANG_LIBS="-L$CHARMLIB -I$CHARMINC $BAL_OBJ $OBJECTFILES $modInitObj $PRE_LIBRARIES"
 fi
 else
   LANG_LIBS="-L$CHARMLIB -I$CHARMINC $BAL_OBJ $OBJECTFILES $modInitObj $PRE_LIBRARIES"
 fi
-ALL_LIBS="$LANG_LIBS $MIDDLE_LIBS $CMK_LIBS $POST_LIBRARIES $CMK_SYSLIBS"
+if [ "$TRACE_WITH_TAU" = 1 ]
+then
+       echo "Linking with the TAU libraries: $TAU_LIBS"
+       ALL_LIBS="$LANG_LIBS $MIDDLE_LIBS $CMK_LIBS $POST_LIBRARIES $CMK_SYSLIBS $TAU_LIBS"
+else
+       ALL_LIBS="$LANG_LIBS $MIDDLE_LIBS $CMK_LIBS $POST_LIBRARIES $CMK_SYSLIBS"
+fi
+Debugf "All libraries are: $ALL_LIBS"
 
 if [ "$BUILD_SHARE" = "1" -a "$CHARM_SHARED" = "1" ]
 then
 
 if [ "$BUILD_SHARE" = "1" -a "$CHARM_SHARED" = "1" ]
 then
index a19b018f4afeb82f6987c12d41321aa71c8cc396..e365a674d76198277d7079008483d290af7c4e91 100644 (file)
@@ -1,3 +1,5 @@
+include $(HOME)/tau2-new/x86_64/lib/Makefile.tau-icpc-mpi-pthread
+
 SRC=../../../src
 CHARMC=../../../bin/charmc $(OPTS)
 
 SRC=../../../src
 CHARMC=../../../bin/charmc $(OPTS)
 
@@ -6,7 +8,7 @@ OBJS = pingpong.o
 all:   cifiles pgm
 
 pgm: $(OBJS)
 all:   cifiles pgm
 
 pgm: $(OBJS)
-       $(CHARMC) -language charm++ -o pgm $(OBJS)
+       $(CHARMC) -language charm++ -o pgm $(OBJS) $(TAU_LIBS)
 
 cifiles: pingpong.ci
        $(CHARMC)  pingpong.ci
 
 cifiles: pingpong.ci
        $(CHARMC)  pingpong.ci