Modifications for TAU tracing in the Charm++ framework.
[charm.git] / src / ck-perf / trace-Tau.C
1 #include <stdlib.h>
2 #include "charm++.h"
3 #include "trace-Tau.h"
4 #include "trace-TauBOC.h"
5 #include "trace-common.h"
6 #include "TAU.h"
7 //#include "tau_selective.cpp"
8 #include "map"
9 #include "stack"
10 #include <string>
11 using namespace std;
12
13 #if defined(_WIN32) && !defined(__CYGWIN__)
14 #include <direct.h>
15 #define CHDIR _chdir
16 #define GETCWD _getcwd
17 #define PATHSEP '\\'
18 #define PATHSEPSTR "\\"
19 #else
20 #include <unistd.h>
21 #define CHDIR chdir
22 #define GETCWD getcwd
23 #define PATHSEP '/'
24 #define PATHSEPSTR "/"
25 #endif
26
27 /*#ifndef PROFILING_ON
28 void TAU_PROFILER_CREATE(void *p, char *n, char *s, taugroup_t t) {
29 dprintf("---> tau
30 create profiler: %s \n", s); }
31
32 void TAU_PROFILER_STOP(void *p) { dprintf("---> tau
33 stop profiler"); }
34
35 void TAU_PROFILER_START(void *p) { dprintf("---> tau
36 start profiler"); }
37
38 void TAU_PROFILE_SET_NODE(int i) { dprintf("---> tau
39 set node"); }
40 #endif
41 */
42
43 #ifdef DEBUG_PROF
44 #define dprintf printf
45 #else // DEBUG_PROF 
46 #define dprintf if (0) printf
47 #endif
48
49 extern bool processFileForInstrumentation(const string& file_name);
50 extern void printExcludeList();
51 extern bool instrumentEntity(const string& function_name);
52 extern int processInstrumentationRequests(char *fname);
53  
54 // Charm++ "processor"(user thread)-private global variable
55 CkpvStaticDeclare(TraceTau*, _trace);
56
57 // This global variable is required for any post-execution 
58 // parallel analysis or activities the trace module might wish to perform.
59 CkGroupID traceTauGID;
60
61 /**
62   For each TraceFoo module, _createTraceFoo() must be defined.
63   This function is called in _createTraces() generated in moduleInit.C
64 */
65
66 void *idle, *comp;
67 //char *name = "default";
68 bool profile = true, snapshotProfiling = false;
69
70 map<const int, void*> events;
71 stack<void*> eventStack;
72 void *EXCLUDED = NULL;
73 void startEntryEvent(int id)
74 {
75   dprintf("---------> starting Entry Event with id: %d\n", id);
76
77   if (events[id] == NULL)
78         {
79           dprintf("-------> create event with id: %d\n", id);
80                 //sprintf(name, "Event %d", id);
81                 if (id == -1)
82                 { /*
83                         char *name = "dummy_thread_ep";
84                         dprintf(" ------> creating event: %s\n", name);
85                         TAU_PROFILER_CREATE(events[id], name, "", TAU_DEFAULT);
86                         dprintf("timer created.\n");
87                         eventStack.push(events[id]);
88                         dprintf(" ------> starting event: %s\n", (char*) name);
89                         TAU_PROFILER_START(eventStack.top());*/
90       //exclude dummy event
91                         dprintf("------> excluding dummy function");
92                         events[id] = EXCLUDED;
93                         eventStack.push(events[id]);
94                 }
95                 else
96                 {
97                         //string check("doFFT(RSFFTMsg* impl_msg)");
98                         //string name_s(_entryTable[id]->name);
99                         //printf("checking name4: %s", _entryTable[id]->name);
100                         //if (check.compare(name_s) != 0)
101                         //{
102                                 char name [500];
103                                 sprintf(name, "%s::%s::%d", _chareTable[_entryTable[id]->chareIdx]->name,
104                                 _entryTable[id]->name, id);
105                                 //should this fuction be excluded from instrumentation?
106                                 if (!instrumentEntity(name))
107                                 {
108                                   //exclude function.
109                                         dprintf("------> excluding function %s\n", name);
110                                         events[id] = EXCLUDED;
111                                         eventStack.push(events[id]);
112                                 }
113                                 else
114                                 {
115                                         dprintf(" ------> creating event: %s\n", name);
116                                         TAU_PROFILER_CREATE(events[id], name, "", TAU_DEFAULT);
117                                         dprintf("timer created.\n");
118                                         eventStack.push(events[id]);
119                                         dprintf("starting event\n");
120                                         dprintf(" ------> starting event: %s\n", (char*) name);
121                                         TAU_PROFILER_START(eventStack.top());
122                           }
123                         dprintf("done.\n");
124                 }
125         }
126         else
127         {
128                 eventStack.push(events[id]);
129           if (events[id] != EXCLUDED)
130                 {
131                         TAU_PROFILER_START(eventStack.top());
132                 }
133         }
134 }
135
136 void stopEntryEvent()
137 {
138   dprintf("stop timer...\n");
139   if (eventStack.top() != EXCLUDED)
140         {
141     TAU_PROFILER_STOP(eventStack.top());
142         }
143         eventStack.pop();
144 }
145
146
147 void _createTraceTau(char **argv)
148 {
149   //TAU_INIT(1, argv);
150   dprintf("arguments:\n");
151   dprintf("[0] = %s, ", argv[0]);
152   dprintf("[1] = %s, ", argv[1]);
153   dprintf("[2] = %s, ", argv[2]);
154         dprintf("\n");
155         string disable = "disable-profiling";
156   if (argv[1] == NULL) { profile = true; }
157         else if (argv[1] == disable) { profile = false; }
158   if (not CkpvAccess(traceOn)) { 
159           dprintf("traceoff selected using snapshot profiling.\n");
160           snapshotProfiling = true; 
161         }
162
163   CkpvInitialize(TraceTau*, _trace);
164   CkpvAccess(_trace) = new TraceTau(argv);
165   CkpvAccess(_traces)->addTrace(CkpvAccess(_trace));
166 }
167
168 TraceTau::TraceTau(char **argv)
169 {
170   if (CkpvAccess(traceOnPe) == 0) return;
171
172   // Process runtime arguments intended for the module
173   CmiGetArgIntDesc(argv,"+TauPar0", &par0, "Fake integer parameter 0");
174   CmiGetArgDoubleDesc(argv,"+TauPar1", &par1, "Fake double parameter 1");
175   //TAU_REGISTER_THREAD();
176         if (profile)
177         {
178                 if (strcmp(CkpvAccess(selective), ""))
179                 {
180                   //printf("select file: %s\n", CkpvAccess(selective));
181                         //processFileForInstrumentation(CkpvAccess(selective));
182                         processInstrumentationRequests(CkpvAccess(selective));
183                         printExcludeList();
184                         if (!instrumentEntity("Main::done(void)::99"))
185                         {
186                           dprintf("selective file working...\n");
187                         }
188                         else
189                           dprintf("selective flile not working...\n");
190                 }
191          
192                 TAU_PROFILER_CREATE(idle, "Idle", "", TAU_DEFAULT);
193                 //TAU_PROFILER_CREATE(entry,name,"", TAU_DEFAULT);
194                 dprintf("before %p\n", comp);  
195           TAU_PROFILER_CREATE(comp, "Main", "", TAU_DEFAULT);
196                 dprintf("after %p\n", comp);  
197
198                 //Need to add an entry timer to the top of the stack because
199                 //traceTauExitFunction calls CkExit() which calls endExecute
200           eventStack.push(EXCLUDED);
201         }
202         else 
203         {
204           dprintf("--> [TAU] creating timers...\n");
205         }
206 }
207
208 void TraceTau::userEvent(int eventID) 
209 {
210   dprintf("[%d] User Point Event id %d encountered\n", CkMyPe(), eventID);
211 }
212
213 void TraceTau::userBracketEvent(int eventID, double bt, double et) {
214   dprintf("[%d] User Bracket Event id %d encountered\n", CkMyPe(), eventID);
215 }
216
217 void TraceTau::creation(envelope *, int epIdx, int num) {
218   dprintf("[%d] Point-to-Point Message for Entry Method id %d sent\n",
219            CkMyPe(), epIdx);
220 }
221
222 void TraceTau::creationMulticast(envelope *, int epIdx, int num, 
223                                     int *pelist) {
224   dprintf("[%d] Multicast Message for Entry Method id %d sent to %d pes\n",
225            CkMyPe(), epIdx, num);
226 }
227
228 void TraceTau::creationDone(int num) {
229   dprintf("[%d] Last initiated send completes\n", CkMyPe());
230 }
231   
232 void TraceTau::messageRecv(char *env, int pe) {
233   dprintf("[%d] Message from pe %d received by scheduler\n", 
234            CkMyPe(), pe);
235 }
236   
237 void TraceTau::beginExecute(CmiObjId *tid)
238 {
239   // CmiObjId is a 4-integer tuple uniquely identifying a migratable
240   //   Charm++ object. Note that there are other non-migratable Charm++
241   //   objects that CmiObjId will not identify.
242   dprintf("[%d] Entry Method invoked using object id\n", CkMyPe());
243                 if (profile) {
244                   startEntryEvent(-1);
245                 }
246                 else
247                 {
248             dprintf("--> [TAU] starting entry timer...\n");
249                 }
250 }
251
252 void TraceTau::beginExecute(envelope *e)
253 {
254   // no message means thread execution
255   if (e == NULL) {
256     dprintf("[%d] Entry Method invoked via thread id %d\n", CkMyPe(),
257              _threadEP);
258                 if (profile) {
259                   startEntryEvent(-1);
260                 }
261                 else
262                 {
263             dprintf("--> [TAU] starting entry timer...\n");
264                 }
265     // Below is what is found in trace-summary.
266     // beginExecute(-1,-1,_threadEP,-1);
267   } else {
268     dprintf("[%d] Entry Method %d invoked via message envelope\n", 
269              CkMyPe(), e->getEpIdx());
270                 if (profile) {
271                   startEntryEvent(e->getEpIdx());
272                 }
273                 else
274                 {
275             dprintf("--> [TAU] starting entry timer...\n");
276                 }
277     // Below is what is found in trace-summary.
278     // beginExecute(-1,-1,e->getEpIdx(),-1);
279   }  
280 }
281
282 void TraceTau::beginExecute(int event,int msgType,int ep,int srcPe, 
283                                int mlen, CmiObjId *idx)
284 {
285   dprintf("[%d] Entry Method %d invoked by parameters\n", CkMyPe(),
286            ep);
287                 if (profile) {
288                   startEntryEvent(ep);
289                 }
290                 else
291                 {
292             dprintf("--> [TAU] starting entry timer...\n");
293                 }
294 }
295
296 void TraceTau::endExecute(void)
297 {
298                 if (profile) {
299                   stopEntryEvent();
300                 }
301                 else
302                 {
303             dprintf("--> [TAU] stoping entry timer...\n");
304                 }
305   dprintf("[%d] Previously executing Entry Method completes\n", CkMyPe());
306 }
307
308 void TraceTau::beginIdle(double curWallTime) {
309   dprintf("[%d] Scheduler has no useful user-work\n", CkMyPe());
310                 if (profile) {
311             TAU_PROFILER_START(idle);
312                 }
313                 else
314                 {
315             dprintf("--> [TAU] starting idle timer...\n");
316                 }
317 }
318
319 void TraceTau::endIdle(double curWallTime) {
320                 if (profile) {
321             TAU_PROFILER_STOP(idle);
322                 }
323                 else
324                 {
325             dprintf("--> [TAU] stopping idle timer...\n");
326                 }
327   dprintf("[%d] Scheduler now has useful user-work\n", CkMyPe());
328 }
329   
330 void TraceTau::beginComputation(void)
331 {
332   dprintf("[%d] Computation Begins\n", CkMyPe());
333   // Code Below shows what trace-summary would do.
334   // initialze arrays because now the number of entries is known.
335   // _logPool->initMem();
336 }
337
338 void TraceTau::endComputation(void)
339 {
340   dprintf("[%d] Computation Ends\n", CkMyPe());
341 }
342
343 void TraceTau::traceBegin(void)
344 {
345                 dprintf("[%d] Tracing Begins\n", CkMyPe());
346                 if (profile) {
347                         dprintf("ptr: %p\n", comp);  
348                         TAU_DB_PURGE();
349       TAU_ENABLE_ALL_GROUPS();
350             TAU_PROFILER_START(comp);
351                 }
352                 else
353                 {
354             dprintf("--> [TAU] starting computation timer...\n");
355                 }
356 }
357 void TraceTau::traceEnd(void)
358 {
359   dprintf("[%d] Tracing Ends\n", CkMyPe());
360                 if (profile){
361                         dprintf("ptr: %p\n", comp);  
362                         //TAU_PROFILER_STOP(comp);
363             TAU_PROFILE_EXIT("tracing complete.");
364       TAU_DISABLE_ALL_GROUPS();
365                 }
366                 else
367                 {
368             dprintf("--> [TAU] stopping computation timer and writing profiles\n");
369                 }
370   dprintf("[%d] Computation Ends\n", CkMyPe());
371 }
372
373 void TraceTau::malloc(void *where, int size, void **stack, int stackSize)
374 {
375   dprintf("[%d] Memory allocation of size %d occurred\n", CkMyPe(), size);
376 }
377
378 void TraceTau::free(void *where, int size) {
379   dprintf("[%d] %d-byte Memory block freed\n", CkMyPe(), size);
380 }
381
382 void TraceTau::traceClose(void)
383 {
384         dprintf("traceClose called.\n");
385   CkpvAccess(_trace)->endComputation();
386   CkpvAccess(_trace)->traceEnd();
387         //TAU_PROFILE_EXIT("closing trace...");
388         //dprintf(" [%d] Exit called \n", CkMyPe());
389         //TAU_PROFILE_EXIT("exiting...");
390   // remove myself from traceArray so that no tracing will be called.
391   CkpvAccess(_traces)->removeTrace(this);
392 }
393
394 extern "C" void traceTauExitFunction() {
395         dprintf("traceTauExitFunction called.\n");
396   // The exit function of any Charm++ module must call CkExit() or
397   // the entire exit process will hang if multiple modules are linked.
398   // FIXME: This is NOT a feature. Something needs to be done about this.
399         //TAU_PROFILE_EXIT("exiting...");
400         //TAU_PROFILE_EXIT("done");
401         //eventStack.push(NULL);
402   CkExit();
403 }
404
405 // Initialization of the parallel trace module.
406 void initTraceTauBOC() {
407   //void *main;
408   dprintf("tracetauboc setting node %d\n", CmiMyPe());
409         if (profile) {
410     TAU_PROFILE_SET_NODE(CmiMyPe());
411         }
412         else
413         {
414           dprintf("---> [TAU] settting node.\n");
415         }
416         //TAU_PROFILER_CREATE(main, "main", "", TAU_DEFAULT);
417         //TAU_PROFILER_START(main);
418 #ifdef __BLUEGENE__
419   if (BgNodeRank()==0) {
420 #else
421     if (CkMyRank() == 0) {
422 #endif
423       registerExitFn(traceTauExitFunction);
424     }
425 }
426
427 #include "TraceTau.def.h"
428
429
430 /*@}*/