clean up tracing comm thread
authorGengbin Zheng <gzheng@illinois.edu>
Sun, 15 Jan 2012 04:30:17 +0000 (20:30 -0800)
committerGengbin Zheng <gzheng@illinois.edu>
Sun, 15 Jan 2012 04:30:17 +0000 (20:30 -0800)
1. make TRACE_COMM)* macros to simplify code
2. fix tracing when converse messages are encountered
3. fix srcpe on comm thread to pretend that the msg is sent now from comm thread (need to be tested abd verify in Projections)

src/arch/gemini_gni/machine.c
src/ck-perf/trace-common.C
src/ck-perf/trace-projections.C
src/ck-perf/trace.h
src/conv-core/conv-trace.h

index 83cc9fd53b231dc4341ea014f8fc8dee11de2b61..6113c93223dd27df30f3f4fe40db407164ee1960 100644 (file)
@@ -863,7 +863,7 @@ static gni_return_t send_smsg_message(int destNode, void *header, int size_heade
         status = GNI_SmsgSendWTag(ep_hndl_array[destNode], header, size_header, msg, size, 0, tag);
         if(status == GNI_RC_SUCCESS)
         {
-#if CMK_SMP_TRACE_COMMTHREAD && 1 
+#if CMK_SMP_TRACE_COMMTHREAD
             if(tag == SMALL_DATA_TAG || tag == LMSG_INIT_TAG)
             { 
                 START_EVENT();
@@ -871,10 +871,7 @@ static gni_return_t send_smsg_message(int destNode, void *header, int size_heade
                     real_data = (char*)msg; 
                 else 
                     real_data = (char*)(((CONTROL_MSG*)msg)->source_addr);
-                traceBeginCommOp( real_data);
-                traceChangeLastTimestamp(CpvAccess(projTraceStart));
-                traceSendMsgComm(real_data);
-                traceEndCommOp(real_data);
+                TRACE_COMM_CREATION(CpvAccess(projTraceStart), real_data);
             }
 #endif
 #if PRINT_SYH
@@ -1254,12 +1251,10 @@ static void PumpNetworkSmsg()
                 msg_nbytes = CmiGetMsgSize(header);
                 msg_data    = CmiAlloc(msg_nbytes);
                 memcpy(msg_data, (char*)header, msg_nbytes);
-#if CMK_SMP_TRACE_COMMTHREAD  
-                traceBeginCommOp(msg_data);
-                traceChangeLastTimestamp(CpvAccess(projTraceStart));
-                traceEndCommOp(msg_data);
-#endif
                 handleOneRecvedMsg(msg_nbytes, msg_data);
+#if CMK_SMP_TRACE_COMMTHREAD
+                TRACE_COMM_RECV(CpvAccess(projTraceStart), msg_data);
+#endif
                 break;
             }
             case LMSG_INIT_TAG:
@@ -1600,23 +1595,19 @@ static void PumpLocalRdmaTransactions()
                     START_EVENT();
                     CmiAssert(SIZEFIELD((void*)(tmp_pd->local_addr)) <= tmp_pd->length);
                     DecreaseMsgInFlight((void*)tmp_pd->local_addr);
+                    handleOneRecvedMsg(tmp_pd->length, (void*)tmp_pd->local_addr); 
 #if CMK_SMP_TRACE_COMMTHREAD
-                    traceBeginCommOp( (void*)tmp_pd->local_addr);
-                    traceChangeLastTimestamp(CpvAccess(projTraceStart));
-                    traceEndCommOp((void*)tmp_pd->local_addr);
+                    TRACE_COMM_RECV(CpvAccess(projTraceStart), (void*)tmp_pd->local_addr);
 #endif
-                    handleOneRecvedMsg(tmp_pd->length, (void*)tmp_pd->local_addr); 
                 }else if (tmp_pd->first_operand <= ONE_SEG) {
                     START_EVENT();
 #if PRINT_SYH
                     printf("Pipeline msg done [%d]\n", myrank);
-#endif
-#if CMK_SMP_TRACE_COMMTHREAD 
-                    traceBeginCommOp( (void*)tmp_pd->local_addr-(tmp_pd->cqwrite_value-1)*ONE_SEG);
-                    traceChangeLastTimestamp(CpvAccess(projTraceStart));
-                    traceEndCommOp((void*)tmp_pd->local_addr-(tmp_pd->cqwrite_value-1)*ONE_SEG);
 #endif
                     handleOneRecvedMsg(tmp_pd->length + (tmp_pd->cqwrite_value-1)*ONE_SEG, (void*)tmp_pd->local_addr-(tmp_pd->cqwrite_value-1)*ONE_SEG); 
+#if CMK_SMP_TRACE_COMMTHREAD
+                    TRACE_COMM_RECV(CpvAccess(projTraceStart), (void*)tmp_pd->local_addr-(tmp_pd->cqwrite_value-1)*ONE_SEG);
+#endif
                 }
                     SendRdmaMsg();
             }
index 2e71d156c36751bb07757bbcf2ebd1f162d513c0..c02d5668a2d3280f72f0a482d5a1eee4b6b3bc2d 100644 (file)
@@ -668,10 +668,13 @@ void traceEndFuncIndexProj(int idx){
 
 #if CMK_SMP_TRACE_COMMTHREAD
 extern "C"
-void traceBeginCommOp(char *msg){
+int traceBeginCommOp(char *msg){
 #if CMK_TRACE_ENABLED
-  if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg))
+  if (CpvAccess(traceOn) && CkpvAccess(_traces) && CkIsCharmMessage(msg)) {
     CkpvAccess(_traces)->beginExecute(msg);
+    return 1;
+  }
+  return 0;
 #endif
 }
 
index 885ca6b9bff7e1fddb19d9c5786080410b16213d..6f21fd3f86041ff445fb6af5c84cb76c4f2376d3 100644 (file)
@@ -1252,19 +1252,20 @@ void TraceProjections::creation(envelope *e, int ep, int num)
   }
 }
 
+//This function is only called from a comm thread in SMP mode. 
 void TraceProjections::creation(char *msg)
 {
 #if CMK_SMP_TRACE_COMMTHREAD
-       //This function is only called from a comm thread
-       //in SMP mode. 
         // msg must be a charm message
        envelope *e = (envelope *)msg;
        int ep = e->getEpIdx();
         if(ep==0) return;
         int num = _entryTable.size();
         CmiAssert(ep < num);
-       if(_entryTable[ep]->traceEnabled)
+       if(_entryTable[ep]->traceEnabled) {
                creation(e, ep, 1);
+                e->setSrcPe(CkMyPe());              // pretend I am the sender
+        }
 #endif
 }
 
index 87dd6dc2feea8f95c1cfe2ebb42bc8535a588f82..f8fa333c1b8690044f84efb64974967f8a2e6c5f 100644 (file)
@@ -1,9 +1,3 @@
-/*****************************************************************************
- * $Source$
- * $Author$
- * $Date$
- * $Revision$
- *****************************************************************************/
 
 #ifndef _TRACE_H
 #define _TRACE_H
@@ -343,7 +337,6 @@ extern "C" {
 #define _TRACE_MALLOC(where, size, stack, stackSize) _TRACE_ONLY(CkpvAccess(_traces)->malloc(where,size,stack,stackSize))
 #define _TRACE_FREE(where, size) _TRACE_ONLY(CkpvAccess(_traces)->free(where, size))
 
-
 #include "trace-bluegene.h"
 
 #endif
index d6b5ee928e8a99c1b15ef3cc1b31b66a2784a5cb..a36cca25d5535e45d51ae804b65c3b5e9bc6d918 100644 (file)
@@ -36,7 +36,7 @@ int  traceRegisterUserEvent(const char*, int e
 );
 
 #if CMK_SMP_TRACE_COMMTHREAD
-void traceBeginCommOp(char *msg);
+int  traceBeginCommOp(char *msg);
 void traceEndCommOp(char *msg);
 void traceSendMsgComm(char *msg);
 #endif
@@ -69,4 +69,22 @@ CpvExtern(int, traceOn);
 
 int  traceAvailable();
 
+/* Comm thread tracing */
+#if CMK_SMP_TRACE_COMMTHREAD
+#define  TRACE_COMM_CREATION(time, msg)   \
+                    if (traceBeginCommOp(msg)) {   \
+                      traceChangeLastTimestamp(time);    \
+                      traceSendMsgComm(msg);   \
+                      traceEndCommOp(msg);    \
+                    }
+#define  TRACE_COMM_RECV(time, msg)   \
+                    if (traceBeginCommOp(msg)) {   \
+                      traceChangeLastTimestamp(time);    \
+                      traceEndCommOp(msg);    \
+                    }
+#else
+#define TRACE_COMM_CREATION(time, msg)
+#define TRACE_COMM_RECV(time, msg)
+#endif
+
 #endif