add PAPI performance counter in autoPerf module
[charm.git] / src / ck-tune / trace-perf.C
index 8bbe06ef0a334571e7f5d1e96c705a856b4683d0..77eef7b3f72caa0c4191d98c29e458c1a2b4eb84 100644 (file)
@@ -1,3 +1,12 @@
+#if CMK_HAS_COUNTER_PAPI
+#ifdef USE_SPP_PAPI
+int papiEvents[NUMPAPIEVENTS];
+#else
+int papiEvents[NUMPAPIEVENTS] = { PAPI_L1_DCM, PAPI_L2_DCM, PAPI_L3_DCM, PAPI_TLB_DM, PAPI_L1_DCH, PAPI_L2_DCH, PAPI_L3_DCH, PAPI_FP_OPS, PAPI_TOT_CYC };
+#endif
+#endif // CMK_HAS_COUNTER_PAPI
+
+
 CkpvStaticDeclare(TraceAutoPerf*, _trace);
 //-------- group information ---------------------------
 
@@ -14,6 +23,115 @@ TraceAutoPerf::TraceAutoPerf(char **argv)
     resetTimings();
     nesting_level = 0;
     whenStoppedTracing = 0; 
+
+    //PAPI related 
+#if CMK_HAS_COUNTER_PAPI
+  // We initialize and create the event sets for use with PAPI here.
+  int papiRetValue;
+  if(CkMyRank()==0){
+    papiRetValue = PAPI_library_init(PAPI_VER_CURRENT);
+    if (papiRetValue != PAPI_VER_CURRENT) {
+      CmiAbort("PAPI Library initialization failure!\n");
+    }
+#if CMK_SMP
+    if(PAPI_thread_init(pthread_self) != PAPI_OK){
+      CmiAbort("PAPI could not be initialized in SMP mode!\n");
+    }
+#endif
+  }
+
+#if CMK_SMP
+  //PAPI_thread_init has to finish before calling PAPI_create_eventset
+  #if CMK_SMP_TRACE_COMMTHREAD
+      CmiNodeAllBarrier();
+  #else
+      CmiNodeBarrier();
+  #endif
+#endif
+  // PAPI 3 mandates the initialization of the set to PAPI_NULL
+  papiEventSet = PAPI_NULL; 
+  if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
+    CmiAbort("PAPI failed to create event set!\n");
+  }
+#ifdef USE_SPP_PAPI
+  //  CmiPrintf("Using SPP counters for PAPI\n");
+  if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
+    papiEvents[0] = PAPI_FP_OPS;
+  }else{
+    if(CmiMyPe()==0){
+      CmiAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
+    }
+  }
+  if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
+    papiEvents[1] = PAPI_TOT_INS;
+  }else{
+    CmiAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
+  }
+  int EventCode;
+  int ret;
+  ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:ACCESS",&EventCode);
+  if(PAPI_query_event(EventCode)==PAPI_OK) {
+    papiEvents[2] = EventCode;
+  }else{
+    CmiAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:ACCESS doesn't exist on this platform!");
+  }
+  ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
+  if(PAPI_query_event(EventCode)==PAPI_OK) {
+    papiEvents[3] = EventCode;
+  }else{
+    CmiAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
+    papiEvents[4] = PAPI_L1_DCM;
+  }else{
+    CmiAbort("WARNING: PAPI_L1_DCM doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
+    papiEvents[5] = PAPI_TOT_CYC;
+  }else{
+    CmiAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
+    papiEvents[6] = PAPI_L2_DCM;
+  }else{
+    CmiAbort("WARNING: PAPI_L2_DCM doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
+    papiEvents[7] = PAPI_L1_DCA;
+  }else{
+    CmiAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
+  }
+#else
+  // just uses { PAPI_L2_DCM, PAPI_FP_OPS } the 2 initialized PAPI_EVENTS
+#endif
+  papiRetValue = PAPI_add_events(papiEventSet, papiEvents, NUMPAPIEVENTS);
+  if (papiRetValue < 0) {
+    if (papiRetValue == PAPI_ECNFLCT) {
+      CmiAbort("PAPI events conflict! Please re-assign event types!\n");
+    } else {
+      char error_str[PAPI_MAX_STR_LEN];
+      PAPI_perror(error_str);
+      //PAPI_perror(papiRetValue,error_str,PAPI_MAX_STR_LEN);
+      CmiPrintf("PAPI failed with error %s val %d\n",error_str,papiRetValue);
+      CmiAbort("PAPI failed to add designated events!\n");
+    }
+  }
+  if(CkMyPe()==0)
+    {
+      CmiPrintf("Registered %d PAPI counters:",NUMPAPIEVENTS);
+      char nameBuf[PAPI_MAX_STR_LEN];
+      for(int i=0;i<NUMPAPIEVENTS;i++)
+       {
+         PAPI_event_code_to_name(papiEvents[i], nameBuf);
+         CmiPrintf("%s ",nameBuf);
+       }
+      CmiPrintf("\n");
+    }
+  memset(papiValues, 0, NUMPAPIEVENTS*sizeof(LONG_LONG_PAPI));
+  memset(previous_papiValues, 0, NUMPAPIEVENTS*sizeof(LONG_LONG_PAPI));
+#endif
+
+
     if (CkpvAccess(traceOnPe) == 0) return;
 }
 
@@ -48,6 +166,9 @@ void TraceAutoPerf::resetAll(){
     currentSummary->numBytes = 0;
     currentSummary->commTime = 0;
     currentSummary->objLoadMax = 0;
+#if CMK_HAS_COUNTER_PAPI
+    memcpy(previous_papiValues, papiValues, sizeof(LONG_LONG_PAPI)*NUMPAPIEVENTS);
+#endif
 }
 
 void TraceAutoPerf::traceBegin(void){
@@ -147,8 +268,36 @@ void TraceAutoPerf::endIdle(double curWallTime) {
     lastEvent =  -1;
 }
 
-void TraceAutoPerf::beginComputation(void) { }
-void TraceAutoPerf::endComputation(void) { }
+void TraceAutoPerf::beginComputation(void) {
+#if CMK_HAS_COUNTER_PAPI
+  // we start the counters here
+  if (PAPI_start(papiEventSet) != PAPI_OK) {
+    CmiAbort("PAPI failed to start designated counters!\n");
+  }
+#endif
+
+}
+
+void TraceAutoPerf::endComputation(void) { 
+#if CMK_HAS_COUNTER_PAPI
+  // we stop the counters here. A silent failure is alright since we
+  // are already at the end of the program.
+  if (PAPI_stop(papiEventSet, papiValues) != PAPI_OK) {
+    CkPrintf("Warning: PAPI failed to stop correctly!\n");
+  }
+  //else 
+  //{
+  //    char eventName[PAPI_MAX_STR_LEN];
+  //    for (int i=0;i<NUMPAPIEVENTS;i++) {
+  //        PAPI_event_code_to_name(papiEvents[i], eventName);
+  //        CkPrintf(" EVENT  %s   counter   %lld \n", eventName, papiValues[i]);
+  //    }
+  //}
+  // NOTE: We should not do a complete close of PAPI until after the
+  // sts writer is done.
+#endif
+
+}
 
 void TraceAutoPerf::malloc(void *where, int size, void **stack, int stackSize)
 {