Add support for PAPI projections tracing on Cray XE
authorEric Bohm <ebohm@illinois.edu>
Tue, 1 May 2012 18:17:17 +0000 (13:17 -0500)
committerEric Bohm <ebohm@illinois.edu>
Tue, 1 May 2012 18:19:11 +0000 (13:19 -0500)
Specifically set up the 6 counters used in the SPP project as the
default set of counters for Cray XE as these are known to be conflict free.

src/arch/gemini_gni-crayxe/conv-mach-papi.h [new file with mode: 0644]
src/arch/gemini_gni-crayxe/conv-mach-papi.sh [new file with mode: 0644]
src/arch/gemini_gni-crayxe/conv-mach.h
src/ck-perf/trace-projections.C
src/ck-perf/trace-projections.h

diff --git a/src/arch/gemini_gni-crayxe/conv-mach-papi.h b/src/arch/gemini_gni-crayxe/conv-mach-papi.h
new file mode 100644 (file)
index 0000000..e1f21e3
--- /dev/null
@@ -0,0 +1,7 @@
+// Chee Wai 3/11/2004
+// This is really stupid, instead of being able to say "#include <papi.h>"
+// here, I am forced to not say anything and wait for the configure script
+// to decide if the library exists before adding the line to conv-mach-opt.h
+//
+// If this is not the intended idiom, then please enlighten me.
+
diff --git a/src/arch/gemini_gni-crayxe/conv-mach-papi.sh b/src/arch/gemini_gni-crayxe/conv-mach-papi.sh
new file mode 100644 (file)
index 0000000..300cb87
--- /dev/null
@@ -0,0 +1,11 @@
+CMK_USE_PAPI=true
+USE_SPP_PAPI=true
+#you should run module load papi
+PAPI_LIBDIR="/opt/cray/papi/4.3.0.1/perf_events/no-cuda/lib"
+PAPI_INCDIR="/opt/cray/papi/4.3.0.1/perf_events/no-cuda/include"
+CMK_INCDIR="$CMK_INCDIR -I$PAPI_INCDIR"
+CMK_LIBDIR="-L $PAPI_LIBDIR"
+CMK_LD="$CMK_LD -Wl,-rpath,$PAPI_LIBDIR"
+CMK_LDXX="$CMK_LDXX -Wl,-rpath,$PAPI_LIBDIR" 
+CMK_LIBS="$CMK_LIBS -lpapi"
+
index d007e19209bba49568b3e77c72a33e000daff7b5..d531d633286dd0f52919acd7f3b333e333ba2185 100644 (file)
@@ -3,6 +3,9 @@
 
 #define CMK_CRAYXE                                         1
 
+// for cray xe we use the known conflict free counter set from the SPP project
+#define USE_SPP_PAPI                                       1
+
 #define XE6_TOPOLOGY                                      1
 
 /* 1 if the machine has a function called "getpagesize()", 0 otherwise .
index 17de54bccbc0015032f2cbefaa0bce4d746301a3..19997745bb596c59a9c454f50cf32810c100f87e 100644 (file)
@@ -100,7 +100,11 @@ On T3E, we need to have file number control by open/close files only when needed
 #endif //CMK_TRACE_LOGFILE_NUM_CONTROL
 
 #if CMK_HAS_COUNTER_PAPI
+#ifdef USE_SPP_PAPI
+int papiEvents[NUMPAPIEVENTS];
+#else
 int papiEvents[NUMPAPIEVENTS] = { PAPI_L2_DCM, PAPI_FP_OPS };
+#endif
 #endif // CMK_HAS_COUNTER_PAPI
 
 /**
@@ -1069,21 +1073,80 @@ TraceProjections::TraceProjections(char **argv):
 
 #if CMK_SMP
   //PAPI_thread_init has to finish before calling PAPI_create_eventset
-  CmiNodeAllBarrier();
+  #if CMK_SMP_TRACE_COMMTHREAD
+      CmiNodeAllBarrier();
+  #else
+      CmiNodeBarrier();
+  #endif
 #endif
   // PAPI 3 mandates the initialization of the set to PAPI_NULL
   papiEventSet = PAPI_NULL; 
   if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
     CmiAbort("PAPI failed to create event set!\n");
   }
+#ifdef USE_SPP_PAPI
+  //  CmiPrintf("Using SPP counters for PAPI\n");
+  if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
+    papiEvents[0] = PAPI_FP_OPS;
+  }else{
+    if(CmiMyPe()==0){
+      CmiAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
+    }
+  }
+  if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
+    papiEvents[1] = PAPI_TOT_INS;
+  }else{
+    CmiAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
+  }
+  int EventCode;
+  int ret;
+  ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
+  if(PAPI_query_event(EventCode)==PAPI_OK) {
+    papiEvents[2] = EventCode;
+  }else{
+    CmiAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
+  }
+  ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
+  if(PAPI_query_event(EventCode)==PAPI_OK) {
+    papiEvents[3] = EventCode;
+  }else{
+    CmiAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
+    papiEvents[4] = PAPI_L1_DCA;
+  }else{
+    CmiAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
+  }
+  if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
+    papiEvents[5] = PAPI_TOT_CYC;
+  }else{
+    CmiAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
+  }
+#else
+  // just uses { PAPI_L2_DCM, PAPI_FP_OPS } the 2 initialized PAPI_EVENTS
+#endif
   papiRetValue = PAPI_add_events(papiEventSet, papiEvents, NUMPAPIEVENTS);
-  if (papiRetValue != PAPI_OK) {
+  if (papiRetValue < 0) {
     if (papiRetValue == PAPI_ECNFLCT) {
       CmiAbort("PAPI events conflict! Please re-assign event types!\n");
     } else {
+      char error_str[PAPI_MAX_STR_LEN];
+      PAPI_perror(papiRetValue,error_str,PAPI_MAX_STR_LEN);
+      CmiPrintf("PAPI failed with error %s val %d\n",error_str,papiRetValue);
       CmiAbort("PAPI failed to add designated events!\n");
     }
   }
+  if(CkMyPe()==0)
+    {
+      CmiPrintf("Registered %d PAPI counters:",NUMPAPIEVENTS);
+      char nameBuf[PAPI_MAX_STR_LEN];
+      for(int i=0;i<NUMPAPIEVENTS;i++)
+       {
+         PAPI_event_code_to_name(papiEvents[i], nameBuf);
+         CmiPrintf("%s ",nameBuf);
+       }
+      CmiPrintf("\n");
+    }
   memset(papiValues, 0, NUMPAPIEVENTS*sizeof(LONG_LONG_PAPI));
 #endif
 }
index b13dee81764d877a103c56b43ebd93034bd6299f..b2601a4c8aec75c68130dee2a9bbebc94fd50947 100644 (file)
 
 #if CMK_HAS_COUNTER_PAPI
 #include <papi.h>
+#ifdef USE_SPP_PAPI
+#define NUMPAPIEVENTS 6
+#else
 #define NUMPAPIEVENTS 2
 #endif
+#endif
 
 #if CMK_PROJECTIONS_USE_ZLIB
 #include <zlib.h>