Blue Gene/Q changes for charm++.
authorSameer Kumar <sameerk@us.ibm.com>
Wed, 16 Feb 2011 22:33:28 +0000 (16:33 -0600)
committerSameer Kumar <sameerk@us.ibm.com>
Wed, 16 Feb 2011 22:33:28 +0000 (16:33 -0600)
src/arch/mpi-bluegeneq/cc-xlc.h [new file with mode: 0644]
src/arch/mpi-bluegeneq/cc-xlc.sh [new file with mode: 0644]
src/arch/mpi-bluegeneq/charmrun [new file with mode: 0755]
src/arch/mpi-bluegeneq/conv-mach.h [new file with mode: 0644]
src/arch/mpi-bluegeneq/conv-mach.sh [new file with mode: 0644]
src/conv-core/convcore.c
src/conv-core/cputopology.C
src/langs/bluegene/blue_defs.h
src/util/uJcontext.c

diff --git a/src/arch/mpi-bluegeneq/cc-xlc.h b/src/arch/mpi-bluegeneq/cc-xlc.h
new file mode 100644 (file)
index 0000000..9effcb3
--- /dev/null
@@ -0,0 +1 @@
+/* no defines needed. */
diff --git a/src/arch/mpi-bluegeneq/cc-xlc.sh b/src/arch/mpi-bluegeneq/cc-xlc.sh
new file mode 100644 (file)
index 0000000..3011e32
--- /dev/null
@@ -0,0 +1,31 @@
+XLC_TYPICAL_PRE=/opt/ibmcmp/vacpp/bg/9.0/
+XLC_PRE=$XLC_TYPICAL_PRE
+
+XLC_TYPICAL_POST=bin/bg
+XLC_POST=$XLC_TYPICAL_POST
+
+# if no floor set, use typical floor path
+if test -n "$BGQ_XLC_PRE"
+then
+  XLC_PRE=$BGQ_XLC_PRE
+fi
+
+XLC_F=$XLC_PRE/xlf/bg/11.1/
+CMK_CC="$XLC_PRE/${XLC_POST}xlc -qcpluscmt -qhalt=e $BGQ_INC -qminimaltoc"
+CMK_CXX="$XLC_PRE/${XLC_POST}xlC -qhalt=e $BGQ_INC -qminimaltoc"
+CMK_LD="$CMK_CC"
+CMK_LDXX="$CMK_CXX"
+CMK_CF77="$XLC_F/${XLC_POST}xlf "
+CMK_CF90="$XLC_F/${XLC_POST}xlf90  -qsuffix=f=f90" 
+CMK_CF90_FIXED="$XLC_PRE/xlf/8.1/${XLC_POST}xlf90 " 
+CMK_C_OPTIMIZE='-O3 -Q  -qminimaltoc'
+CMK_CXX_OPTIMIZE='-O3 -Q -qminimaltoc'
+CMK_AR='ar cq'
+CMK_NM='nm '
+CMK_QT="aix"
+#CMK_NATIVE_CC="/opt/ibmcmp/vacpp/bg/9.0/bin/xlc"
+#CMK_NATIVE_CXX="/opt/ibmcmp/vacpp/bg/9.0/bin/xlC"
+CMK_NATIVE_LD="$CMK_NATIVE_CC"
+CMK_NATIVE_LDXX="$CMK_NATIVE_CXX"
+CMK_RANLIB="ranlib"
+CMK_F90LIBS="-L$XLC_F/lib -lxlf90 -lxlopt -lxl -lxlfmath"
diff --git a/src/arch/mpi-bluegeneq/charmrun b/src/arch/mpi-bluegeneq/charmrun
new file mode 100755 (executable)
index 0000000..93058a0
--- /dev/null
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Conv-host for Blue Gene:
+#  Translates +pN-style conv-host options into 
+# mpirun -npN options.
+#  Copies executable to $BL_HOME directory before running.
+#
+# You MUST have a copy of MMCS running in another terminal window.
+#
+
+if [ -z "$BL_HOME" ]
+then
+       echo "You must set the BL_HOME environment variable"
+       exit 1
+fi
+
+firstArg=""
+args=""
+pes=1
+
+while [ $# -gt 0 ]
+do
+       case $1 in
+       +p)
+               pes=$2
+               shift
+               ;;
+       +p*)
+               pes=`echo $1 | awk '{print substr($1,3)}'`
+               ;;
+       *) 
+               [ -z "$firstArg" ] && firstArg="$1"
+               args="$args $1"
+               ;;
+       esac
+       shift
+done
+
+if [ $firstArg -nt $BL_HOME/$firstArg ]
+then
+  echo "Copying $firstArg to $BL_HOME..."
+  cp $firstArg $BL_HOME 
+  [ $? -ne 0 ] && exit 1
+fi
+
+echo "Running on $pes processors: $args"
+exec /BlueLight/bin/CIORUN -p $pes $args
diff --git a/src/arch/mpi-bluegeneq/conv-mach.h b/src/arch/mpi-bluegeneq/conv-mach.h
new file mode 100644 (file)
index 0000000..c4d9875
--- /dev/null
@@ -0,0 +1,77 @@
+#ifndef _CONV_MACH_H
+#define _CONV_MACH_H
+
+#define CMK_NO_OUTSTANDING_SENDS                          0
+
+#define CMK_64BIT                                          1
+
+//#define CMK_MEMORY_PREALLOCATE_HACK                        1
+
+#define CMK_CONVERSE_MPI                                   1
+
+#define CMK_NO_SOCKETS                                    1
+
+#define CMK_DEFAULT_MAIN_USES_COMMON_CODE                  1
+
+#define CMK_GETPAGESIZE_AVAILABLE                          1
+
+#define CMK_IS_HETERO                                      0
+
+#define CMK_MALLOC_USE_GNU_MALLOC                          0
+#define CMK_MALLOC_USE_OS_BUILTIN                          1
+
+#define CMK_MEMORY_PAGESIZE                                8192
+#define CMK_MEMORY_PROTECTABLE                             1
+
+#define CMK_NODE_QUEUE_AVAILABLE                           0
+
+#define CMK_SHARED_VARS_EXEMPLAR                           0
+#define CMK_SHARED_VARS_UNAVAILABLE                        1
+#define CMK_SHARED_VARS_UNIPROCESSOR                       0
+
+#define CMK_SIGNAL_NOT_NEEDED                              0
+#define CMK_SIGNAL_USE_SIGACTION                           0
+#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
+
+#define CMK_SYNCHRONIZE_ON_TCP_CLOSE                       0
+
+#define CMK_THREADS_USE_CONTEXT                            0
+#define CMK_THREADS_USE_JCONTEXT                           1
+#define CMK_THREADS_USE_PTHREADS                           0
+#define CMK_THREADS_ARE_WIN32_FIBERS                       0
+
+#define CMK_THREADS_REQUIRE_NO_CPV                         0
+
+#define CMK_TIMER_USE_GETRUSAGE                            0
+#define CMK_TIMER_USE_SPECIAL                              0
+#define CMK_TIMER_USE_TIMES                                0
+// This needs to be compiled with gcc only
+#define CMK_TIMER_USE_BLUEGENEQ                                   1
+
+#define CMK_TYPEDEF_INT2 short
+#define CMK_TYPEDEF_INT4 int
+#define CMK_TYPEDEF_INT8 long long
+#define CMK_TYPEDEF_UINT2 unsigned short
+#define CMK_TYPEDEF_UINT4 unsigned int
+#define CMK_TYPEDEF_UINT8 unsigned long long
+#define CMK_TYPEDEF_FLOAT4 float
+#define CMK_TYPEDEF_FLOAT8 double
+
+#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   1
+#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     0
+
+
+#define CMK_WEB_MODE                                       1
+#define CMK_DEBUG_MODE                                     0
+
+#define CMK_LBDB_ON                                       1
+
+#undef CMK_CCS_AVAILABLE
+#define CMK_CCS_AVAILABLE                                 0
+
+#define CMK_BLUEGENEQ                                      1
+
+#define CMK_NO_ISO_MALLOC                                  1
+
+#endif
+
diff --git a/src/arch/mpi-bluegeneq/conv-mach.sh b/src/arch/mpi-bluegeneq/conv-mach.sh
new file mode 100644 (file)
index 0000000..e5eb471
--- /dev/null
@@ -0,0 +1,62 @@
+BGQ_TYPICAL_FLOOR=/bgsys/drivers/ppcfloor
+
+# if no floor set, use typical floor path
+if test -z "$BGQ_FLOOR"
+then
+  BGQ_FLOOR=$BGQ_TYPICAL_FLOOR
+fi
+
+# if no install path (for experimental) set, use floor
+if test -z "$BGQ_INSTALL"
+then
+  BGQ_INSTALL=$BGQ_TYPICAL_FLOOR
+fi
+
+BGQ_BIN=$BGQ_FLOOR/gnu-linux/bin
+BGQ_INC="-I$BGQ_INSTALL/comm/gcc/include -I$BGQ_INSTALL/spi/include -I$BGQ_INSTALL -I$BGQ_INSTALL/spi/include/kernel/cnk/"
+
+BGQ_LIB="-L$BGQ_INSTALL/comm/gcc/lib -lmpich -lopa -ldl -L$BGQ_INSTALL/comm/sys/lib -lpami -L$BGQ_INSTALL/spi/lib -lSPI -lSPI_cnk -lpthread -lrt"
+
+# test if compiler binary present
+if test ! -x $BGQ_BIN/powerpc64-bgq-linux-g++
+then
+ echo "ERROR: Invalid BGQ_INSTALL or BGQ_FLOOR, C/C++ compiler missing"
+ exit 1
+fi
+
+OPTS_CPP="$OPTS_CPP"
+GCC_OPTS="-Wno-deprecated -mminimal-toc $BGQ_INC"
+OPTS_LD="$OPTS_LD"
+
+CMK_CPP_CHARM="$BGQ_BIN/powerpc64-bgq-linux-cpp -P"
+CMK_CPP_C="$BGQ_BIN/powerpc64-bgq-linux-cpp -E "
+CMK_CXX="$BGQ_BIN/powerpc64-bgq-linux-g++ $GCC_OPTS "
+CMK_GCXX="$BGQ_BIN/powerpc64-bgq-linux-g++ $GCC_OPTS "
+CMK_CC="$BGQ_BIN/powerpc64-bgq-linux-gcc $GCC_OPTS "
+CMK_CXXPP="$BGQ_BIN/powerpc64-bgq-linux-g++ -E "
+CMK_CF77="$BGQ_BIN/powerpc64-bgq-linux-gfortran "
+CMK_CF90='f90'
+CMK_RANLIB="$BGQ_BIN/powerpc64-bgq-linux-ranlib "
+CMK_AR="$BGQ_BIN/powerpc64-bgq-linux-ar q "
+CMK_SYSLIBS="$BGQ_LIB"
+CMK_LIBS='-lckqt'
+#CMK_LD="$BGQ_INSTALL/comm/gcc/bin/mpicc"
+#CMK_LDXX="$BGQ_INSTALL/comm/gcc/bin/mpicxx"
+CMK_LD="$CMK_CC"
+CMK_LDXX="$CMK_CXX"
+CMK_LD_LIBRARY_PATH="-Wl,-rpath,$CHARMLIBSO/"
+#CMK_SEQ_LIBS=''
+#CMK_SEQ_CC="$BGQ_BIN/powerpc64-bgq-linux-gcc -Wno-deprecated "
+#CMK_SEQ_LD="$CMK_SEQ_CC"
+#CMK_SEQ_CXX="$BGQ_BIN/powerpc64-bgq-linux-g++ -Wno-deprecated "
+#CMK_SEQ_LDXX="$CMK_SEQ_CXX"
+CMK_NATIVE_CC='gcc '
+CMK_NATIVE_LD='gcc '
+CMK_NATIVE_CXX='g++ -Wno-deprecated '
+CMK_NATIVE_LDXX='g++'
+CMK_F90LIBS='-lf90math -lfio -lU77 -lf77math '
+CMK_MOD_NAME_ALLCAPS=1
+CMK_MOD_EXT="mod"
+CMK_F90_USE_MODDIR=1
+CMK_F90_MODINC="-p"
+CMK_QT="generic64"
index e49c18d1592abc1ae9e920b8e08f795da499a53a..edc98e5bf06dd7c96e04a90b0d0ff8d313fbc03e 100644 (file)
@@ -1076,71 +1076,62 @@ double CmiTimer()
 
 void CmiTimerInit() {}
 
-#if 0
-#include "common/bgp_personality.h"
-#include <spi/bgp_SPI.h>
+#include "dcmf.h"
 
-#define SPRN_TBRL 0x10C  /* Time Base Read Lower Register (user & sup R/O) */
-#define SPRN_TBRU 0x10D  /* Time Base Read Upper Register (user & sup R/O) */
-#define SPRN_PIR  0x11E  /* CPU id */
+double CmiWallTimer () {
+  return DCMF_Timer();
+}
 
-static inline unsigned long long BGPTimebase(void)
+double CmiCpuTimer()
 {
-  unsigned volatile u1, u2, lo;
-  union
-  {
-    struct { unsigned hi, lo; } w;
-    unsigned long long d;
-  } result;
-                                                                         
-  do {
-    asm volatile ("mfspr %0,%1" : "=r" (u1) : "i" (SPRN_TBRU));
-    asm volatile ("mfspr %0,%1" : "=r" (lo) : "i" (SPRN_TBRL));
-    asm volatile ("mfspr %0,%1" : "=r" (u2) : "i" (SPRN_TBRU));
-  } while (u1!=u2);
-                                                                         
-  result.w.lo = lo;
-  result.w.hi = u2;
-  return result.d;
+  return CmiWallTimer();
 }
 
-static unsigned long long inittime_wallclock = 0;
+double CmiTimer()
+{
+  return CmiWallTimer();
+}
+#endif
+
+
+#if CMK_TIMER_USE_BLUEGENEQ  /* This module just compiles with GCC charm. */
+
+CpvStaticDeclare(unsigned long, inittime);
 CpvStaticDeclare(double, clocktick);
 
 int CmiTimerIsSynchronized()
 {
-  return 0;
+  return 1;
 }
 
+#include "hwi/include/bqc/A2_inlines.h"
+#include "spi/include/kernel/process.h"
+
 void CmiTimerInit()
 {
-  _BGP_Personality_t dst;
   CpvInitialize(double, clocktick);
-  int size = sizeof(_BGP_Personality_t);
-  rts_get_personality(&dst, size);
+  CpvInitialize(unsigned long, inittime);
+
+  Personality_t  pers;
+  Kernel_GetPersonality(&pers, sizeof(pers));
+  uint32_t clockMhz = pers.Kernel_Config.FreqMHz;
+  CpvAccess(clocktick) = 1.0 / (clockMhz * 1e6); 
 
-  CpvAccess(clocktick) = 1.0 / (dst.Kernel_Config.FreqMHz * 1e6);
+  fprintf(stderr, "Blue Gene/Q running at clock speed of %d Mhz\n", clockMhz);
 
   /* try to synchronize calling barrier */
   CmiBarrier();
   CmiBarrier();
   CmiBarrier();
 
-  inittime_wallclock = BGPTimebase (); 
+  CpvAccess(inittime) = GetTimeBase (); 
 }
 
 double CmiWallTimer()
 {
   unsigned long long currenttime;
-  currenttime = BGPTimebase();
-  return CpvAccess(clocktick)*(currenttime-inittime_wallclock);
-}
-#endif
-
-#include "dcmf.h"
-
-double CmiWallTimer () {
-  return DCMF_Timer();
+  currenttime = GetTimeBase();
+  return CpvAccess(clocktick)*(currenttime-CpvAccess(inittime));
 }
 
 double CmiCpuTimer()
index 5c12ab1e35b17838913d8ab133efb0602e3613ac..9c8f88d28288839899d1daa018a021e224a809e4 100644 (file)
@@ -480,6 +480,8 @@ extern "C" void CmiInitCPUTopology(char **argv)
   }
   CmiNodeAllBarrier();
   return;
+#elif CMK_BLUEGENEQ
+  return;
 #elif CMK_CRAYXT
   if(CmiMyRank() == 0) {
     int numPes = cpuTopo.numPes = CmiNumPes();
index 829198fb3145fd2922c6a1d9dd09187a9a1c4cf7..fb1d9d2f35431717512c4307cf10a6c552d45599 100644 (file)
@@ -3,7 +3,7 @@
 
 #define BLUEGENE_TIMING        1
 
-#if CMK_HAS_VALUES_H
+#if (CMK_HAS_VALUES_H && (!CMK_BLUEGENEQ))
 #   include <values.h>
 #   define INVALIDTIME  MAXDOUBLE
 #   define CMK_MAXINT   MAXINT
index 6cc82390c267779a31f5aa1d462af340c379f447..2516fdc87e97574a9c341a346aae5c61ffa5c90a 100644 (file)
@@ -62,6 +62,8 @@ int setJcontext (const uJcontext_t *u)
 {
        register uJcontext_t *mu=(uJcontext_t *)u;
        
+       CmiAssert (mu != NULL);
+
        /* Call the user's swap function if needed (brings in thread stack) */
        if (mu->uc_swap) mu->uc_swap(mu);
        
@@ -89,6 +91,7 @@ int setJcontext (const uJcontext_t *u)
                /* FIXME: only if stack grows down (they all do...) */
                new_sp+=mu->uc_stack.ss_size-caller_distance;
                
+#ifndef CMK_BLUEGENEQ
                VERBOSE( printf("About to switch to stack %p ",new_sp); printStack(); )
                if (1) { /* change to new stack */
 #ifdef _MSC_VER 
@@ -115,15 +118,35 @@ int setJcontext (const uJcontext_t *u)
 #endif
                }
                VERBOSE( printf("After alloca"); printStack(); )
-               
                /* Call the user function for the thread */
                mu_fn(mu->_uc_args[0],mu->_uc_args[1]);
-               
+
                /* Back from user function-- jump to next thread */
                if (mu->uc_link!=0)
-                       setJcontext(mu->uc_link);
+                       setJcontext(mu->uc_link);
                else
                        threadFatal("uc_link not set-- thread should never return");
+#else
+               //Start the thread by changing the stack pointer and calling the start function
+               uint64_t startiar = *((uint64_t*)mu_fn);
+               //uint64_t sp  = ((uint64_t)(ptr->stackptr) + ptr->stacksize - 1024) & ~(0x1f);
+               asm volatile("mr 3, %0;"
+                            "mtlr 3;"
+                            "mr 1, %1;"
+                            "mr 3, %2;"
+                            "mr 4, %3;"
+                            "blr;"
+                            : : "r" (startiar), "r" (new_sp), "r" (mu->_uc_args[0]), "r" (mu->_uc_args[1]) : "r1", "r3", "r4", "memory");
+               
+               //mu->uc_link cannot be null. Call setJcontext to start next thread or return to master
+               startiar = *((uint64_t*)setJcontext);
+               asm volatile("mr 3, %0;"
+                            "mtlr 3;"
+                            "mr 3, %1;"
+                            "blr;"
+                            : : "r" (startiar), "r" (mu->uc_link) : "r3", "memory");
+#endif 
+               
        }
        return 0;
 }