shmem config files for crayXT
authorGengbin Zheng <gzheng@illinois.edu>
Wed, 21 Oct 2009 23:33:30 +0000 (23:33 +0000)
committerGengbin Zheng <gzheng@illinois.edu>
Wed, 21 Oct 2009 23:33:30 +0000 (23:33 +0000)
src/arch/shmem-crayxt/charmrun [new file with mode: 0755]
src/arch/shmem-crayxt/conv-mach.h [new file with mode: 0644]
src/arch/shmem-crayxt/conv-mach.sh [new file with mode: 0644]

diff --git a/src/arch/shmem-crayxt/charmrun b/src/arch/shmem-crayxt/charmrun
new file mode 100755 (executable)
index 0000000..c768c1e
--- /dev/null
@@ -0,0 +1,283 @@
+#!/bin/sh
+#
+# Conv-host for MPI:
+#  Translates +pN-style conv-host options into 
+# mpirun -npN options.
+
+args=""
+pes=1
+machinefile=""
+
+while [ $# -gt 0 ]
+do
+       case $1 in
+       +ppn)
+               args=$args" +ppn "$2
+               shift
+               ;;
+       +ppn*)
+               args=$args" "$1
+               ;;
+       +p)
+               pes=$2
+               shift
+               ;;
+       +p*)
+               pes=`echo $1 | awk '{print substr($1,3)}'`
+               ;;
+        -machinefile)
+               machinefile=$2
+               args=" "$1" "$2" "$args
+               shift
+               ;;
+       *) 
+               args=$args" "$1
+               ;;
+       esac
+       shift
+done
+
+
+printf "\nRunning on $pes processors: $args\n"
+
+
+if [ -n "$PBS_NODEFILE" ]
+then
+# we are in a job shell
+  aprun=`which aprun 2>/dev/null`
+  if test -n "$aprun"
+  then
+    echo aprun -n $pes $args
+    $aprun -n $pes $args
+  else
+    mpirun_cmd=`which mpirun 2>/dev/null`
+    if test -n "$mpirun_cmd"
+    then
+      if echo $mpirun_cmd | grep 'mvapich2'  > /dev/null 2>/dev/null
+      then
+        # if daemon not started, start it
+        if ! mpdtrace > /dev/null 2>/dev/null
+        then
+          mvapich2-start-mpd
+        fi
+        mpirun -np $pes $args
+        #    mpdallexit
+      else   # normal case
+        test -z "$machinefile" && args=-machinefile" "$PBS_NODEFILE" "$args
+        echo mpirun -np $pes $args
+        mpirun -np $pes $args
+      fi
+    else
+      echo "Charmrun> can not locate mpirun in order to run the program."
+      exit 1
+    fi
+  fi
+elif [ -n "$LSB_HOSTS" ]
+then
+# Tungsten
+  echo cmpirun -lsf -poll -no_smp -gm_long 200000 $args 
+  cmpirun -lsf -poll -no_smp -gm_long 200000 $args 
+elif [ -n "$PBS_QUEUE" -o -n "$LSF_QUEUE" ]
+then
+# Interactive mode: create, and submit a batch job
+        script="charmrun_script.$$.sh"
+        indir=`pwd`
+        output="$indir/charmrun_script.$$.stdout"
+        result="$indir/charmrun_script.$$.result"
+       rm -f $result
+# Some machine specific 
+       USE_LSF=0
+# 10 minutes   
+       walllimit=10
+       queue_stat=qstat
+       queue_qsub=qsub
+       queue_kill=qdel
+       hostname=`hostname`
+       case "$hostname" in
+       turing*.turing.uiuc.edu) 
+               ppn='#PBS -l nodes='$pes':ppn=1'
+               extra='-machinefile $PBS_NODEFILE'
+               ;;
+       tg-login*|honest*.ncsa.uiuc.edu)
+               # always ppn=2
+               nodes=`expr \( $pes + 1 \) / 2`
+               test $pes -eq 1 && ppns=1 || ppns=2
+               ppn='#PBS -l nodes='$nodes':ppn='$ppns
+               extra='-machinefile $PBS_NODEFILE'
+               ;;
+       co-login*.ncsa.uiuc.edu)
+               mem='#PBS -l mem=500mb'
+               ncpus="#PBS -l ncpus=$pes"
+               ;;
+       tun*)
+               USE_LSF=1
+               queue_stat=bjobs
+               queue_qsub=bsub
+               queue_kill=bkill
+               ;;
+       abe*)
+               # always ppn=2
+               nodes=`expr \( $pes + 1 \) / 2`
+               test $pes -eq 1 && ppns=1 || ppns=2
+               ppn='#PBS -l nodes='$nodes':ppn='$ppns
+               extra='-machinefile $PBS_NODEFILE'
+               ;;
+        kraken*)
+                ncores=`expr \( $pes + 11 \) / 12 \* 12`
+               ncpus="#PBS -l size=$ncores"
+               ;;
+       *)
+               ncpus="#PBS -l ncpus=$pes"
+               ;;
+       esac
+       if test $USE_LSF -eq 0
+       then
+         mpirun=`which mpirun 2>/dev/null`
+          npcmd="-np "
+          if test -z "$mpirun"
+          then
+            mpirun=`which aprun 2>/dev/null`
+            npcmd="-n "
+          fi
+          cat > $script << EOF
+#!/bin/sh
+# This is a charmrun-generated PBS batch job script.
+# The lines starting with #PBS are queuing system flags:
+#
+$ppn
+#
+$ncpus
+#
+#PBS -l walltime=$walllimit:00
+#
+$mem
+#
+#PBS -q $PBS_QUEUE
+#
+#PBS -N autobuild
+#
+#PBS -j oe
+#
+#PBS -o $output
+
+cd $indir
+
+cat \$PBS_NODEFILE
+echo $mpirun $npcmd $pes $extra $args
+$mpirun $npcmd $pes $extra $args
+
+# Save mpirun exit status
+status=\$?
+echo \$status > $result
+EOF
+       else
+#  use LSF
+         mpirun="cmpirun -lsf -poll -no_smp -gm_long 200000"
+          cat > $script << EOF
+#!/bin/sh
+# This is a charmrun-generated PBS batch job script.
+# The lines starting with #PBS are queuing system flags:
+#
+#BSUB -J autobuild
+#BSUB -W 0:$walllimit
+#BSUB -n $pes
+#BSUB -o $output
+
+cd $indir
+echo \$LSB_MCPU_HOSTS
+$mpirun $args
+# Save mpirun exit status
+status=\$?
+echo \$status > $result
+EOF
+       fi
+
+End() {
+       echo "Charmrun> $queue_kill $jobid ..."
+       $queue_kill $jobid
+       rm -f $script
+       exit $1
+}
+
+        echo "Submitting batch job for> $mpirun -np $pes $args"
+        echo " using the command> $queue_qsub $script"
+        chmod 755 $script
+       while [ -z "$jobid" ]
+       do
+         [ $USE_LSF = 0 ] && jobid=`$queue_qsub $script|tail -1`
+         [ $USE_LSF = 1 ] && jobid=`$queue_qsub < $script|tail -1|sed -e 's/[^0-9]*//g'`
+       done
+       echo "Job enqueued under job ID $jobid"
+# kill job if interrupted
+       trap 'End 1' 2 3
+       retry=0
+# Wait for the job to complete, by checking its status
+        while [ true ]
+        do
+                $queue_stat $jobid > tmp.$$
+               exitstatus=$?
+                if test -f $output
+                then
+# The job is done-- print its output
+                        rm tmp.$$
+# When job hangs, result file does not exist
+                       test -f $result && status=`cat $result` || status=1
+                       test $status -eq 0 && status=`grep 'End of program' $output > /dev/null 2>&1`
+                       cat $output
+                       rm -f $result
+                       test -f $status && rm -f $script $output
+                       exit $status
+                fi
+# The job is still queued or running-- print status and wait
+                tail -1 tmp.$$
+                rm tmp.$$
+# Job ID may not exist now
+               if test $exitstatus -ne 0
+               then
+# retry a few times when error occurs
+                       retry=`expr $retry + 1`
+                       if test $retry -gt 6
+                       then
+                               echo "Charmrun> too many errors, abort!"
+                               exit 1
+                       else
+                               sleep 15
+                       fi
+               else
+# job still in queue
+                       retry=0
+                       sleep 20
+               fi
+        done
+else
+  mpirun_cmd=`which mpirun 2>/dev/null`
+  if test -n "$mpirun_cmd"
+  then
+    [ -n "$MPI_MACHINEFILE" ] && args=" -machinefile $MPI_MACHINEFILE $args"
+    setarch_cmd=`which setarch 2>/dev/null`
+    if [ -n "$setarch_cmd" -a -x "$setarch_cmd" ]
+    then
+      # Disables randomization of the virtual address  space  (turns  on
+      #          ADDR_NO_RANDOMIZE).
+      cur_arch=`uname -m`
+      echo "charmrun>  $setarch_cmd $cur_arch -R  mpirun -np $pes $args"
+      $setarch_cmd $cur_arch -R  mpirun -np $pes $args
+    else
+      echo "charmrun> mpirun -np $pes $args"
+      mpirun -np $pes $args
+    fi
+  else
+    mpiexec_cmd=`which mpiexec 2>/dev/null`
+    if test -n "$mpiexec_cmd"
+    then
+      echo "charmrun> $mpiexec_cmd -n $pes $args"
+      echo
+      "$mpiexec_cmd" -n $pes $args
+    else
+      echo "Don't know how to run MPI program."
+      exit 1
+    fi
+  fi
+fi
+
+
diff --git a/src/arch/shmem-crayxt/conv-mach.h b/src/arch/shmem-crayxt/conv-mach.h
new file mode 100644 (file)
index 0000000..7d21078
--- /dev/null
@@ -0,0 +1,63 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef _CONV_MACH_H
+#define _CONV_MACH_H
+
+#define CMK_CRAYXT                                         1
+
+#define CMK_MALLOC_USE_GNU_MALLOC                          0
+#define CMK_MALLOC_USE_OS_BUILTIN                          1
+
+#define CMK_GETPAGESIZE_AVAILABLE                          1
+#define CMK_MEMORY_PAGESIZE                                8192
+#define CMK_MEMORY_PROTECTABLE                             0
+
+#define CMK_NODE_QUEUE_AVAILABLE                           0
+
+#define CMK_RSH_IS_A_COMMAND                               0
+#define CMK_RSH_NOT_NEEDED                                 1
+#define CMK_RSH_USE_REMSH                                  0
+
+#define CMK_SHARED_VARS_EXEMPLAR                           0
+#define CMK_SHARED_VARS_UNAVAILABLE                        1
+#define CMK_SHARED_VARS_UNIPROCESSOR                       0
+
+#define CMK_SIGNAL_NOT_NEEDED                              0
+#define CMK_SIGNAL_USE_SIGACTION                           0
+#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
+
+#define CMK_THREADS_REQUIRE_NO_CPV                         0
+#define CMK_THREADS_COPY_STACK                             0
+
+#define CMK_TIMER_USE_GETRUSAGE                            1
+#define CMK_TIMER_USE_SPECIAL                              0
+#define CMK_TIMER_USE_TIMES                                0
+
+#define CMK_TYPEDEF_INT2 short
+#define CMK_TYPEDEF_INT4 int
+#define CMK_TYPEDEF_INT8 long
+#define CMK_TYPEDEF_UINT2 unsigned short
+#define CMK_TYPEDEF_UINT4 unsigned int
+#define CMK_TYPEDEF_UINT8 unsigned long
+#define CMK_TYPEDEF_FLOAT4 float
+#define CMK_TYPEDEF_FLOAT8 double
+
+#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   1
+#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     0
+
+#define CMK_DEBUG_MODE                                     0
+#define CMK_WEB_MODE                                       1
+
+#define CMK_LBDB_ON                                       1
+
+#define CMK_SHMEM_H                                       <mpp/shmem.h>
+#define CMK_SHMEM_INIT                                    shmem_init()
+#define CMK_SHMEM_LOCK                                    1
+
+#endif
+
diff --git a/src/arch/shmem-crayxt/conv-mach.sh b/src/arch/shmem-crayxt/conv-mach.sh
new file mode 100644 (file)
index 0000000..d59a29f
--- /dev/null
@@ -0,0 +1,50 @@
+#CMK_DEFS="-I/opt/xt-mpt/1.5.47/mpich2-64/T/include "
+#CMK_LD_DEFS="-lrca "
+
+PGCC=`CC -V 2>/dev/null | grep pgCC`
+
+CMK_CPP_CHARM="/lib/cpp -P"
+CMK_CPP_C="cc -E $CMK_DEFS "
+CMK_CXXPP="CC -E $CMK_DEFS "
+CMK_CC="cc $CMK_DEFS "
+CMK_CXX="CC  $CMK_DEFS "
+CMK_LD="$CMK_CC $CMK_LD_DEFS"
+CMK_LDXX="$CMK_CXX $CMK_LD_DEFS"
+CMK_LIBS="-lckqt -lsma -lrca"
+CMK_QT='generic64'
+
+CMK_RANLIB="ranlib"
+CMK_LD_LIBRARY_PATH="-Wl,-rpath,$CHARMLIBSO/"
+
+# compiler for compiling sequential programs
+if test -n "$PGCC"
+then
+# gcc is needed for building QT
+CMK_SEQ_CC="gcc -fPIC"
+CMK_SEQ_CXX="pgCC -fPIC "
+else
+CMK_SEQ_CC="gcc -fPIC"
+CMK_SEQ_CXX="g++ -fPIC "
+fi
+CMK_SEQ_LD="$CMK_SEQ_CC "
+CMK_SEQ_LDXX="$CMK_SEQ_CXX "
+CMK_SEQ_LIBS=""
+
+# compiler for native programs
+CMK_NATIVE_CC="gcc "
+CMK_NATIVE_LD="gcc "
+CMK_NATIVE_CXX="g++ "
+CMK_NATIVE_LDXX="g++ "
+CMK_NATIVE_LIBS=""
+
+
+# for F90 compiler
+CMK_CF77="ftn "
+CMK_CF90="ftn "
+CMK_F90LIBS=""
+CMK_F90_USE_MODDIR=1
+CMK_F90_MODINC="-I"
+CMK_MOD_EXT="mod"
+
+CMK_NO_BUILD_SHARED=true
+