Remove deprecated net layer 21/3321/9
authorEvan Ramos <evan@hpccharm.com>
Tue, 21 Nov 2017 19:06:59 +0000 (13:06 -0600)
committerEvan Ramos <evan@hpccharm.com>
Thu, 14 Dec 2017 21:19:16 +0000 (15:19 -0600)
Change-Id: Ic031f2bf655e91bb52603a1b966ddbc70f93dc29

131 files changed:
.cproject
.gitignore
.project
build
charm.spec
doc/armci/manual.tex
doc/bignetsim/install.tex
doc/bigsim/install.tex
doc/charm++/advancedlb.tex
doc/charm++/checkpoint.tex
doc/charm++/quickbigsim.tex
doc/converse/ccs.tex
doc/debugger/implementation.tex
doc/debugger/manual.tex
doc/faq/install.tex
doc/jade/manual.tex
doc/libraries/msa.tex
doc/netfem/manual.tex
doc/pose/sample.tex
examples/pose/ASIM/Makefile
examples/pose/ChunkSim/Makefile
examples/pose/LBSim/Makefile
examples/pose/MiniNet/Makefile
examples/pose/PHOLD/Makefile
examples/pose/PatternGen/Makefile
examples/pose/Ring/Makefile
examples/pose/SimBenchmark/Makefile
relink.script
src/arch/net-darwin-x86_64/conv-mach-pxshm.h [deleted file]
src/arch/net-darwin-x86_64/conv-mach-pxshm.sh [deleted file]
src/arch/net-darwin-x86_64/conv-mach-smp.h [deleted file]
src/arch/net-darwin-x86_64/conv-mach-smp.sh [deleted file]
src/arch/net-darwin-x86_64/conv-mach.h [deleted file]
src/arch/net-darwin-x86_64/conv-mach.sh [deleted file]
src/arch/net-linux-amd64/vdir_link [deleted file]
src/arch/net-linux-arm7/conv-mach-ibverbs.h [deleted file]
src/arch/net-linux-arm7/conv-mach-ibverbs.sh [deleted file]
src/arch/net-linux-arm7/conv-mach-pthreads.h [deleted file]
src/arch/net-linux-arm7/conv-mach-pthreads.sh [deleted file]
src/arch/net-linux-arm7/conv-mach-pxshm.h [deleted file]
src/arch/net-linux-arm7/conv-mach-pxshm.sh [deleted file]
src/arch/net-linux-arm7/conv-mach-smp.h [deleted file]
src/arch/net-linux-arm7/conv-mach-smp.sh [deleted file]
src/arch/net-linux-arm7/conv-mach-sysvshm.h [deleted file]
src/arch/net-linux-arm7/conv-mach-sysvshm.sh [deleted file]
src/arch/net-linux-arm7/conv-mach.h [deleted file]
src/arch/net-linux-arm7/conv-mach.sh [deleted file]
src/arch/net-linux-ppc/cc-gcc64.h [deleted file]
src/arch/net-linux-ppc/cc-gcc64.sh [deleted file]
src/arch/net-linux-ppc/cc-xlc.h [deleted file]
src/arch/net-linux-ppc/cc-xlc.sh [deleted file]
src/arch/net-linux-ppc/cc-xlc64.h [deleted file]
src/arch/net-linux-ppc/cc-xlc64.sh [deleted file]
src/arch/net-linux-ppc/conv-mach-ibverbs.h [deleted file]
src/arch/net-linux-ppc/conv-mach-ibverbs.sh [deleted file]
src/arch/net-linux-ppc/conv-mach-mx.h [deleted file]
src/arch/net-linux-ppc/conv-mach-mx.sh [deleted file]
src/arch/net-linux-ppc/conv-mach-mx64.h [deleted file]
src/arch/net-linux-ppc/conv-mach-mx64.sh [deleted file]
src/arch/net-linux-ppc/conv-mach-smp.h [deleted file]
src/arch/net-linux-ppc/conv-mach-smp.sh [deleted file]
src/arch/net-linux-ppc/conv-mach.h [deleted file]
src/arch/net-linux-ppc/conv-mach.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-clustermatic.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-clustermatic.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-ibud.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-ibud.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-ibverbs.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-ibverbs.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-pxshm.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-pxshm.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-smp.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-smp.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach-sysvshm.h [deleted file]
src/arch/net-linux-x86_64/conv-mach-sysvshm.sh [deleted file]
src/arch/net-linux-x86_64/conv-mach.h [deleted file]
src/arch/net-linux-x86_64/conv-mach.sh [deleted file]
src/arch/net-linux/conv-mach-clustermatic.h [deleted file]
src/arch/net-linux/conv-mach-clustermatic.sh [deleted file]
src/arch/net-linux/conv-mach-gm.h [deleted file]
src/arch/net-linux/conv-mach-gm.sh [deleted file]
src/arch/net-linux/conv-mach-mx.h [deleted file]
src/arch/net-linux/conv-mach-mx.sh [deleted file]
src/arch/net-linux/conv-mach-pxshm.h [deleted file]
src/arch/net-linux/conv-mach-pxshm.sh [deleted file]
src/arch/net-linux/conv-mach-scyld.h [deleted file]
src/arch/net-linux/conv-mach-scyld.sh [deleted file]
src/arch/net-linux/conv-mach-smp.h [deleted file]
src/arch/net-linux/conv-mach-smp.sh [deleted file]
src/arch/net-linux/conv-mach-sysvshm.h [deleted file]
src/arch/net-linux/conv-mach-sysvshm.sh [deleted file]
src/arch/net-linux/conv-mach.h [deleted file]
src/arch/net-linux/conv-mach.sh [deleted file]
src/arch/net/Makefile.machine [deleted file]
src/arch/net/conv-common.h [deleted file]
src/arch/net/conv-mach-local.h [deleted file]
src/arch/net/conv-mach-local.sh [deleted file]
src/arch/net/conv-mach-syncft.h [deleted file]
src/arch/net/conv-mach-tcp.h [deleted file]
src/arch/net/conv-mach-tcp.sh [deleted file]
src/arch/net/machine-ammasso.c [deleted file]
src/arch/net/machine-ammasso.h [deleted file]
src/arch/net/machine-dgram.c [deleted file]
src/arch/net/machine-eth.c [deleted file]
src/arch/net/machine-gm.c [deleted file]
src/arch/net/machine-ibud.c [deleted file]
src/arch/net/machine-ibverbs.c [deleted file]
src/arch/net/machine-mx.c [deleted file]
src/arch/net/machine-pxshm.c [deleted file]
src/arch/net/machine-recover.c [deleted file]
src/arch/net/machine-sysvshm.c [deleted file]
src/arch/net/machine-tcp.c [deleted file]
src/arch/net/machine.c [deleted file]
src/arch/ofi/conv-common.h
src/arch/template/README
src/arch/template/conv-common.h
src/arch/template/machine.c
src/arch/win32/conv-autoconfig.h
src/arch/win32/unistd.h
src/conv-ccs/conv-ccs.c
src/conv-core/conv-config.h
src/conv-core/convcore.c
src/conv-core/converse.h
src/libs/ck-libs/ampi/ampiCC.in
src/libs/ck-libs/armci/ARMCI_README.txt
src/scripts/Makefile
src/scripts/charmc
src/scripts/configure.ac
src/util/ckdll_system.C
src/util/simd.h
tests/ampi/speed/msgspeed.h

index ba6c2fcca0fa545184bbfd14f84e1b70434efb4e..9eefbd97f308c86373166a1ab6cbb8e493f2da48 100644 (file)
--- a/.cproject
+++ b/.cproject
 <folderInfo id="cdt.managedbuild.config.gnu.exe.debug.590241402." name="/" resourcePath="">
 <toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1701969183" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
 <targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.807248394" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
-<builder autoBuildTarget="charm++" buildPath="${workspace_loc:/charm/net-linux-amd64/tmp}" enableAutoBuild="true" id="cdt.managedbuild.target.gnu.builder.exe.debug.1025102572" incrementalBuildTarget="charm++" keepEnvironmentInBuildfile="false" managedBuildOn="false" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="8" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
+<builder autoBuildTarget="charm++" buildPath="${workspace_loc:/charm/netlrts-linux-amd64/tmp}" enableAutoBuild="true" id="cdt.managedbuild.target.gnu.builder.exe.debug.1025102572" incrementalBuildTarget="charm++" keepEnvironmentInBuildfile="false" managedBuildOn="false" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="8" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
 <tool id="cdt.managedbuild.tool.gnu.archiver.base.926487193" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
 <tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.976682201" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
 <option id="gnu.cpp.compiler.exe.debug.option.optimization.level.344431787" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
 <option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1707866992" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
 <option id="gnu.cpp.compiler.option.include.paths.1853938627" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
-<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/net-linux-amd64/tmp}&quot;"/>
-<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/net-linux-amd64/tmp/libs/ck-libs/pythonCCS}&quot;"/>
+<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/netlrts-linux-amd64/tmp}&quot;"/>
+<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/netlrts-linux-amd64/tmp/libs/ck-libs/pythonCCS}&quot;"/>
 </option>
 <option id="gnu.cpp.compiler.option.preprocessor.def.1120432124" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
 <listOptionValue builtIn="false" value="CMK_SEPARATE_PATH=1"/>
@@ -43,7 +43,7 @@
 <option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.1414076724" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
 <option id="gnu.c.compiler.exe.debug.option.debugging.level.917143894" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
 <option id="gnu.c.compiler.option.include.paths.783568532" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" valueType="includePath">
-<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/net-linux-amd64/tmp}&quot;"/>
+<listOptionValue builtIn="false" value="&quot;${workspace_loc:/charm/netlrts-linux-amd64/tmp}&quot;"/>
 </option>
 <option id="gnu.c.compiler.option.preprocessor.def.symbols.2106822378" name="Defined symbols (-D)" superClass="gnu.c.compiler.option.preprocessor.def.symbols" valueType="definedSymbols">
 <listOptionValue builtIn="false" value="CMK_SEPARATE_PATH=1"/>
@@ -67,7 +67,7 @@
 </toolChain>
 </folderInfo>
 <sourceEntries>
-<entry excluding="bin/|include/|lib/|lib_so/|tmp/|net-**|mpi-**" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name=""/>
+<entry excluding="bin/|include/|lib/|lib_so/|tmp/|netlrts-**|mpi-**" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name=""/>
 </sourceEntries>
 </configuration>
 </storageModule>
index cf8d2bd000004c18013debd4a643858b0e971754..18e9cb4985974cbbfcaf2e2d91a24304f83158ff 100644 (file)
@@ -20,17 +20,15 @@ tmp
 src/util/pup_f.f90
 
 # Ignore directories created from compilation
-/net-*
 /netlrts-*
+/verbs-*
 /mpi-*
-/lapi*
-/vmi-*
 /multicore-*
 /uth-*
-/bluegenep*
 /pami*
 /gemini_gni-crayxe*
 /gni-*
+/ofi-*
 
 # Ignore cruft that would result from compiling charm example/test code
 # Note this is negated inside src/arch to permit addition of new charmrun files in there
index 85814e5bcc513ff1d2f9040c6e3e424efc08d4ee..9dc36a9d586278b778d3db0fe034f9620c537da9 100644 (file)
--- a/.project
+++ b/.project
@@ -30,7 +30,7 @@
                                </dictionary>
                                <dictionary>
                                        <key>org.eclipse.cdt.make.core.buildLocation</key>
-                                       <value>${workspace_loc:/charm/net-linux-amd64/tmp}</value>
+                                       <value>${workspace_loc:/charm/netlrts-linux-amd64/tmp}</value>
                                </dictionary>
                                <dictionary>
                                        <key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
diff --git a/build b/build
index 2c671a3032f0361f0a73531c75b43ca0b707bb46..9cf341ba37ed76f4008057a1889fe224453eca6d 100755 (executable)
--- a/build
+++ b/build
@@ -33,7 +33,7 @@ syntax() {
   echo ''
   fi
   echo '<versions>: ' 
-  ( cd $src ; ls -1 | egrep -v '(^shmem$)|(^mpi$)|(^sim$)|(^net(lrts)?$)|(^multicore$)|(^util$)|(^common$)|(^uth$)|(^conv-mach-fix.sh$)|(^win64$)|(^(gemini_)?gni$)|(^pami(lrts)?$)|(^verbs$)|(^ofi$)|(^template$)|(^cuda$)' | pr -3 -t )
+  ( cd $src ; ls -1 | egrep -v '(^shmem$)|(^mpi$)|(^sim$)|(^netlrts?$)|(^multicore$)|(^util$)|(^common$)|(^uth$)|(^conv-mach-fix.sh$)|(^win64$)|(^(gemini_)?gni$)|(^pami(lrts)?$)|(^verbs$)|(^ofi$)|(^template$)|(^cuda$)' | pr -3 -t )
   echo ''
   echo '<options>: compiler and platform specific options'
   echo 'icc iccstatic xlc xlc64 gcc clang pgcc cc mpicxx'
@@ -63,7 +63,7 @@ syntax() {
   echo '  smp             support for SMP, multithreaded charm on each node'
   echo '  omp             support for the integrated LLVM OpenMP runtime'
   echo '  ibverbs         use Infiniband Verbs for communication (only for netlrts-linux-x86_64; prefer verbs-*)'
-  echo '  tcp             use TCP sockets for communication (only for net version)'
+  echo '  tcp             use TCP sockets for communication (only for netlrts)'
   echo '  pthreads        compile with pthreads Converse threads'
   echo ''
   echo 'Advanced options:'
@@ -467,6 +467,11 @@ fi
 
 [ "x$VERSION" = "x" ] && syntax && exit 1
 
+if [ "x_$ARCH" = "x_net" ]; then
+  echo "Error: net-* has been removed, please use netlrts or verbs.";
+  exit 1;
+fi
+
 #Check if building verbs on Omni-Path 
 if [ "x_$ARCH" = "x_verbs" ] && type /usr/sbin/opafabricinfo >/dev/null 2>&1; then
   echo "WARNING: Detected Omni-Path diagnostic tools.";
index 8e0fb4ddae042d5c9cd81b4a22439d14bfcdf7b0..bc0be5e45090ad69e7c787ea8a6c680dbf57eebc 100644 (file)
@@ -29,7 +29,7 @@ Charm++ RPM for Linux
 %setup -n %{name}-%{version}-%{release}
 bzip2 -dc  %{_sourcedir}/Charm-%{version}-%{release}.tar.bz2 | tar xvf -
 %build
-./build AMPI net-linux-x86_64 %{charm_options} --with-production
+./build AMPI netlrts-linux-x86_64 %{charm_options} --with-production
 %clean
 rm -rf $RPM_BUILD_ROOT
 %install
index 7e482b923e6f1d83460a6bf16b32b225c1b2a66e..628f7c62bac7f64ef57cba4f2e6353501c4450b4 100644 (file)
@@ -71,7 +71,7 @@ information and motivation for the development of this library.
 Build charm target ARMCI (instead of charm or AMPI):
 \begin{verbatim}
 > cd charm
-> ./build ARMCI net-linux -O3
+> ./build ARMCI netlrts-linux -O3
 \end{verbatim}
 
 \subsection{Writing a Simple ARMCI Program}
index a92eaddafc7f0e1336776ca0a2a8e28729736767..9b9a21f2383f1531bb922f222f2c019c2bea0b22 100644 (file)
@@ -1,13 +1,13 @@
 \subsection{Compiling BigSimulator}
 
 To compile the simulator which is called BigSimulator (or BigNetSim), we need
-the regular Charm++ build (net-linux-x86\_64 in our example).  It needs to be
+the regular Charm++ build (netlrts-linux-x86\_64 in our example).  It needs to be
 complemented with a few more libraries from BigSim and with the Pose
 discrete-event simulator. These pieces can be built, respectively, with:
 
 \begin{verbatim}
-./build bgampi net-linux-x86_64 -O2
-./build pose net-linux-x86_64 -O2
+./build bgampi netlrts-linux-x86_64 -O2
+./build pose netlrts-linux-x86_64 -O2
 \end{verbatim}
 
 Access to the discrete-event simulation is realized via a Charm++ package
index 6cbe388a735478d4360db3c1e5a8e57bbc4d3dce..62319239edc3df7817315648757678787b1cfb6f 100644 (file)
@@ -16,7 +16,7 @@ needs to provide {\em target} and {\em platform} selections:
 
 For example, to compile on a 64-bit Linux machine, one would type:
 \begin{verbatim}
-./build charm++ net-linux-x86_64 -O2
+./build charm++ netlrts-linux-x86_64 -O2
 \end{verbatim}
 
 \noindent which builds essential \charmpp{} kernel using UDP sockets as the
@@ -26,7 +26,7 @@ on MPI using:
 ./build charm++ mpi-linux-x86_64 -O2
 \end{verbatim}
 
-For other platforms, net-linux-x86\_64 should be replaced by whatever platform is being used. 
+For other platforms, netlrts-linux-x86\_64 should be replaced by whatever platform is being used. 
 See the charm/README file for a complete list of supported platforms.
 
 \subsubsection{Building Only the BigSim Emulator}
@@ -35,7 +35,7 @@ The BigSim Emulator is implemented on top of Converse in Charm++.  To compile
 the BigSim Emulator, one can compile Emulator libraries directly on top of
 normal \charmpp{} using ``bgampi'' as the compilation target, like
 \begin{verbatim}
-./build bgampi net-linux-x86_64 -O2
+./build bgampi netlrts-linux-x86_64 -O2
 \end{verbatim}
 
 With Emulator libraries, one can write BigSim applications using its
@@ -47,16 +47,16 @@ In order to build Charm++ or AMPI on top of BigSim Emulator (which itself is
 implemented on top of Converse), a special build option ``bigemulator'' needs
 to be specified:
 \begin{verbatim}
-./build bgampi net-linux-x86_64 bigemulator -O2
+./build bgampi netlrts-linux-x86_64 bigemulator -O2
 \end{verbatim}
 
 The ``bgampi'' option is the compilation {\em target} that tells ``build'' to
 compile BigSim Emulator libraries in addition to \charmpp{} kernel libraries.
-The ``bigemulator'' option is a build {\em option} to platform ``net-linux'',
+The ``bigemulator'' option is a build {\em option} to platform ``netlrts-linux'',
 which tells ``build'' to build Charm++ on top of the BigSim Emulator. 
 
 The above ``build" command creates a directory named
-``net-linux-x86\_64-bigemulator" under charm, which contains all the header
+``netlrts-linux-x86\_64-bigemulator" under charm, which contains all the header
 files and libraries needed for compiling a user application.
 With this version of Charm++, one can run normal Charm++ and AMPI application
 on top of the emulator (in a virtualized environment).
index ae7775f93f7034c6c9771fc76e2aba0cc8f4eaf5..056166d02ea216698e83a94afec04c0363e257f6 100644 (file)
@@ -308,7 +308,7 @@ The database data structure named \kw{LDStats} is defined in {\em CentralLB.h}:
 \end{verbatim}
 
 \begin{enumerate}
-\item {\em LBRealType} is the data type for load balancer measured time. It is "double" by default. User can specify the type to float at \charmpp{} compile time if want. For example, ./build charm++ net-linux-x86\_64 {-}{-}with-lbtime-type=float;
+\item {\em LBRealType} is the data type for load balancer measured time. It is "double" by default. User can specify the type to float at \charmpp{} compile time if want. For example, ./build charm++ netlrts-linux-x86\_64 {-}{-}with-lbtime-type=float;
 \item {\em procs} array defines processor attributes and usage data for each
 processor;
 \item {\em objData} array records per object information, {\em LDObjData} is defined in {\em lbdb.h};
index 0e04d424ef4572c85309aef458b5a377e2381178..991cf10bf6eb5f036c626f4f40c0d35e095a1f24 100644 (file)
@@ -273,7 +273,7 @@ functionality available, the parameter \emph{syncft} must be provided
 at build time:
 
 \begin{alltt}
-   ./build charm++ net-linux-x86_64 syncft
+   ./build charm++ netlrts-linux-x86_64 syncft
 \end{alltt} 
 
 At present, only a few of the machine layers underlying the \charmpp{}
index 6b1d08b8e66fe742c6175e2b655d066f76571b52..94abf28ca58bb90f22883aa42e8ef775f83812b8 100644 (file)
@@ -12,10 +12,10 @@ For more information, please refer to BigSim and Projections manuals.
 
 \item build charm (and AMPI) with bigemulator and bigsim (replace ``linux" with ``darwin" for mac):\\
         cd charm\\
-        ./build charm++ net-linux-x86\_64 bigemulator bigsim\\
-        ./build AMPI net-linux-x86\_64 bigemulator bigsim\\
+        ./build charm++ netlrts-linux-x86\_64 bigemulator bigsim\\
+        ./build AMPI netlrts-linux-x86\_64 bigemulator bigsim\\
 
-\item compile your code using charm or AMPI compilers located in ``net-linux-x86\_64-bigemulator-bigsim/bin", for example:\\
+\item compile your code using charm or AMPI compilers located in ``netlrts-linux-x86\_64-bigemulator-bigsim/bin", for example:\\
         cd tests/ampi/jacobi3d; make\\
 
 \item run your application emulating the target machine, for example:\\
@@ -27,7 +27,7 @@ For more information, please refer to BigSim and Projections manuals.
 
 \item build BigFastSim:\\
         cd BigFastSim/Release\\
-        vim makefile  \#change CHARMPATH=\$(HOME)/charm/net-linux-x86\_64-bigemulator-bigsim/\\
+        vim makefile  \#change CHARMPATH=\$(HOME)/charm/netlrts-linux-x86\_64-bigemulator-bigsim/\\
         make\\
 
 \item copy simulator to trace files' directory:\\
index 1596d353260e98e51c5f09bdd1464df7676f5cd1..e26a1e434c7b5c4ddfebc3fc05ed059bb822c9dc 100644 (file)
@@ -13,7 +13,7 @@ The following sections describe both these parts.
 A CCS client accesses a running \converse{} program by talking to
 a {\tt server-host}, which receives the CCS requests and relays them
 to the appropriate processor.  The {\tt server-host} is \kw{charmrun}
-for net- versions, and is the first processor for all other versions.
+for netlrts- versions, and is the first processor for all other versions.
 
 
 \section{CCS: Starting a Server}
index d64bd5a62bfd5ad8d66f08e2f09529c050f1646a..de576ff4042160a7bbd7966b6afb383d3276ffcb 100644 (file)
@@ -68,10 +68,10 @@ arbitrary non-Converse programs. A CCS client accesses a running
 Converse program by talking to a \texttt{server-host} which
 receives the CCS requests and relays them to the appropriate
 processor. The \texttt{server-host} is \texttt{charmrun}
-\cite{charmman} for net-versions and is the first processor for
+\cite{charmman} for netlrts- versions and is the first processor for
 all other versions.
 
-In the case of the net-version of \charmpp{}, a Converse program
+In the case of the netlrts- version of \charmpp{}, a Converse program
 is started as a server by running the \charmpp{} program using the
 additional runtime option ``\textit{++server}''. This opens the CCS
 server on any TCP port number. The TCP port number can be
index 57d4e49ba9b674fcc0039ed52eadc22a504b5ae7..22e5c3d8f0826f469d9822ead26028cecab3c57d 100644 (file)
@@ -150,7 +150,7 @@ from within the \texttt{File} menu as shown in Figure~\ref{menu}.
 \end{figure}
 
 
-\emph{Note: charmdebug command line launching only works on net-*, netlrts-*, and verbs-*
+\emph{Note: charmdebug command line launching only works on netlrts-* and verbs-*
 builds of \charmpp{} }
 
 To replay a previously recorded session:
@@ -239,7 +239,7 @@ to remote running applications in the near future.
 %%Behavior on a cluster can be
 %%recorded and then replayed in a local run.
 
-Due to the current implementation, the debugging tool is limited to net-*, netlrts-*, and verbs-*
+Due to the current implementation, the debugging tool is limited to netlrts-* and verbs-*
 versions. Other builds of \charmpp{} might have unexpected behavior. In the near
 future this will be extended at least to the mpi-* versions.
 
index 906339e7fb455a245f9e764bfdbfbaa8dd5c80ea..6de606f9d9a9b6408d8432ea213711f1a3e30101 100644 (file)
@@ -37,7 +37,7 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 \subsection{If the interactive script fails, how do I compile Charm++?}
 
 %<p>First, on a typical Linux machine, unpack the tarball, cd into "charm",
-%and type <tt>./build AMPI net-linux -g</tt>.
+%and type <tt>./build AMPI netlrts-linux -g</tt>.
 
 %<p>Next, if your machine is similar to one of the machines below, 
 %use the listed "build" command:
@@ -49,8 +49,8 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %      <td>Myrinet Linux Cluster. <br>
 %              640 2GHz G5 processors as 2-way nodes.</td>
 %      <td>
-%              Debug: ./build AMPI net-ppc-darwin -g<br>
-%              Production: ./build AMPI net-ppc-darwin gm -O3<br>
+%              Debug: ./build AMPI netlrts-ppc-darwin -g<br>
+%              Production: ./build AMPI netlrts-ppc-darwin gm -O3<br>
 %              To run a job, use "rjc" to find nodes, then "charmrun".
 %      </td>
 %</tr>
@@ -83,8 +83,8 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %              2,560 3.2 GHz Intel Xeon processors as 2-way nodes with Myrinet, 3 GB RAM per node.
 %      </td>
 %      <td>
-%              Use: ./build AMPI net-linux gm -O , or<br>
-%                   ./build AMPI net-linux gm icc ifort -O (Intel compilers) , or<br>
+%              Use: ./build AMPI netlrts-linux gm -O , or<br>
+%                   ./build AMPI netlrts-linux gm icc ifort -O (Intel compilers) , or<br>
 %                   ./build AMPI mpi-linux cmpi -O (Champion MPI)<br>
 %      </td>
 %</tr>
@@ -94,7 +94,7 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %              1,744 1.3/1.5 GHz Intel Itanium-2 processors as 2-way nodes with Myrinet, 4 GB RAM per node.
 %      </td>
 %      <td>
-%              Use: ./build AMPI net-linux-ia64 gm -O , or<br>
+%              Use: ./build AMPI netlrts-linux-ia64 gm -O , or<br>
 %                   ./build AMPI mpi-linux-ia64 gm -nobs -O (MPICH over GM)<br>
 %      </td>
 %</tr>
@@ -113,7 +113,7 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %              968 1 GHz 32-bit Intel Pentium III processors as 2-way nodes with Myrinet.
 %      </td>
 %      <td>
-%              Debug: ./build AMPI net-linux gm icc -O<br>
+%              Debug: ./build AMPI netlrts-linux gm icc -O<br>
 %              Production: ./build AMPI mpi-linux vmi icc -O<br>
 %              To run an interactive job with net version, use "charmrun". To run with mpi version, use vmirun. Checkout NCSA's webpage for how to write a job script.<br>
 %      </td>
@@ -124,7 +124,7 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %              256 800 MHz 64-bit Intel Itanium processors as 2-way nodes with Myrinet.
 %      </td>
 %      <td>
-%              Use: ./build AMPI net-linux-ia64 gm ecc -O , or<br>
+%              Use: ./build AMPI netlrts-linux-ia64 gm ecc -O , or<br>
 %                   ./build AMPI mpi-linux-ia64 vmi ecc -O<br>
 %              To run an interactive job with net version, use "charmrun". To run with mpi version, use vmirun.<br>
 %      </td>
@@ -142,8 +142,8 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %              140 1.7 GHz 32-bit Athlon MP 2000 processors as 2-way nodes.
 %      </td>
 %      <td>
-%              Debug: ./build AMPI net-linux -g<br>
-%              Production: ./build AMPI net-linux -O<br>
+%              Debug: ./build AMPI netlrts-linux -g<br>
+%              Production: ./build AMPI netlrts-linux -O<br>
 %              To run interactive use "frun", "fsub" for batch submission.<br> 
 %      </td>
 %</tr>
@@ -159,7 +159,7 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %option like "icc", "pgcc", "xlc", "xlc64", or many others--
 %see the README file or charm/src/arch/common for a complete list.
 %For platform specific options, one can run build command with "help" option, for example: <br>
-%./build charm++ net-linux help
+%./build charm++ netlrts-linux help
 
 %<li>Under Linux, if you or your communication layer uses pthreads,
 %and you use Charm threads (such as AMPI), and your Glibc is less than
@@ -167,9 +167,9 @@ Yes.  Keep src, bin, lib, lib\_so, include, tmp.  You will not need tests, examp
 %This version links in a special version of pthreads that works with
 %our user-level threads.
 
-%<li>On the net- version, if you're having trouble with rsh/ssh or just
+%<li>On the netlrts- version, if you're having trouble with rsh/ssh or just
 %want to spawn processes on your own machine, add the "local" option, 
-%like <tt>./build AMPI net-linux local -g</tt>
+%like <tt>./build AMPI netlrts-linux local -g</tt>
 
 %<li>Run "./build --help" to display detailed help page for supported build options
 
index ac274974758eb764e2b9b94729e1653459a61e3b..8addd350b1195c293898ac41ef42bff27fbfe7e1 100644 (file)
@@ -160,7 +160,7 @@ Following is an example of how to build \jade under linux and ethernet
 environment, with debugging info produced:
 
 \begin{alltt}
-> build jade net-linux -g
+> build jade netlrts-linux -g
 \end{alltt}
 
 \section{Compiling and Running \jade Programs}
index e149dd426cb709bd48d33ed86c3263452870524d..ab27aa4317e5350b892fbcb344dca03c51db956d 100644 (file)
@@ -54,8 +54,8 @@ calculations are required to access data.
 
 To use MSA in a \charmpp\ program:
 \begin{itemize}
-\item build \charmpp\ for your architecture, e.g. \texttt{net-linux}.
-\item \texttt{cd charm/net-linux/tmp/libs/ck-libs/multiphaseSharedArrays/; make}
+\item build \charmpp\ for your architecture, e.g. \texttt{netlrts-linux}.
+\item \texttt{cd charm/netlrts-linux/tmp/libs/ck-libs/multiphaseSharedArrays/; make}
 \item \texttt{\#include ``msa/msa.h''} in your header file.
 \item Compile using \texttt{charmc} with the option ``\texttt{-module
       msa}''
index 82123a6dc6e440e412e29e4abe5e9a739e4667fb..931b2e3089727296816998ddcbe34ff0b6917825 100644 (file)
@@ -29,7 +29,7 @@ the network via the network protocol CCS (Converse Client/Server).
 
 NetFEM is part of \charmpp{}, so it can be downloaded
 as part of charm.  To build NetFEM, just build FEM normally,
-or else do a make in charm/net-linux/tmp/libs/ck-libs/netfem/.
+or else do a make in charm/netlrts-linux/tmp/libs/ck-libs/netfem/.
 
 To link with NetFEM, add \kw{-module netfem} to your
 program's link line.  Note that you do {\em not} need to use
index 16a911ebf474ec6c848b4d6c8f1bbbcbbb38e01e..af0eae0d02e28d854abdf4fcd8be1e5290c6df5f 100644 (file)
@@ -1,7 +1,7 @@
 \section{Compiling, Running and Debugging a Sample \pose{} program}
 
 Sample code is available in the Charm++ source distribution.  Assuming a
-net-linux build of Charm++, look in {\tt charm/net-linux/examples/pose}.
+netlrts-linux build of Charm++, look in {\tt charm/netlrts-linux/examples/pose}.
 The SimBenchmark directory contains a synthetic benchmark simulation and is
 fairly straightforward to understand.
 
index 82cae1b1173a252da404d5d652f22951028a016a..4b5b152a3401f24a27153f76a2c9b105b4ee43b5 100644 (file)
@@ -17,7 +17,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index 2f16504a0d65bad9d858bd9dd18d48d47bd0f62e..8fdc1b276652288d0e3edcaef58c7bbfbf62ca0b 100644 (file)
@@ -17,7 +17,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index 9911942b21316bc72e7473a9409bf4ebf9042641..9e08545d820864b7e775db982c7192b3e5c7f98f 100644 (file)
@@ -17,7 +17,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index c58c22c8742ad1216c489b9279710fc7895d3d7b..a87605b34dc8e4c684bfb7c12df5fae13725fd73 100644 (file)
@@ -22,7 +22,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index d63c96eaf7a267f6dcde56957a9ff4418be052ad..d95f48ea898f42e2ae3ceea7a00199c5d1b5a5f4 100644 (file)
@@ -17,7 +17,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index e330d77223cb8f0b132d57219d3d5361928a1d5f..a26eea36be559c3732a5713b24f61a7aad844707 100644 (file)
@@ -25,7 +25,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index 2f2246e59730b00454436b1ba6790b5d8663378e..26f6125f01d5be5c7764dcf8f0fd2e0653ce992d 100644 (file)
@@ -25,7 +25,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index fb1860504998cb1b4f7941448366c79087dc9f2a..6dc6e1b7f77974284ca44ae61582649612439e2f 100644 (file)
@@ -17,7 +17,7 @@ CHARMC=$(CHARMBIN)/charmc -I$(CHARMINC)/pose $(OPTS)
 
 # Default Location variables
 # **************************
-#POSE_ROOT=$(HOME)/charm/net-linux/lib
+#POSE_ROOT=$(HOME)/charm/netlrts-linux/lib
 
 # Default module names
 # - MAKE MODIFICATIONS HERE
index f785bc94a9d16903850f916d7cbceb5ff0fbabac..aaf2da807b9016906a932174a20a6c154674d3ab 100755 (executable)
@@ -7,7 +7,7 @@
 #
 # Usage: relink.script <build-version-name-string>
 #
-# Example: relink.script net-linux-smp.
+# Example: relink.script netlrts-linux-smp.
 # by Chee Wai Lee      10/13/2003
 #
 # rewritten by
diff --git a/src/arch/net-darwin-x86_64/conv-mach-pxshm.h b/src/arch/net-darwin-x86_64/conv-mach-pxshm.h
deleted file mode 100644 (file)
index f534175..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_PXSHM
-#define CMK_USE_PXSHM                  1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-darwin-x86_64/conv-mach-pxshm.sh b/src/arch/net-darwin-x86_64/conv-mach-pxshm.sh
deleted file mode 100644 (file)
index 40a223c..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS "
diff --git a/src/arch/net-darwin-x86_64/conv-mach-smp.h b/src/arch/net-darwin-x86_64/conv-mach-smp.h
deleted file mode 100644 (file)
index 70aa94c..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-
-#define CMK_SMP                                                   1
-
-
-#undef CMK_SHARED_VARS_UNAVAILABLE
-#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
-#define CMK_SHARED_VARS_UNAVAILABLE                        0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
-
-#undef CMK_THREADS_USE_JCONTEXT
-#define CMK_THREADS_USE_JCONTEXT                           1
-
-#undef CMK_USE_POLL
-#define CMK_USE_POLL                                       1
-#undef CMK_USE_KQUEUE
-#define CMK_USE_KQUEUE                                     0
-
-/*#if !CMK_GCC_X86_ASM || !CMK_GCC_X86_ASM_ATOMICINCREMENT */
-/* #define CMK_PCQUEUE_LOCK                                    1 */
-/*#endif */
diff --git a/src/arch/net-darwin-x86_64/conv-mach-smp.sh b/src/arch/net-darwin-x86_64/conv-mach-smp.sh
deleted file mode 100644 (file)
index ad6551b..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-CMK_DEFS="$CMK_DEFS -D_REENTRANT "
-CMK_LIBS="-lpthread $CMK_LIBS "
-CMK_QT='generic64-light'
-CMK_SMP="1"
diff --git a/src/arch/net-darwin-x86_64/conv-mach.h b/src/arch/net-darwin-x86_64/conv-mach.h
deleted file mode 100644 (file)
index f8f6bde..0000000
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef _CONV_MACH_H
-#define _CONV_MACH_H
-
-// specify the version of the UNIX APIs that we want to use (for ucontext headers)
-#define _XOPEN_SOURCE                                       
-
-#define CMK_64BIT    1
-
-#define CMK_NETPOLL                                        1
-
-#define CMK_ASYNC_NOT_NEEDED                               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
-#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
-
-#define CMK_GETPAGESIZE_AVAILABLE                          0
-
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-#define CMK_MALLOC_USE_GNUOLD_MALLOC                       0
-
-#define CMK_MEMORY_BUILD_GNU_HOOKS                         0
-#define CMK_MEMORY_PAGESIZE                                4096
-#define CMK_MEMORY_PROTECTABLE                             0
-
-
-#define CMK_SSH_IS_A_COMMAND                               1
-#define CMK_SSH_NOT_NEEDED                                 0
-
-#define CMK_SHARED_VARS_UNAVAILABLE                        1
-#define CMK_SHARED_VARS_UNIPROCESSOR                       0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  0
-
-#define CMK_THREADS_USE_PTHREADS                           0
-#define CMK_THREADS_ARE_WIN32_FIBERS                       0
-#define CMK_THREADS_USE_CONTEXT                            0
-#define CMK_THREADS_USE_JCONTEXT                           1
-
-#define CMK_SIGNAL_NOT_NEEDED                              0
-#define CMK_SIGNAL_USE_SIGACTION                           0
-#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
-
-#define CMK_THREADS_REQUIRE_NO_CPV                         0
-#define CMK_THREADS_COPY_STACK                             0
-
-#define CMK_TIMER_USE_GETRUSAGE                            1
-#define CMK_TIMER_USE_SPECIAL                              0
-#define CMK_TIMER_USE_TIMES                                0
-
-
-#define CMK_DEBUG_MODE                                    0 
-#define CMK_WEB_MODE                                       1
-
-#define CMK_LBDB_ON                                       1
-
-#define CMK_STACKSIZE_DEFAULT                             262144
-/*
-#define CMK_NO_ISO_MALLOC                                 1
-*/
-
-
-/* Mac-specific optimizations */
-#undef CMK_USE_POLL
-#define CMK_USE_POLL                                       0
-#define CMK_USE_KQUEUE                                     1
-#define PXSHM_OSSPINLOCK                                   1
-#define PXSHM_LOCK                                         0
-#define PXSHM_FENCE                                        0
-
-
-#endif
-
diff --git a/src/arch/net-darwin-x86_64/conv-mach.sh b/src/arch/net-darwin-x86_64/conv-mach.sh
deleted file mode 100644 (file)
index b5a9718..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-. $CHARMINC/cc-clang.sh
-. $CHARMINC/conv-mach-darwin.sh
diff --git a/src/arch/net-linux-amd64/vdir_link b/src/arch/net-linux-amd64/vdir_link
deleted file mode 100644 (file)
index cf464fc..0000000
+++ /dev/null
@@ -1 +0,0 @@
-net-linux-x86_64
diff --git a/src/arch/net-linux-arm7/conv-mach-ibverbs.h b/src/arch/net-linux-arm7/conv-mach-ibverbs.h
deleted file mode 100644 (file)
index d26d758..0000000
+++ /dev/null
@@ -1,38 +0,0 @@
-#undef CMK_USE_IBVERBS
-#define CMK_USE_IBVERBS                                1
-
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                            1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#undef CMK_THREADS_USE_CONTEXT
-#undef CMK_THREADS_USE_JCONTEXT
-#define CMK_THREADS_USE_CONTEXT                            0
-#define CMK_THREADS_USE_JCONTEXT                           1
-
-/*
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-*/
-
-#undef CMK_BROADCAST_SPANNING_TREE
-#define CMK_BROADCAST_SPANNING_TREE 0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE  1
-
-#undef CMK_DIRECT                 
-#define CMK_DIRECT                                        1
-
-#undef CMK_DISABLE_SYNC
-#define CMK_DISABLE_SYNC       1
-
-
-#undef CMK_IBVERBS_FAST_START
-#define CMK_IBVERBS_FAST_START                         0
diff --git a/src/arch/net-linux-arm7/conv-mach-ibverbs.sh b/src/arch/net-linux-arm7/conv-mach-ibverbs.sh
deleted file mode 100644 (file)
index 7aa875d..0000000
+++ /dev/null
@@ -1,18 +0,0 @@
-#default ibverbs path for openib
-if test -z "$CMK_INCDIR"
-then
-  #openib-1.1
-  if test -f /opt/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/opt/ofed/include/"
-    CMK_LIBDIR="-L/opt/ofed/lib64"
-       fi
-  if test -f /usr/local/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/usr/local/ofed/include/"
-    CMK_LIBDIR="-L/usr/local/ofed/lib64"
-       fi
-fi
-
-CMK_LIBS="$CMK_LIBS -libverbs"
-
diff --git a/src/arch/net-linux-arm7/conv-mach-pthreads.h b/src/arch/net-linux-arm7/conv-mach-pthreads.h
deleted file mode 100644 (file)
index 94e01fd..0000000
+++ /dev/null
@@ -1,10 +0,0 @@
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#undef CMK_MALLOC_USE_GNUOLD_MALLOC
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-                                                                                
-#undef CMK_THREADS_USE_CONTEXT
-#undef CMK_THREADS_USE_PTHREADS
-#define CMK_THREADS_USE_PTHREADS                           1
-
diff --git a/src/arch/net-linux-arm7/conv-mach-pthreads.sh b/src/arch/net-linux-arm7/conv-mach-pthreads.sh
deleted file mode 100644 (file)
index 34cb4c6..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-
-CMK_LIBS="$CMK_LIBS -lpthread "
diff --git a/src/arch/net-linux-arm7/conv-mach-pxshm.h b/src/arch/net-linux-arm7/conv-mach-pxshm.h
deleted file mode 100644 (file)
index f534175..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_PXSHM
-#define CMK_USE_PXSHM                  1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux-arm7/conv-mach-pxshm.sh b/src/arch/net-linux-arm7/conv-mach-pxshm.sh
deleted file mode 100644 (file)
index 7ef2c6e..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS -lrt"
diff --git a/src/arch/net-linux-arm7/conv-mach-smp.h b/src/arch/net-linux-arm7/conv-mach-smp.h
deleted file mode 100644 (file)
index 5e89297..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-
-#define CMK_SMP                                                   1
-
-
-#undef CMK_SHARED_VARS_UNAVAILABLE
-#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
-#define CMK_SHARED_VARS_UNAVAILABLE                        0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-/*#define CMK_MMAP_PROBE                                     1 */
-
-/*#define CMK_PCQUEUE_LOCK                                1 */
-/*#define CMK_USE_MFENCE                                          1 */
-/* Replaced by CMK_NOT_USE_TLS_THREAD as default */
-/*#define CMK_USE_TLS_THREAD                                 1*/
diff --git a/src/arch/net-linux-arm7/conv-mach-smp.sh b/src/arch/net-linux-arm7/conv-mach-smp.sh
deleted file mode 100644 (file)
index 0ada4bc..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-CMK_DEFS=' -D_REENTRANT '
-CMK_LIBS="-lpthread $CMK_LIBS "
-CMK_SMP='1'
diff --git a/src/arch/net-linux-arm7/conv-mach-sysvshm.h b/src/arch/net-linux-arm7/conv-mach-sysvshm.h
deleted file mode 100644 (file)
index 1847dd0..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_SYSVSHM
-#define CMK_USE_SYSVSHM                        1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux-arm7/conv-mach-sysvshm.sh b/src/arch/net-linux-arm7/conv-mach-sysvshm.sh
deleted file mode 100644 (file)
index 40a223c..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS "
diff --git a/src/arch/net-linux-arm7/conv-mach.h b/src/arch/net-linux-arm7/conv-mach.h
deleted file mode 100644 (file)
index affa8e7..0000000
+++ /dev/null
@@ -1,63 +0,0 @@
-
-#ifndef _CONV_MACH_H
-#define _CONV_MACH_H
-
-#define CMK_NETPOLL         1
-
-#define CMK_ARM                                          1 
-
-#define CMK_ASYNC_NOT_NEEDED                               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
-#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
-
-#define CMK_DLL_CC  "g++ -shared -O3 -o "
-
-#define CMK_GETPAGESIZE_AVAILABLE                          1
-
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#define CMK_MEMORY_PAGESIZE                                4096
-#define CMK_MEMORY_PROTECTABLE                             0
-
-
-#define CMK_SSH_IS_A_COMMAND                               1
-#define CMK_SSH_NOT_NEEDED                                 0
-
-#define CMK_SHARED_VARS_UNAVAILABLE                        1
-#define CMK_SHARED_VARS_UNIPROCESSOR                       0
-
-#define CMK_THREADS_USE_CONTEXT                            0
-#define CMK_THREADS_USE_JCONTEXT                           0
-#define CMK_THREADS_USE_PTHREADS                           0
-#define CMK_THREADS_ARE_WIN32_FIBERS                       0
-
-#define CMK_SIGNAL_NOT_NEEDED                              0
-#define CMK_SIGNAL_USE_SIGACTION                           0
-#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
-
-#define CMK_THREADS_REQUIRE_NO_CPV                         0
-#define CMK_THREADS_COPY_STACK                             0
-
-#define CMK_TIMER_USE_RDTSC                                0
-#define CMK_TIMER_USE_GETRUSAGE                            1
-#define CMK_TIMER_USE_SPECIAL                              0
-#define CMK_TIMER_USE_TIMES                                0
-
-#define CMK_64BIT   0 
-#define CMK_32BIT   1
-#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   0
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     1
-
-
-#define CMK_DEBUG_MODE                                    0 
-#define CMK_WEB_MODE                                       1
-
-#define CMK_LBDB_ON                                       1
-
-
-#endif
-
-
diff --git a/src/arch/net-linux-arm7/conv-mach.sh b/src/arch/net-linux-arm7/conv-mach.sh
deleted file mode 100644 (file)
index 31f74b9..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-. $CHARMINC/cc-gcc.sh
-
-#CMK_DEFS="$CMK_DEFS -DHAVE_USR_INCLUDE_MALLOC_H=1 "
-
-CMK_XIOPTS=''
-CMK_WARNINGS_ARE_ERRORS='-Werror'
-CMK_CXX_OPTIMIZE='-O3'
-
-CMK_QT='generic64-light'
-CMK_CF90=`which f95 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-#    CMK_FPP="cpp -P -CC"
-#    CMK_CF90="$CMK_CF90 -fpic -fautomatic -fdollar-ok "
-#    CMK_CF90_FIXED="$CMK_CF90 -ffixed-form "
-#    CMK_F90LIBS="-lgfortran "
-#    CMK_F90_USE_MODDIR=1
-#    CMK_F90_MODINC="-I"
-#    CMK_MOD_NAME_ALLCAPS=
-#    CMK_MOD_EXT="mod"
-    . $CHARMINC/conv-mach-gfortran.sh
-else
-    CMK_CF77='g77 '
-    CMK_CF90='f90 '
-    CMK_CF90_FIXED="$CMK_CF90 -W132 "
-    CMK_F90LIBS='-L/usr/absoft/lib -L/opt/absoft/lib -lf90math -lfio -lU77 -lf77math '
-    CMK_F77LIBS='-lg2c '
-    CMK_F90_USE_MODDIR=1
-    CMK_F90_MODINC='-p'
-fi
diff --git a/src/arch/net-linux-ppc/cc-gcc64.h b/src/arch/net-linux-ppc/cc-gcc64.h
deleted file mode 100644 (file)
index 996ac44..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-
-#undef CMK_64BIT
-#define CMK_64BIT                                              1
diff --git a/src/arch/net-linux-ppc/cc-gcc64.sh b/src/arch/net-linux-ppc/cc-gcc64.sh
deleted file mode 100644 (file)
index d0da68d..0000000
+++ /dev/null
@@ -1,48 +0,0 @@
-CMK_64bit=" -m64 "
-CMK_PIC="-fpic"
-
-CMK_CPP_CHARM="cpp -P"
-CMK_CPP_C="gcc -E"
-CMK_CC="gcc $CMK_64bit "
-CMK_CXX="g++ $CMK_64bit "
-CMK_RANLIB="ranlib"
-CMK_LIBS="-lckqt"
-#CMK_LD="$CMK_CC -Wl,--allow-multiple-definition "
-#CMK_LDXX="$CMK_CXX -Wl,--allow-multiple-definition "
-CMK_LD="$CMK_CC $CMK_64bit"
-CMK_LDXX="$CMK_CXX $CMK_64bit"
-CMK_LD_SHARED="-shared"
-CMK_LD_LIBRARY_PATH="-Wl,-rpath,$CHARMLIBSO/"
-CMK_XIOPTS=""
-CMK_QT="generic64-light"
-
-# fortran compiler Absoft or gnu f95
-CMK_CF77="g77 $CMK_64bit"
-CMK_F77LIBS="-lg2c "
-CMK_CF90=`which xlf90_r 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-# xlf
-  bindir=`dirname $CMK_CF90`
-  libdir="$bindir/../lib64"
-  CMK_64bit=" -q64 "
-  CMK_CF90="$CMK_CF90 $CMK_64bit -qpic=large -qthreaded -qlanglvl=90std -qwarn64 -qspill=32648 -qsuppress=1513-029:1518-012:1518-059 -qsuffix=f=f90:cpp=F90 "
-  CMK_CF90_FIXED="$CMK_CF90 $CMK_64bit -qsuffix=f=f:cpp=F -qfixed=132 "
-  CMK_F90LIBS="-L/opt/ibmcmp/xlf/11.1/bin/../../../xlsmp/1.7/lib64 -L$libdir -lxl -lxlf90 -lxlfmath -lxlopt -lxlsmp"
-  CMK_F90_USE_MODDIR=1
-  CMK_F90_MODINC="-I"
-else
-# gnu f95
-  CMK_CF90=`which f95 2>/dev/null`
-  if test -n "$CMK_CF90"
-  then
-    CMK_FPP="cpp -P -CC"
-    CMK_CF90="$CMK_CF90 -m64 -fpic -fautomatic -fdollar-ok $CMK_64bit"
-    CMK_CF90_FIXED="$CMK_CF90 -m64 -ffixed-form $CMK_64bit"
-    CMK_F90LIBS="-lgfortran "
-    CMK_F90_USE_MODDIR=1
-    CMK_F90_MODINC="-I"
-  fi
-fi
-
-CMK_COMPILER='gcc'
diff --git a/src/arch/net-linux-ppc/cc-xlc.h b/src/arch/net-linux-ppc/cc-xlc.h
deleted file mode 100644 (file)
index 40a8c17..0000000
+++ /dev/null
@@ -1 +0,0 @@
-/* empty */
diff --git a/src/arch/net-linux-ppc/cc-xlc.sh b/src/arch/net-linux-ppc/cc-xlc.sh
deleted file mode 100644 (file)
index b9f02f5..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-CMK_CC='xlc_r '
-CMK_CXX='xlC_r '
-CMK_C_OPTIMIZE='-O3 -qstrict -Q  '
-CMK_CXX_OPTIMIZE='-O3 -qstrict -Q '
-CMK_PIC='-qpic=small'
-
-CMK_LD="$CMK_CC  "
-CMK_LDXX="$CMK_CXX  "
-
-CMK_QT="aix"
-
-CMK_AR='ar cq'
-CMK_NM='nm '
-
-CMK_CF77='xlf_r  '
-CMK_CF90='xlf90_r  -qsuffix=f=f90' 
-CMK_CF90_FIXED='xlf90_r ' 
-
-CMK_COMPILER='xlc'
diff --git a/src/arch/net-linux-ppc/cc-xlc64.h b/src/arch/net-linux-ppc/cc-xlc64.h
deleted file mode 100644 (file)
index a67cb6b..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-
-#define CMK_64BIT                                               1
diff --git a/src/arch/net-linux-ppc/cc-xlc64.sh b/src/arch/net-linux-ppc/cc-xlc64.sh
deleted file mode 100644 (file)
index a8ff0a4..0000000
+++ /dev/null
@@ -1,24 +0,0 @@
-COMMENT="Enable 64-bit mode (-q64)"
-CMK_CC='xlc_r -q64 '
-#CMK_CXX='xlC_r -q64 '
-CMK_CXX='xlC_r -q64 '
-CMK_C_OPTIMIZE='-O3 -qstrict '
-CMK_CXX_OPTIMIZE='-O3 -qstrict '
-CMK_LD="$CMK_CC "
-CMK_LDXX="$CMK_CXX "
-
-CMK_QT="generic64-light"
-
-CMK_NATIVE_CC='xlc_r -q64'
-CMK_NATIVE_LD='xlc_r -q64'
-CMK_NATIVE_CXX='xlC_r -q64'
-CMK_NATIVE_LDXX='xlC_r -q64'
-
-CMK_CF77='xlf_r -q64 '
-CMK_CF90='xlf90_r -q64 -qsuffix=f=f90' 
-CMK_CF90_FIXED='xlf90_r -q64 ' 
-
-CMK_AR='ar cq'
-CMK_NM='nm '
-
-CMK_COMPILER='xlc'
diff --git a/src/arch/net-linux-ppc/conv-mach-ibverbs.h b/src/arch/net-linux-ppc/conv-mach-ibverbs.h
deleted file mode 100644 (file)
index 8cc1396..0000000
+++ /dev/null
@@ -1,36 +0,0 @@
-#undef CMK_USE_IBVERBS
-#define CMK_USE_IBVERBS                                1
-
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                            1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#undef CMK_THREADS_USE_CONTEXT
-#undef CMK_THREADS_USE_JCONTEXT
-#define CMK_THREADS_USE_CONTEXT                            0
-#define CMK_THREADS_USE_JCONTEXT                           1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_SPANNING_TREE
-#define CMK_BROADCAST_SPANNING_TREE 0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE  1
-
-#undef CMK_DIRECT                 
-#define CMK_DIRECT                                        1
-
-#undef CMK_DISABLE_SYNC
-#define CMK_DISABLE_SYNC       1
-
-
-#undef CMK_IBVERBS_FAST_START
-#define CMK_IBVERBS_FAST_START                         0
diff --git a/src/arch/net-linux-ppc/conv-mach-ibverbs.sh b/src/arch/net-linux-ppc/conv-mach-ibverbs.sh
deleted file mode 100644 (file)
index 1a34136..0000000
+++ /dev/null
@@ -1,23 +0,0 @@
-#default ibverbs path for openib
-if test -z "$CMK_INCDIR"
-then
-  #openib-1.1
-  if test -f /opt/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/opt/ofed/include/"
-    CMK_LIBDIR="-L/opt/ofed/lib64"
-  fi
-  if test -f /usr/local/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/usr/local/ofed/include/"
-    CMK_LIBDIR="-L/usr/local/ofed/lib64"
-  fi
-  if test -f /usr/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/usr/include/"
-    CMK_LIBDIR="-L/usr/lib64"
-  fi
-fi
-
-CMK_LIBS="$CMK_LIBS -libverbs"
-
diff --git a/src/arch/net-linux-ppc/conv-mach-mx.h b/src/arch/net-linux-ppc/conv-mach-mx.h
deleted file mode 100644 (file)
index a1a67bc..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-#undef CMK_USE_MX
-#define CMK_USE_MX                                         1
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                        1
-
-//#undef CMK_MULTICAST_LIST_USE_COMMON_CODE
-//#define CMK_MULTICAST_LIST_USE_COMMON_CODE              0
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#undef CMK_BARRIER_USE_COMMON_CODE
-#define CMK_BARRIER_USE_COMMON_CODE                        0
-
diff --git a/src/arch/net-linux-ppc/conv-mach-mx.sh b/src/arch/net-linux-ppc/conv-mach-mx.sh
deleted file mode 100644 (file)
index e620fae..0000000
+++ /dev/null
@@ -1,18 +0,0 @@
-#default mx dir
-#guess where the myriexpress.h is installed
-if test -z "$CMK_INCDIR"
-then
-  if test -f /opt/mx/include/myriexpress.h
-  then
-    CMK_INCDIR="-I/opt/mx/include"
-    CMK_LIBDIR="-L/opt/mx/lib32"
-  else
-    echo
-    echo "Please specify path to mx installation in conv-mach-mx.sh!"
-    echo
-    exit 1
-  fi
-fi
-
-
-CMK_LIBS="$CMK_LIBS -lmyriexpress"
diff --git a/src/arch/net-linux-ppc/conv-mach-mx64.h b/src/arch/net-linux-ppc/conv-mach-mx64.h
deleted file mode 100644 (file)
index 6392bde..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-
-#include "conv-mach-mx.h"
diff --git a/src/arch/net-linux-ppc/conv-mach-mx64.sh b/src/arch/net-linux-ppc/conv-mach-mx64.sh
deleted file mode 100644 (file)
index b6268a9..0000000
+++ /dev/null
@@ -1,18 +0,0 @@
-#default mx dir
-#guess where the myriexpress.h is installed
-if test -z "$CMK_INCDIR"
-then
-  if test -f /opt/mx/include/myriexpress.h
-  then
-    CMK_INCDIR="-I/opt/mx/include"
-    CMK_LIBDIR="-L/opt/mx/lib64"
-  else
-    echo
-    echo "Please specify path to mx installation in conv-mach-mx.sh!"
-    echo
-    exit 1
-  fi
-fi
-
-
-CMK_LIBS="$CMK_LIBS -lmyriexpress"
diff --git a/src/arch/net-linux-ppc/conv-mach-smp.h b/src/arch/net-linux-ppc/conv-mach-smp.h
deleted file mode 100644 (file)
index eae84da..0000000
+++ /dev/null
@@ -1,21 +0,0 @@
-
-#define CMK_SMP                                                   1
-
-
-#undef CMK_SHARED_VARS_UNAVAILABLE
-#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
-#define CMK_SHARED_VARS_UNAVAILABLE                        0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#define CMK_MMAP_PROBE                                     0
-
-#define CMK_NOT_USE_TLS_THREAD                             1
-#if ( defined(__xlc__) || defined(__xlC__) ) && CMK_POWER7
-#warning "XLC compiler on Power7 does not support memory fence correctly, pcqueue lock is turned back on."
-#define CMK_PCQUEUE_LOCK                                   1
-#endif
diff --git a/src/arch/net-linux-ppc/conv-mach-smp.sh b/src/arch/net-linux-ppc/conv-mach-smp.sh
deleted file mode 100644 (file)
index 0ada4bc..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-CMK_DEFS=' -D_REENTRANT '
-CMK_LIBS="-lpthread $CMK_LIBS "
-CMK_SMP='1'
diff --git a/src/arch/net-linux-ppc/conv-mach.h b/src/arch/net-linux-ppc/conv-mach.h
deleted file mode 100644 (file)
index 440522f..0000000
+++ /dev/null
@@ -1,60 +0,0 @@
-#ifndef _CONV_MACH_H
-#define _CONV_MACH_H
-
-#define CMK_NETPOLL         1
-
-#define CMK_ASYNC_NOT_NEEDED                               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
-#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
-
-#define CMK_GETPAGESIZE_AVAILABLE                          0
-
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#define CMK_MEMORY_PAGESIZE                                4096
-#define CMK_MEMORY_PROTECTABLE                             0
-
-
-#define CMK_SSH_IS_A_COMMAND                               1
-#define CMK_SSH_NOT_NEEDED                                 0
-
-#define CMK_SHARED_VARS_UNAVAILABLE                        1
-#define CMK_SHARED_VARS_UNIPROCESSOR                       0
-
-#define CMK_THREADS_USE_CONTEXT                            1
-#define CMK_THREADS_USE_PTHREADS                           0
-#define CMK_THREADS_ARE_WIN32_FIBERS                       0
-
-#define CMK_SIGNAL_NOT_NEEDED                              0
-#define CMK_SIGNAL_USE_SIGACTION                           0
-#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
-
-#define CMK_THREADS_REQUIRE_NO_CPV                         0
-#define CMK_THREADS_COPY_STACK                             0
-
-#define CMK_TIMER_USE_RDTSC                                0
-#define CMK_TIMER_USE_GETRUSAGE                            1
-#define CMK_TIMER_USE_SPECIAL                              0
-#define CMK_TIMER_USE_TIMES                                0
-
-#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   0
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     1
-
-
-#define CMK_DEBUG_MODE                                    0 
-#define CMK_WEB_MODE                                       1
-
-#define CMK_LBDB_ON                                       1
-
-#define CMK_STACKSIZE_DEFAULT                              65536
-
-#define __ONESIDED_IMPL 1
-
-#define __ONESIDED_NO_HARDWARE  1
-
-#endif
-
-
diff --git a/src/arch/net-linux-ppc/conv-mach.sh b/src/arch/net-linux-ppc/conv-mach.sh
deleted file mode 100644 (file)
index 66cb883..0000000
+++ /dev/null
@@ -1,42 +0,0 @@
-. $CHARMINC/cc-gcc.sh
-
-isLinux=true
-
-CMK_CC_FLAGS="$CMK_CC_FLAGS -m32"
-CMK_CXX_FLAGS="$CMK_CXX_FLAGS -m32"
-#CMK_LD="$CMK_CC -Wl,--allow-multiple-definition "
-#CMK_LDXX="$CMK_CXX -Wl,--allow-multiple-definition "
-CMK_LD="$CMK_CC "
-CMK_LDXX="$CMK_CXX "
-CMK_LD_FLAGS="$CMK_CC_FLAGS -m32"
-CMK_LDXX_FLAGS="$CMK_CXX_FLAGS -m32"
-CMK_XIOPTS=''
-CMK_QT='generic64-light'
-
-# fortran compiler Absoft or gnu f95
-CMK_CF77='g77 '
-CMK_F77LIBS='-lg2c '
-CMK_CF90=`which xlf90_r 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-# xlf
-  bindir=`dirname $CMK_CF90`
-  libdir="$bindir/../lib"
-  CMK_CF90="$CMK_CF90 -qpic=large -qthreaded -qlanglvl=90std -qwarn64 -qspill=32648 -qsuppress=1513-029:1518-012:1518-059 -qsuffix=f=f90:cpp=F90 "
-  CMK_CF90_FIXED="$CMK_CF90 -qsuffix=f=f:cpp=F -qfixed=132 "
-  CMK_F90LIBS="-L/opt/ibmcmp/xlf/11.1/bin/../../../xlsmp/1.7/lib -L$libdir -lxl -lxlf90 -lxlfmath -lxlopt -lxlsmp"
-  CMK_F90_USE_MODDIR=1
-  CMK_F90_MODINC='-I'
-else
-# gnu f95
-  CMK_CF90=`which f95 2>/dev/null`
-  if test -n "$CMK_CF90"
-  then
-    CMK_FPP='cpp -P -CC'
-    CMK_CF90="$CMK_CF90 -fpic -fautomatic -fdollar-ok "
-    CMK_CF90_FIXED="$CMK_CF90 -ffixed-form "
-    CMK_F90LIBS='-lgfortran '
-    CMK_F90_USE_MODDIR=1
-    CMK_F90_MODINC='-I'
-  fi
-fi
diff --git a/src/arch/net-linux-x86_64/conv-mach-clustermatic.h b/src/arch/net-linux-x86_64/conv-mach-clustermatic.h
deleted file mode 100644 (file)
index 621b5ac..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-#define CMK_BPROC                                          1
-
-#undef CMK_ASYNC_NOT_NEEDED
-#undef CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN
-#define CMK_ASYNC_NOT_NEEDED                               1
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 0
-
-#undef CMK_SSH_NOT_NEEDED
-#define CMK_SSH_NOT_NEEDED                                1
diff --git a/src/arch/net-linux-x86_64/conv-mach-clustermatic.sh b/src/arch/net-linux-x86_64/conv-mach-clustermatic.sh
deleted file mode 100644 (file)
index eaded2c..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-CMK_LIBS="$CMK_LIBS -lbproc"
-CMK_NATIVE_LIBS="$CMK_NATIVE_LIBS -lbproc"
-CMK_BPROC=true
diff --git a/src/arch/net-linux-x86_64/conv-mach-ibud.h b/src/arch/net-linux-x86_64/conv-mach-ibud.h
deleted file mode 100644 (file)
index 9905541..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-#undef CMK_USE_IBUD
-#define CMK_USE_IBUD                           1
-
-
-// FIXME: See if I need to include any of these flags
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                            1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_SPANNING_TREE
-#define CMK_BROADCAST_SPANNING_TREE 0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE  1
-
-#undef CMK_DIRECT                 
-#define CMK_DIRECT                                        1
-
-#undef CMK_DISABLE_SYNC
-#define CMK_DISABLE_SYNC       1
-
-
diff --git a/src/arch/net-linux-x86_64/conv-mach-ibud.sh b/src/arch/net-linux-x86_64/conv-mach-ibud.sh
deleted file mode 100644 (file)
index 7aa875d..0000000
+++ /dev/null
@@ -1,18 +0,0 @@
-#default ibverbs path for openib
-if test -z "$CMK_INCDIR"
-then
-  #openib-1.1
-  if test -f /opt/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/opt/ofed/include/"
-    CMK_LIBDIR="-L/opt/ofed/lib64"
-       fi
-  if test -f /usr/local/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/usr/local/ofed/include/"
-    CMK_LIBDIR="-L/usr/local/ofed/lib64"
-       fi
-fi
-
-CMK_LIBS="$CMK_LIBS -libverbs"
-
diff --git a/src/arch/net-linux-x86_64/conv-mach-ibverbs.h b/src/arch/net-linux-x86_64/conv-mach-ibverbs.h
deleted file mode 100644 (file)
index a5a675f..0000000
+++ /dev/null
@@ -1,33 +0,0 @@
-#undef CMK_USE_IBVERBS
-#define CMK_USE_IBVERBS                                1
-
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                            1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-/*
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-*/
-
-#undef CMK_BROADCAST_SPANNING_TREE
-#define CMK_BROADCAST_SPANNING_TREE 0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE  1
-
-#undef CMK_DIRECT                 
-#define CMK_DIRECT                                        1
-
-#undef CMK_DISABLE_SYNC
-#define CMK_DISABLE_SYNC       1
-
-
-#undef CMK_IBVERBS_FAST_START
-#define CMK_IBVERBS_FAST_START                         0
diff --git a/src/arch/net-linux-x86_64/conv-mach-ibverbs.sh b/src/arch/net-linux-x86_64/conv-mach-ibverbs.sh
deleted file mode 100644 (file)
index 7aa875d..0000000
+++ /dev/null
@@ -1,18 +0,0 @@
-#default ibverbs path for openib
-if test -z "$CMK_INCDIR"
-then
-  #openib-1.1
-  if test -f /opt/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/opt/ofed/include/"
-    CMK_LIBDIR="-L/opt/ofed/lib64"
-       fi
-  if test -f /usr/local/ofed/include/infiniband/verbs.h
-  then
-    CMK_INCDIR="-I/usr/local/ofed/include/"
-    CMK_LIBDIR="-L/usr/local/ofed/lib64"
-       fi
-fi
-
-CMK_LIBS="$CMK_LIBS -libverbs"
-
diff --git a/src/arch/net-linux-x86_64/conv-mach-pxshm.h b/src/arch/net-linux-x86_64/conv-mach-pxshm.h
deleted file mode 100644 (file)
index f534175..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_PXSHM
-#define CMK_USE_PXSHM                  1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux-x86_64/conv-mach-pxshm.sh b/src/arch/net-linux-x86_64/conv-mach-pxshm.sh
deleted file mode 100644 (file)
index 7ef2c6e..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS -lrt"
diff --git a/src/arch/net-linux-x86_64/conv-mach-smp.h b/src/arch/net-linux-x86_64/conv-mach-smp.h
deleted file mode 100644 (file)
index 5e89297..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-
-#define CMK_SMP                                                   1
-
-
-#undef CMK_SHARED_VARS_UNAVAILABLE
-#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
-#define CMK_SHARED_VARS_UNAVAILABLE                        0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
-
-#undef CMK_MALLOC_USE_GNU_MALLOC
-#undef CMK_MALLOC_USE_OS_BUILTIN
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-/*#define CMK_MMAP_PROBE                                     1 */
-
-/*#define CMK_PCQUEUE_LOCK                                1 */
-/*#define CMK_USE_MFENCE                                          1 */
-/* Replaced by CMK_NOT_USE_TLS_THREAD as default */
-/*#define CMK_USE_TLS_THREAD                                 1*/
diff --git a/src/arch/net-linux-x86_64/conv-mach-smp.sh b/src/arch/net-linux-x86_64/conv-mach-smp.sh
deleted file mode 100644 (file)
index 0ada4bc..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-CMK_DEFS=' -D_REENTRANT '
-CMK_LIBS="-lpthread $CMK_LIBS "
-CMK_SMP='1'
diff --git a/src/arch/net-linux-x86_64/conv-mach-sysvshm.h b/src/arch/net-linux-x86_64/conv-mach-sysvshm.h
deleted file mode 100644 (file)
index 1847dd0..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_SYSVSHM
-#define CMK_USE_SYSVSHM                        1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       0
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux-x86_64/conv-mach-sysvshm.sh b/src/arch/net-linux-x86_64/conv-mach-sysvshm.sh
deleted file mode 100644 (file)
index 40a223c..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS "
diff --git a/src/arch/net-linux-x86_64/conv-mach.h b/src/arch/net-linux-x86_64/conv-mach.h
deleted file mode 100644 (file)
index c6c4ce6..0000000
+++ /dev/null
@@ -1,63 +0,0 @@
-
-#ifndef _CONV_MACH_H
-#define _CONV_MACH_H
-
-#define CMK_NETPOLL         1
-
-#define CMK_AMD64                                         1
-
-#define CMK_ASYNC_NOT_NEEDED                               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
-#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
-
-#define CMK_DLL_CC  "g++ -shared -O3 -o "
-
-#define CMK_GETPAGESIZE_AVAILABLE                          1
-
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#define CMK_MEMORY_PAGESIZE                                4096
-#define CMK_MEMORY_PROTECTABLE                             0
-
-
-#define CMK_SSH_IS_A_COMMAND                               1
-#define CMK_SSH_NOT_NEEDED                                 0
-
-#define CMK_SHARED_VARS_UNAVAILABLE                        1
-#define CMK_SHARED_VARS_UNIPROCESSOR                       0
-
-#define CMK_THREADS_USE_CONTEXT                            1
-#define CMK_THREADS_USE_PTHREADS                           0
-#define CMK_THREADS_ARE_WIN32_FIBERS                       0
-
-#define CMK_SIGNAL_NOT_NEEDED                              0
-#define CMK_SIGNAL_USE_SIGACTION                           0
-#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
-
-#define CMK_THREADS_REQUIRE_NO_CPV                         0
-#define CMK_THREADS_COPY_STACK                             0
-
-#define CMK_TIMER_USE_RDTSC                                0
-#define CMK_TIMER_USE_GETRUSAGE                            1
-#define CMK_TIMER_USE_SPECIAL                              0
-#define CMK_TIMER_USE_TIMES                                0
-
-
-#define CMK_64BIT    1
-
-#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   0
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     1
-
-
-#define CMK_DEBUG_MODE                                    0 
-#define CMK_WEB_MODE                                       1
-
-#define CMK_LBDB_ON                                       1
-
-
-#endif
-
-
diff --git a/src/arch/net-linux-x86_64/conv-mach.sh b/src/arch/net-linux-x86_64/conv-mach.sh
deleted file mode 100644 (file)
index b52f460..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-. $CHARMINC/cc-gcc.sh
-
-#CMK_DEFS="$CMK_DEFS -DHAVE_USR_INCLUDE_MALLOC_H=1 "
-CMK_XIOPTS=''
-CMK_WARNINGS_ARE_ERRORS='-Werror'
-CMK_CXX_OPTIMIZE='-O3'
-
-CMK_QT='generic64-light'
-
-CMK_CF90=`which f95 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-#    CMK_FPP="cpp -P -CC"
-#    CMK_CF90="$CMK_CF90 -fpic -fautomatic -fdollar-ok "
-#    CMK_CF90_FIXED="$CMK_CF90 -ffixed-form "
-#    CMK_F90LIBS="-lgfortran "
-#    CMK_F90_USE_MODDIR=1
-#    CMK_F90_MODINC="-I"
-#    CMK_MOD_NAME_ALLCAPS=
-#    CMK_MOD_EXT="mod"
-    . $CHARMINC/conv-mach-gfortran.sh
-else
-    CMK_CF77='g77 '
-    CMK_CF90='f90 '
-    CMK_CF90_FIXED="$CMK_CF90 -W132 "
-    CMK_F90LIBS='-L/usr/absoft/lib -L/opt/absoft/lib -lf90math -lfio -lU77 -lf77math '
-    CMK_F77LIBS='-lg2c '
-    CMK_F90_USE_MODDIR=1
-    CMK_F90_MODINC='-p'
-fi
diff --git a/src/arch/net-linux/conv-mach-clustermatic.h b/src/arch/net-linux/conv-mach-clustermatic.h
deleted file mode 100644 (file)
index 9ec1fc0..0000000
+++ /dev/null
@@ -1 +0,0 @@
-#include "conv-mach-scyld.h"
diff --git a/src/arch/net-linux/conv-mach-clustermatic.sh b/src/arch/net-linux/conv-mach-clustermatic.sh
deleted file mode 100644 (file)
index 2a737f6..0000000
+++ /dev/null
@@ -1,2 +0,0 @@
-
-. $CHARMINC/conv-mach-scyld.sh
diff --git a/src/arch/net-linux/conv-mach-gm.h b/src/arch/net-linux/conv-mach-gm.h
deleted file mode 100644 (file)
index aa73218..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-
-#undef CMK_USE_GM
-#define CMK_USE_GM                                         1
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                        1
-
-#undef CMK_MULTICAST_LIST_USE_COMMON_CODE
-#define CMK_MULTICAST_LIST_USE_COMMON_CODE                0
-
-#undef __ONESIDED_IMPL
-#define __ONESIDED_IMPL 1
-
-#undef __ONESIDED_GM_HARDWARE
-#undef __ONESIDED_NO_HARDWARE
-#define __ONESIDED_GM_HARDWARE  1
-
-#undef CMK_BARRIER_USE_COMMON_CODE
-#define CMK_BARRIER_USE_COMMON_CODE                        0
diff --git a/src/arch/net-linux/conv-mach-gm.sh b/src/arch/net-linux/conv-mach-gm.sh
deleted file mode 100644 (file)
index 796f41c..0000000
+++ /dev/null
@@ -1,23 +0,0 @@
-#default gm dir
-#guess where the gm.h is installed
-if test -z "$CMK_INCDIR"
-then
-  # gm ver 1.0
-  if test -f /usr/gm/include/gm.h
-  then
-    CMK_INCDIR="-I /usr/gm/include"
-    CMK_LIBDIR="-L /usr/gm/lib"
-  # gm ver 2.0
-  elif test -f /opt/gm/include/gm.h
-  then
-    CMK_INCDIR="-I /opt/gm/include"
-    CMK_LIBDIR="-L /opt/gm/lib"
-  elif test -f /usr/local/gm/include/gm.h
-  then
-    CMK_INCDIR="-I /usr/local/gm/include"
-    CMK_LIBDIR="-L /usr/local/gm/lib"
-  fi
-fi
-
-
-CMK_LIBS="$CMK_LIBS -lgm"
diff --git a/src/arch/net-linux/conv-mach-mx.h b/src/arch/net-linux/conv-mach-mx.h
deleted file mode 100644 (file)
index 490d9dd..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-
-#undef CMK_USE_MX
-#define CMK_USE_MX                                         1
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL                                        1
-
-//#undef CMK_MULTICAST_LIST_USE_COMMON_CODE
-//#define CMK_MULTICAST_LIST_USE_COMMON_CODE              0
diff --git a/src/arch/net-linux/conv-mach-mx.sh b/src/arch/net-linux/conv-mach-mx.sh
deleted file mode 100644 (file)
index 4b4480d..0000000
+++ /dev/null
@@ -1,17 +0,0 @@
-#default mx dir
-#guess where the myriexpress.h is installed
-if test -z "$CMK_INCDIR"
-then
-  if test -f /turing/software/mx/include/myriexpress.h
-  then
-    CMK_INCDIR="-I/turing/software/mx/include"
-    CMK_LIBDIR="-L/turing/software/mx/lib"
-  elif test -f /opt/mx/include/myriexpress.h
-  then
-    CMK_INCDIR="-I/opt/mx/include"
-    CMK_LIBDIR="-L/opt//mx/lib"
-  fi
-fi
-
-
-CMK_LIBS="$CMK_LIBS -lmyriexpress"
diff --git a/src/arch/net-linux/conv-mach-pxshm.h b/src/arch/net-linux/conv-mach-pxshm.h
deleted file mode 100644 (file)
index 63e2f5a..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_PXSHM
-#define CMK_USE_PXSHM                  1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       1
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux/conv-mach-pxshm.sh b/src/arch/net-linux/conv-mach-pxshm.sh
deleted file mode 100644 (file)
index 7ef2c6e..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS -lrt"
diff --git a/src/arch/net-linux/conv-mach-scyld.h b/src/arch/net-linux/conv-mach-scyld.h
deleted file mode 100644 (file)
index 621b5ac..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-#define CMK_BPROC                                          1
-
-#undef CMK_ASYNC_NOT_NEEDED
-#undef CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN
-#define CMK_ASYNC_NOT_NEEDED                               1
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 0
-
-#undef CMK_SSH_NOT_NEEDED
-#define CMK_SSH_NOT_NEEDED                                1
diff --git a/src/arch/net-linux/conv-mach-scyld.sh b/src/arch/net-linux/conv-mach-scyld.sh
deleted file mode 100644 (file)
index eaded2c..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-CMK_LIBS="$CMK_LIBS -lbproc"
-CMK_NATIVE_LIBS="$CMK_NATIVE_LIBS -lbproc"
-CMK_BPROC=true
diff --git a/src/arch/net-linux/conv-mach-smp.h b/src/arch/net-linux/conv-mach-smp.h
deleted file mode 100644 (file)
index a3a5e03..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-
-#define CMK_SMP                                                   1
-
-
-#undef CMK_SHARED_VARS_UNAVAILABLE
-#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
-#define CMK_SHARED_VARS_UNAVAILABLE                        0
-#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
diff --git a/src/arch/net-linux/conv-mach-smp.sh b/src/arch/net-linux/conv-mach-smp.sh
deleted file mode 100644 (file)
index 6f9e735..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-CMK_DEFS=' -D_REENTRANT '
-CMK_LIBS="-lpthread $CMK_LIBS "
-CMK_QT='linux-smp-32'
-CMK_SMP='1'
diff --git a/src/arch/net-linux/conv-mach-sysvshm.h b/src/arch/net-linux/conv-mach-sysvshm.h
deleted file mode 100644 (file)
index f1a478b..0000000
+++ /dev/null
@@ -1,14 +0,0 @@
-#undef CMK_USE_SYSVSHM
-#define CMK_USE_SYSVSHM                        1
-
-#undef CMK_IMMEDIATE_MSG
-#define CMK_IMMEDIATE_MSG       1
-
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP  0
-
-#undef CMK_NETPOLL
-#define CMK_NETPOLL   1
diff --git a/src/arch/net-linux/conv-mach-sysvshm.sh b/src/arch/net-linux/conv-mach-sysvshm.sh
deleted file mode 100644 (file)
index 40a223c..0000000
+++ /dev/null
@@ -1 +0,0 @@
-CMK_LIBS="$CMK_LIBS "
diff --git a/src/arch/net-linux/conv-mach.h b/src/arch/net-linux/conv-mach.h
deleted file mode 100644 (file)
index b1b7d2d..0000000
+++ /dev/null
@@ -1,63 +0,0 @@
-#ifndef _CONV_MACH_H
-#define _CONV_MACH_H
-
-#define CMK_NETPOLL         1
-
-#define CMK_ASYNC_NOT_NEEDED                               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
-#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
-#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
-#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
-
-#define CMK_DLL_CC  "g++ -shared -O3 -o "
-
-#define CMK_GETPAGESIZE_AVAILABLE                          0
-
-#define CMK_MALLOC_USE_GNU_MALLOC                          0
-#define CMK_MALLOC_USE_OS_BUILTIN                          1
-
-#define CMK_MEMORY_PAGESIZE                                4096
-#define CMK_MEMORY_PROTECTABLE                             1
-
-
-#define CMK_SSH_IS_A_COMMAND                               1
-#define CMK_SSH_NOT_NEEDED                                 0
-
-#define CMK_SHARED_VARS_UNAVAILABLE                        1
-#define CMK_SHARED_VARS_UNIPROCESSOR                       0
-
-#define CMK_THREADS_USE_CONTEXT                            0
-#define CMK_THREADS_USE_PTHREADS                           0
-#define CMK_THREADS_ARE_WIN32_FIBERS                       0
-
-#define CMK_SIGNAL_NOT_NEEDED                              0
-#define CMK_SIGNAL_USE_SIGACTION                           0
-#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
-
-#define CMK_THREADS_REQUIRE_NO_CPV                         0
-#define CMK_THREADS_COPY_STACK                             0
-
-#define CMK_TIMER_USE_RDTSC                                0
-#define CMK_TIMER_USE_GETRUSAGE                            1
-#define CMK_TIMER_USE_SPECIAL                              0
-#define CMK_TIMER_USE_TIMES                                0
-
-
-#define CMK_64BIT    0
-
-#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   0
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     1
-
-
-#define CMK_DEBUG_MODE                                    0 
-#define CMK_WEB_MODE                                       1
-
-#define CMK_LBDB_ON                                       1
-
-#define __ONESIDED_IMPL 1
-
-#define __ONESIDED_NO_HARDWARE  1
-
-#endif
-
-
diff --git a/src/arch/net-linux/conv-mach.sh b/src/arch/net-linux/conv-mach.sh
deleted file mode 100644 (file)
index 0bae00f..0000000
+++ /dev/null
@@ -1,55 +0,0 @@
-. $CHARMINC/cc-gcc.sh
-
-CMK_CPP_C_FLAGS="$CMK_CPP_C_FLAGS -m32"
-CMK_CC_FLAGS="$CMK_CC_FLAGS -m32"
-CMK_CXX_FLAGS="$CMK_CXX_FLAGS -m32"
-#CMK_LD="$CMK_CC -Wl,--allow-multiple-definition "
-#CMK_LDXX="$CMK_CXX -Wl,--allow-multiple-definition "
-CMK_LD="$CMK_CC "
-CMK_LDXX="$CMK_CXX "
-CMK_LD_FLAGS="$CMK_CC_FLAGS "
-CMK_LDXX_FLAGS="$CMK_CXX_FLAGS "
-
-CMK_XIOPTS=''
-CMK_QT='i386-gcc'
-
-CMK_CF90=`which f95 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-    . $CHARMINC/conv-mach-gfortran.sh
-else
-
-# fortran compiler f77 or gnu f95
-CMK_CF77=`which g77 2>/dev/null`
-if test -z "$CMK_CF77"
-then
-  CMK_CF77=`which f77 2>/dev/null`
-fi
-if test -n "$CMK_CF77"
-then
-  CMK_F77LIBS='-lg2c '
-fi
-
-CMK_CF90=`which f90 2>/dev/null`
-if test -n "$CMK_CF90"
-then
-# absoft
-  CMK_CF90_FIXED="$CMK_CF90 -W132 "
-  CMK_F90LIBS='-L/usr/absoft/lib -L/opt/absoft/lib -lf90math -lfio -lU77 -lf77math '
-  CMK_F90_USE_MODDIR=1
-  CMK_F90_MODINC='-p'
-else
-# gnu f95
-  CMK_CF90=`which f95 2>/dev/null`
-  if test -n "$CMK_CF90"
-  then
-    CMK_FPP='cpp -P -CC'
-    CMK_CF90="$CMK_CF90 -fpic -fautomatic -fdollar-ok "
-    CMK_CF90_FIXED="$CMK_CF90 -ffixed-form "
-    CMK_F90LIBS='-lgfortran '
-    CMK_F90_USE_MODDIR=1
-    CMK_F90_MODINC='-I'
-  fi
-fi
-
-fi
diff --git a/src/arch/net/Makefile.machine b/src/arch/net/Makefile.machine
deleted file mode 100644 (file)
index ff575d7..0000000
+++ /dev/null
@@ -1,5 +0,0 @@
-$(L)/libconv-machine.a: machine.c machine-dgram.c machine-tcp.c machine-eth.c machine-mx.c machine-gm.c machine.h machine-smp.c machine-ammasso.h machine-ammasso.c machine-ibverbs.c machine-pxshm.c machine-sysvshm.c conv-onesided.c immediate.c machine-recover.c machine-commthd-util.c $(CVHEADERS)
-
-charmrun: sockRoutines.c sockRoutines.h ccs-server.c ccs-server.h sockRoutines-seq.o $(L)/libck.a $(CVLIBS)
-       if [ ! -f charmrun ]; then \
-                       (cd charmrun-src ; $(MAKE) OPTS='$(OPTS) -Wno-error' -j1) ; fi
diff --git a/src/arch/net/conv-common.h b/src/arch/net/conv-common.h
deleted file mode 100644 (file)
index 3bc78cc..0000000
+++ /dev/null
@@ -1,49 +0,0 @@
-
-#define CMK_NET_VERSION                                           1
-
-#define CMK_CMIPRINTF_IS_A_BUILTIN                         1
-
-#define CMK_CMIDELIVERS_USE_COMMON_CODE                    1
-
-#define CMK_HANDLE_SIGUSR                                  1
-
-#define CMK_MULTICAST_GROUP_TYPE                struct { unsigned pe, id; }
-#define CMK_MULTICAST_DEF_USE_COMMON_CODE                  1
-#define CMK_MULTICAST_LIST_USE_COMMON_CODE                 1
-#define CMK_MULTICAST_GROUP_USE_COMMON_CODE                1
-
-/* the first 4 fields of the header are defined in machine-dgram.c
-   and are used for the udp retransmission protocol implementation.
-   Parameters stratid and root are for the communication library.
-   Stratid is the stratgy id and root is used in the broadcast.
-*/
-#define CMK_MSG_HEADER_BASIC   CMK_MSG_HEADER_EXT
-#define CMK_MSG_HEADER_EXT_    CmiUInt2 d0,d1,d2,d3,d4,d5,hdl,stratid,root,xhdl,info,redID;
-#define CMK_MSG_HEADER_EXT       { CMK_MSG_HEADER_EXT_ }
-#define CMK_MSG_HEADER_BIGSIM_  { CMK_MSG_HEADER_EXT_ CMK_BIGSIM_FIELDS }
-
-#define CMK_SPANTREE_MAXSPAN                               4
-#define CMK_SPANTREE_USE_COMMON_CODE                       1
-
-#define CMK_VECTOR_SEND_USES_COMMON_CODE                   1
-
-#define CMK_IMMEDIATE_MSG                                 1
-
-#define CMK_PERSISTENT_COMM                               0
-
-#define CMK_OBJECT_QUEUE_AVAILABLE                        0
-#define CMK_USE_SPECIAL_MESSAGE_QUEUE_CHECK                1
-
-#define CMK_BROADCAST_SPANNING_TREE                               0
-#define CMK_BROADCAST_HYPERCUBE                                   1
-
-#define CMK_BARRIER_USE_COMMON_CODE                        1
-
-#define CMK_MACHINE_PROGRESS_DEFINED                       1
-
-#define NODE_0_IS_CONVHOST                                 0
-
-/* call cpu timer for LB */
-#define CMK_LB_CPUTIMER                                           0
-
-
diff --git a/src/arch/net/conv-mach-local.h b/src/arch/net/conv-mach-local.h
deleted file mode 100644 (file)
index 9c549aa..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-#undef CMK_CHARMRUN_LOCAL
-#define CMK_CHARMRUN_LOCAL 1
-
diff --git a/src/arch/net/conv-mach-local.sh b/src/arch/net/conv-mach-local.sh
deleted file mode 100644 (file)
index 6f793a9..0000000
+++ /dev/null
@@ -1 +0,0 @@
-COMMENT="Changes charmrun to always start node-programs using fork, not ssh (like ++local)"
diff --git a/src/arch/net/conv-mach-syncft.h b/src/arch/net/conv-mach-syncft.h
deleted file mode 100644 (file)
index ad2441c..0000000
+++ /dev/null
@@ -1,17 +0,0 @@
-
-#undef CMK_MSG_HEADER_BASIC
-#undef CMK_MSG_HEADER_EXT_
-//#undef CMK_MSG_HEADER_EXT
-//#undef CMK_MSG_HEADER_BIGSIM_
-/* expand the header to store the restart phase counter(pn) */
-#define CMK_MSG_HEADER_BASIC   CMK_MSG_HEADER_EXT
-#define CMK_MSG_HEADER_EXT_    CmiUInt2 d0,d1,d2,d3,d4,d5,hdl,pn,d8,stratid,root,xhdl,info,dd,redID,pad2;
-//#define CMK_MSG_HEADER_EXT    { CMK_MSG_HEADER_EXT_ }
-//#define CMK_MSG_HEADER_BIGSIM_    { CmiUInt2 d0,d1,d2,d3,d4,d5,hdl,xhdl,pn,info; int nd, n; double rt; CmiInt2 tID; CmiUInt2 hID; char t; int msgID; int srcPe;}
-//#define CMK_MSG_HEADER_BIGSIM_  { CMK_MSG_HEADER_EXT_ CMK_BIGSIM_FIELDS }
-
-#define CmiGetRestartPhase(m)       ((((CmiMsgHeaderExt*)m)->pn))
-
-#define __FAULT__                                         1
-
-#define CMK_MEM_CHECKPOINT                                1
diff --git a/src/arch/net/conv-mach-tcp.h b/src/arch/net/conv-mach-tcp.h
deleted file mode 100644 (file)
index 7d44739..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-
-#undef CMK_USE_TCP
-#define CMK_USE_TCP                                         1
-
-#if CMK_SMP
-#undef CMK_USE_POLL
-#endif
-
-#define CMK_NETPOLL                                        1
-
-/*
-#undef CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN
-#undef CMK_ASYNC_NOT_NEEDED
-#define CMK_ASYNC_NOT_NEEDED                               1
-*/
-
-#undef CMK_BROADCAST_SPANNING_TREE
-#undef CMK_BROADCAST_HYPERCUBE
-#define CMK_BROADCAST_SPANNING_TREE                                0
-#define CMK_BROADCAST_HYPERCUBE                                    1
diff --git a/src/arch/net/conv-mach-tcp.sh b/src/arch/net/conv-mach-tcp.sh
deleted file mode 100644 (file)
index e69de29..0000000
diff --git a/src/arch/net/machine-ammasso.c b/src/arch/net/machine-ammasso.c
deleted file mode 100644 (file)
index bf7970d..0000000
+++ /dev/null
@@ -1,2966 +0,0 @@
-/**
- ** Ammasso implementation of Converse NET version
- ** Contains Ammasso specific
- ** code for:
- ** CmiMachineInit()
- ** CmiCommunicationInit()
- ** CmiNotifyIdle()
- ** DeliverViaNetwork()
- ** CommunicationServer()
- ** CmiMachineExit()
- **
- ** Written By:    Isaac Dooley      idooley2@uiuc.edu
- ** 03/12/05       Esteban Pauli     etpauli2@uiuc.edu
- **                Filippo Gioachin  gioachin@uiuc.edu
- **                David Kunzman     kunzman2@uiuc.edu
- **
- ** Change Log:
- **   03/12/05 : DMK : Initial Version
- **   04/30/05 : Filippo : Revised Version
- **
- ** Todo List:
- **
- **/
-
-#ifndef ALIGN8
-#define ALIGN8(x)   (int)((~7)&((x)+7))
-#endif
-
-/* DYNAMIC ALLOCATOR: Limit of the allowed pinned memory */
-#define MAX_PINNED_MEMORY   100000000
-/* DYNAMIC ALLOCATOR END */
-
-#define WASTE_TIME 600
-// In order to use CC_POST_CHECK, the last argument to cc_qp_post_sq must be "nWR"
-#define CC_POST_CHECK(routine,args,nodeTo) {\
-        int retry=10000000; \
-        while (ammasso_check_post_err(routine args, #routine, __LINE__, &nWR, nodeTo, retry) == 1) { \
-          int i; \
-          retry += WASTE_TIME; \
-          for (i=0; i<=WASTE_TIME; ++i) { retry --; } \
-        } \
-      }
-
-#define CC_CHECK(routine,args) \
-        ammasso_check_err(routine args, #routine, __LINE__);
-
-static void ammasso_check_err(cc_status_t returnCode,const char *routine,int line) {
-  if (returnCode!=CC_OK) {
-    char buf[128];
-    char *errMsg = cc_status_to_string(returnCode);
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE3(5,"Fatal CC error while executing %s at %s:%d\n", routine, __FILE__, line);
-    MACHSTATE2(5,"  Description: %d, %s\n",returnCode,errMsg);
-    sprintf(buf,"Fatal CC error while executing %s at %s:%d\n"
-           "  Description: %d, %s\n", routine, __FILE__, line,returnCode,errMsg);
-    CmiAbort(buf);
-  }
-}
-
-// We pass the pointer used in the cc_qp_post_sq call as last parameter, since
-// when this function is called, cc_pq_post_sq has already been called
-static int ammasso_check_post_err(cc_status_t returnCode,const char *routine,int line, int *nWR, int nodeTo, int retry) {
-  if (returnCode == CCERR_TOO_MANY_WRS_POSTED && *nWR != 1 && retry>0) {
-    cc_wc_t wc;
-    // drain the send completion queue and retry
-    while (cc_cq_poll(contextBlock->rnic, nodes[nodeTo].send_cq, &wc) == CC_OK) {
-      MACHSTATE1(5, "Error posting send request - INFO: Send completed with node %d... now waiting for acknowledge...", nodeTo);
-    }
-    MACHSTATE(5, "Error posting send request - Retrying...");
-    return 1;
-  }
-
-  if (returnCode != CC_OK || *nWR != 1) {
-    char buf[128];
-    char *errMsg = cc_status_to_string(returnCode);
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE3(5,"Fatal CC error while executing %s at %s:%d\n", routine, __FILE__, line);
-    MACHSTATE3(5,"  Description: %d, %s (nWR = %d)\n",returnCode,errMsg,nWR);
-    sprintf(buf,"Fatal CC error while executing %s at %s:%d\n"
-           "  Description: %d, %s (nWR = %d)\n", routine, __FILE__, line,returnCode,errMsg,*nWR);
-    CmiAbort(buf);
-  }
-  return 0;
-}
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Function ProtoTypes /////////////////////////////////////////////////////////////////////////////
-
-void CmiMachineInit(char **argv);
-
-void CmiAmmassoNodeAddressesStoreHandler(int pe, struct sockaddr_in *addr, int port);
-
-void AmmassoDoIdle(void);
-void CmiNotifyIdle(void);
-static CmiIdleState* CmiNotifyGetState(void);
-static void CmiNotifyBeginIdle(CmiIdleState *s);
-static void CmiNotifyStillIdle(CmiIdleState *s);
-
-void sendAck(OtherNode node);
-AmmassoToken *getQPSendToken(OtherNode node);
-int sendDataOnQP(char* data, int len, OtherNode node, char flags);
-void DeliverViaNetwork(OutgoingMsg msg, OtherNode otherNode, int rank, unsigned int broot, int copy);
-static void CommunicationServer(int withDelayMs, int where);
-void CmiMachineExit(void);
-
-void AsynchronousEventHandler(cc_rnic_handle_t rnic, cc_event_record_t *eventRecord, void *cb);
-void CheckRecvBufForMessage(OtherNode node);
-//void CompletionEventHandler(cc_rnic_handle_t rnic, cc_cq_handle_t cq, void *cb);
-//void CompletionEventHandlerWithAckFlag(cc_rnic_handle_t rnic, cc_cq_handle_t cq, void *cb, int breakOnAck);
-
-void CmiAmmassoOpenQueuePairs(void);
-
-void processAmmassoControlMessage(char* msg, int len, Tailer *tail, OtherNode from);
-int ProcessMessage(char* msg, int len, Tailer *tail, OtherNode from);
-
-OtherNode getNodeFromQPId(cc_qp_id_t qp_id);
-OtherNode getNodeFromQPHandle(cc_qp_handle_t qp);
-
-void establishQPConnection(OtherNode node, int reuseQPFlag);
-void reestablishQPConnection(OtherNode node);
-void closeQPConnection(OtherNode node, int destroyQPFlag);
-
-void BufferAlloc(int n);
-void TokenAlloc(int n);
-void RequestTokens(OtherNode node, int n);
-void GrantTokens(OtherNode node, int n);
-void RequestReleaseTokens(OtherNode node, int n);
-void ReleaseTokens(OtherNode node, int n);
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Function Bodies /////////////////////////////////////////////////////////////////////////////////
-
-/* Callbacks used by the DYNAMIC ALLOCATOR */
-void AllocatorCheck (void) {
-  int i, limit;
-  char buf[24];
-  for (i=0; i<contextBlock->numNodes; ++i) {
-    if (i==contextBlock->myNode) continue;
-    limit = nodes[i].num_sendTokens - nodes[i].max_used_tokens - 10;
-    CmiPrintf("[%d] AllocatorCheck called: node %d, limit %d\n",CmiMyPe(),i,limit);
-    if (limit > 0) {
-      ReleaseTokens(&nodes[i], limit);
-      CmiPrintf("[%d] Releasing %d tokens to %d\n", CmiMyPe(), limit, i);
-      nodes[i].max_used_tokens = 0;
-    }
-  }
-}
-/* DYNAMIC ALLOCATOR END */
-
-void BufferAlloc(int n) {
-  int i;
-  char buf[128];
-  cc_stag_index_t newStagIndex;
-  AmmassoBuffer *newBuffers;
-
-  MACHSTATE1(3, "Allocating %d new Receive Buffers",n);
-
-  // Try to allocate the memory for n receiving buffers
-  newBuffers = (AmmassoBuffer*) CmiAlloc(n*sizeof(AmmassoBuffer));
-
-  if (newBuffers == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "BufferAlloc() - ERROR: Unable to allocate memory for RECV buffers");
-    sprintf(buf, "BufferAlloc() - ERROR: Unable to allocate memory for RECV buffers");
-    CmiAbort(buf);
-  }
-
-  contextBlock->pinnedMemory += n*sizeof(AmmassoBuffer);
-  CC_CHECK(cc_nsmr_register_virt,(contextBlock->rnic,
-                                 CC_ADDR_TYPE_VA_BASED,
-                                 (cc_byte_t*)newBuffers,
-                                 n*sizeof(AmmassoBuffer),
-                                 contextBlock->pd_id,
-                                 0, 0,
-                                 CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE | CC_ACF_REMOTE_WRITE,
-                                 &newStagIndex)
-          );
-
-  for (i=0; i<n; ++i) {
-    newBuffers[i].tail.length = 0;
-    newBuffers[i].next = &(newBuffers[i+1]);
-    newBuffers[i].stag = newStagIndex;
-  }
-  newBuffers[n-1].next = NULL;
-  if (contextBlock->freeRecvBuffers == NULL) {
-    contextBlock->freeRecvBuffers = newBuffers;
-  } else {
-    contextBlock->last_freeRecvBuffers->next = newBuffers;
-  }
-  contextBlock->last_freeRecvBuffers = &newBuffers[n-1];
-  contextBlock->num_freeRecvBuffers += n;
-}
-
-void TokenAlloc(int n) {
-  int i;
-  char buf[128];
-  cc_stag_index_t newStagIndex;
-  AmmassoToken *sendToken, *tokenScanner;
-  cc_data_addr_t *sendSgl;
-  AmmassoBuffer *sendBuffer;
-
-  MACHSTATE1(3, "Allocating %d new Tokens",n);
-
-  // Try to allocate the memory for n sending buffers
-  sendBuffer = (AmmassoBuffer*) CmiAlloc(n*sizeof(AmmassoBuffer));
-
-  if (sendBuffer == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "TokenAlloc() - ERROR: Unable to allocate memory for SEND buffers");
-    sprintf(buf, "TokenAlloc() - ERROR: Unable to allocate memory for SEND buffers");
-    CmiAbort(buf);
-  }
-
-  contextBlock->pinnedMemory += n*sizeof(AmmassoBuffer);
-  CC_CHECK(cc_nsmr_register_virt,(contextBlock->rnic,
-                                 CC_ADDR_TYPE_VA_BASED,
-                                 (cc_byte_t*)sendBuffer,
-                                 n*sizeof(AmmassoBuffer),
-                                 contextBlock->pd_id,
-                                 0, 0,
-                                 CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE,
-                                 &newStagIndex)
-          );
-
-  // Allocate the send tokens
-  sendToken = (AmmassoToken*) CmiAlloc(n*ALIGN8(sizeof(AmmassoToken)));
-
-  if (sendToken == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "TokenAlloc() - ERROR: Unable to allocate memory for send TOKEN buffers");
-    sprintf(buf, "TokenAlloc() - ERROR: Unable to allocate memory for send TOKEN buffers");
-    CmiAbort(buf);
-  }
-
-  sendSgl = (cc_data_addr_t*) CmiAlloc(n*ALIGN8(sizeof(cc_data_addr_t)));
-
-  if (sendSgl == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "TokenAlloc() - ERROR: Unable to allocate memory for send SGL buffers");
-    sprintf(buf, "TokenAlloc() - ERROR: Unable to allocate memory for send SGL buffers");
-    CmiAbort(buf);
-  }
-
-  tokenScanner = sendToken;
-  for (i=0; i<n; ++i) {
-    sendSgl->stag = newStagIndex;
-    sendSgl->length = AMMASSO_BUFSIZE + sizeof(Tailer);
-    sendSgl->to = (unsigned long)&(sendBuffer[i]);
-    tokenScanner->wr.wr_id = (unsigned long)tokenScanner;
-    tokenScanner->wr.wr_type = CC_WR_TYPE_RDMA_WRITE;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_count = 1;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_list = sendSgl;
-    tokenScanner->wr.signaled = 1;
-    tokenScanner->localBuf = (AmmassoBuffer*)&(sendBuffer[i]);
-    LIST_ENQUEUE(contextBlock->,freeTokens,tokenScanner);
-    sendSgl = (cc_data_addr_t*)(((char*)sendSgl)+ALIGN8(sizeof(cc_data_addr_t)));
-    tokenScanner = (AmmassoToken*)(((char*)tokenScanner)+ALIGN8(sizeof(AmmassoToken)));
-  }
-}
-
-void RequestTokens(OtherNode node, int n) {
-  char buf[24];
-  *((int*)buf) = n;
-  sendDataOnQP(buf, sizeof(int), node, AMMASSO_MOREBUFFERS);
-}
-
-void GrantTokens(OtherNode node, int n) {
-  int i;
-  char *buf;
-  AmmassoBuffer *buffer;
-  AmmassoBuffer *prebuffer;
-  AmmassoTokenDescription *tokenDesc;
-  if (node->pending != NULL) return;
-  if (n*sizeof(AmmassoTokenDescription) + sizeof(int) > AMMASSO_BUFSIZE) {
-    n = (AMMASSO_BUFSIZE-sizeof(int)) / sizeof(AmmassoTokenDescription);
-  }
-  if (contextBlock->num_freeRecvBuffers < n) {
-    int quantity = (n - contextBlock->num_freeRecvBuffers + 1023) & (~1023);
-    BufferAlloc(quantity);
-  }
-  buf = (char*) CmiAlloc(n*sizeof(AmmassoTokenDescription) + sizeof(int));
-  *((int*)buf) = n;
-  tokenDesc = (AmmassoTokenDescription*)(buf+sizeof(int));
-  buffer = contextBlock->freeRecvBuffers;
-  for (i=0; i<n; ++i) {
-    tokenDesc[i].stag = buffer->stag;
-    tokenDesc[i].to = (unsigned long)buffer;
-    prebuffer = buffer;
-    buffer = buffer->next;
-  }
-  node->pending = contextBlock->freeRecvBuffers;
-  node->last_pending = prebuffer;
-  node->num_pending = n;
-  prebuffer->next = NULL;
-  contextBlock->num_freeRecvBuffers -= n;
-  contextBlock->freeRecvBuffers = buffer;
-  sendDataOnQP(buf, n*sizeof(AmmassoTokenDescription) + sizeof(int), node, AMMASSO_ALLOCATE);
-  CmiFree(buf);
-}
-
-void RequestReleaseTokens(OtherNode node, int n) {
-  char buf[24];
-  *((int*)buf) = n;
-  sendDataOnQP(buf, sizeof(int), node, AMMASSO_RELEASE);
-}
-
-void ReleaseTokens(OtherNode node, int n) {
-  int i, nWR;
-  AmmassoToken *token;
-  AmmassoBuffer *tokenBuf;
-  cc_data_addr_t *tokenSgl;
-
-  if (node->num_sendTokens < n) n = node->num_sendTokens - 1;
-  if (n <= 0) return;
-  token = node->sendTokens;
-  tokenBuf = token->localBuf;
-
-  tokenBuf->tail.length = 1;
-  tokenBuf->tail.ack = 0;  // do not send any ACK with this message
-  tokenBuf->tail.flags = AMMASSO_RELEASED;
-
-  // Setup the local SGL
-  tokenSgl = token->wr.wr_u.rdma_write.local_sgl.sge_list;
-  tokenSgl->length = sizeof(Tailer);
-  tokenSgl->to = (unsigned long)&tokenBuf->tail;
-  token->wr.wr_u.rdma_write.remote_to = (unsigned long)&token->remoteBuf->tail;
-
-  CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, &token->wr, 1, &nWR),node->myNode);
-
-  if (contextBlock->freeTokens == NULL) {
-    contextBlock->freeTokens = node->sendTokens;
-  } else {
-    contextBlock->last_freeTokens->next = node->sendTokens;
-  }
-  for (i=1; i<n; ++i) token = token->next;
-  contextBlock->last_freeTokens = token;
-  node->sendTokens = token->next;
-  token->next = NULL;
-  contextBlock->num_freeTokens += n;
-  node->num_sendTokens -= n;
-}
-
-/* CmiMachineInit()
- *   This is called as the node is starting up.  It does some initialization of the machine layer.
- */
-void CmiMachineInit(char **argv) {
-
-  char buf[128];
-  cc_status_t rtn;
-
-  AMMASSO_STATS_INIT
-
-  AMMASSO_STATS_START(MachineInit)
-
-  MACHSTATE(2, "CmiMachineInit() - INFO: (***** Ammasso Specific*****) - Called... Initializing RNIC...");
-  MACHSTATE1(1, "CmiMachineInit() - INFO: Cmi_charmrun_pid = %d", Cmi_charmrun_pid);
-
-
-  //CcdCallOnConditionKeep(CcdPERIODIC, (CcdVoidFn)periodicFunc, NULL);
-
-
-  // Allocate a context block that will be used throughout this machine layer
-  if (contextBlock != NULL) {
-    MACHSTATE(5, "CmiMachineInit() - ERROR: contextBlock != NULL");
-    sprintf(buf, "CmiMachineInit() - ERROR: contextBlock != NULL");
-    CmiAbort(buf);
-  }
-  contextBlock = (mycb_t*)malloc(sizeof(mycb_t));
-  if (contextBlock == NULL) {
-    MACHSTATE(5, "CmiMachineInit() - ERROR: Unable to malloc memory for contextBlock");
-    sprintf(buf, "CmiMachineInit() - ERROR: Unable to malloc memory for contextBlock");
-    CmiAbort(buf);
-  }
-
-  // Initialize the contextBlock by zero-ing everything out and then setting anything special
-  memset(contextBlock, 0, sizeof(mycb_t));
-  contextBlock->rnic = -1;
-
-  MACHSTATE(1, "CmiMachineInit() - INFO: (PRE-OPEN_RNIC)");
-
-  // Check to see if in stand-alone mode
-  if (Cmi_charmrun_pid != 0) {
-
-    // Try to Open the RNIC
-    //   TODO : Look-up the difference between CC_PBL_PAGE_MODE and CC_PBL_BLOCK_MODE
-    //   TODO : Would a call to cc_rnic_enum or cc_rnic_query do any good here?
-    rtn = cc_rnic_open(0, CC_PBL_PAGE_MODE, contextBlock, &(contextBlock->rnic));
-    if (rtn != CC_OK) {
-      MACHSTATE2(5, "CmiMachineInit() - ERROR: Unable to open RNIC: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      sprintf(buf, "CmiMachineInit() - ERROR: Unable to open RNIC: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      CmiAbort(buf);
-    }
-
-    MACHSTATE(1, "CmiMachineInit() - INFO: (PRE-SET-ASYNC-HANDLER)");
-
-    // Set the asynchronous event handler function
-    CC_CHECK(cc_eh_set_async_handler,(contextBlock->rnic, AsynchronousEventHandler, contextBlock));
-
-    /*
-    MACHSTATE(3, "CmiMachineInit() - INFO: (PRE-SET-CE-HANDLER)");
-
-    // Set the Completion Event Handler
-    contextBlock->eh_id = 0;
-    CC_CHECK(cc_eh_set_ce_handler,(contextBlock->rnic, CompletionEventHandler, &(contextBlock->eh_id)));
-    */
-
-    MACHSTATE(1, "CmiMachineInit() - INFO: (PRE-PD-ALLOC)");
-
-    // Allocate the Protection Domain
-    CC_CHECK(cc_pd_alloc,(contextBlock->rnic, &(contextBlock->pd_id)));
-
-    MACHSTATE(1, "CmiMachineInit() - INFO: RNIC Open For Business!!!");
-
-  } else {  // Otherwise, not in stand-alone mode
-
-    // Flag the rnic variable as invalid
-    contextBlock->rnic = -1;
-  }
-
-  MACHSTATE(2, "CmiMachineInit() - INFO: Completed Successfully !!!");
-
-  AMMASSO_STATS_END(MachineInit)
-}
-
-void CmiCommunicationInit(char **argv)
-{
-}
-
-void CmiAmmassoNodeAddressesStoreHandler(int pe, struct sockaddr_in *addr, int port) {
-
-  // DMK : NOTE : The hope is that this can be used to request the RMDA addresses of the other nodes after the
-  //              initial addresses from charmrun are given to the node.  Get the address here, use that to request
-  //              the RDMA address, use the RDMA address to create the QP connection (in establishQPConnection(), which
-  //              only subtracts one from the address at the moment... the way our cluster is setup).
-
-  MACHSTATE1(2, "CmiNodeAddressesStoreHandler() - INFO: pe = %d", pe);
-  MACHSTATE1(1, "                                       addr = { sin_family = %d,", addr->sin_family);
-  MACHSTATE1(1, "                                                sin_port = %d,", addr->sin_port);
-  MACHSTATE4(1, "                                                sin_addr.s_addr = %d.%d.%d.%d }", (addr->sin_addr.s_addr & 0xFF), ((addr->sin_addr.s_addr >> 8) & 0xFF), ((addr->sin_addr.s_addr >> 16) & 0xFF), ((addr->sin_addr.s_addr >> 24) & 0xFF));
-  MACHSTATE1(1, "                                       port = %d", port);
-}
-
-
-void AmmassoDoIdle(void) {
-
-  int i;
-  cc_wc_t wc;
-
-  AMMASSO_STATS_START(AmmassoDoIdle)
-
-  /* DYNAMIC ALLOCATOR: Callbacks */
-  /*if (contextBlock->conditionRegistered == 0) {
-    CcdCallOnConditionKeep(CcdPERIODIC_1s, (CcdVoidFn) AllocatorCheck, NULL);
-    //CcdCallFnAfter((CcdVoidFn) AllocatorCheck, NULL, 100);
-    contextBlock->conditionRegistered = 1;
-    }*/
-  /* DYNAMIC ALLOCATOR END */
-
-  for (i = 0; i < contextBlock->numNodes; i++) {
-    if (i == contextBlock->myNode) continue;
-    CheckRecvBufForMessage(&(nodes[i]));
-    while (cc_cq_poll(contextBlock->rnic, nodes[i].send_cq, &wc) == CC_OK) {
-      MACHSTATE1(3, "AmmassoDoIdle() - INFO: Send completed with node %d... now waiting for acknowledge...", i);
-    }
-  }
-
-  AMMASSO_STATS_END(AmmassoDoIdle)
-}
-
-void CmiNotifyIdle(void) {
-  AmmassoDoIdle();
-}
-
-static CmiIdleState* CmiNotifyGetState(void) {
-  return NULL;
-}
-
-static void CmiNotifyBeginIdle(CmiIdleState *s) {
-  AmmassoDoIdle();
-}
-
-static void CmiNotifyStillIdle(CmiIdleState *s) {
-  AmmassoDoIdle();
-}
-
-/* NOTE: if the ack overflows, we cannot use this method of sending, but we need
-   to send a special message for the purpose */
-void sendAck(OtherNode node) {
-
-  int nWR;
-
-  AMMASSO_STATS_START(sendAck)
-
-  MACHSTATE2(3, "sendAck() - Ammasso - INFO: Called... sending ACK %d to node %d", *node->remoteAck, node->myNode);
-
-  if (*node->remoteAck < ACK_MASK) {
-
-    // Send an ACK message to the specified QP/Connection/Node
-    CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, node->ack_sq_wr, 1, &nWR),node->myNode);
-
-  } else {
-    // Rare case: happens only after days of run! In this case, do not update
-    // directly the ack, but zero it on the other side, wait one second to be
-    // safe, and then send a regular message with the ACK
-    AmmassoToken *token;
-    AmmassoBuffer *tokenBuf;
-    int tmp_ack = *node->remoteAck;
-    *node->remoteAck = 0;
-    CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, node->ack_sq_wr, 1, &nWR),node->myNode);
-
-    sleep(1);
-
-    token = getQPSendToken(node);
-    tokenBuf = token->localBuf;
-    tokenBuf->tail.ack = tmp_ack;
-    tokenBuf->tail.flags = ACK_WRAPPING;
-    tokenBuf->tail.length = 1; // So it will be seen by the receiver
-    token->wr.wr_u.rdma_write.local_sgl.sge_list->length = sizeof(Tailer);
-    token->wr.wr_u.rdma_write.local_sgl.sge_list->to = (unsigned long)&tokenBuf->tail;
-    token->wr.wr_u.rdma_write.remote_to = (unsigned long)&token->remoteBuf->tail;
-    CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, &token->wr, 1, &nWR),node->myNode);
-    LIST_ENQUEUE(node->,usedTokens,token);
-    node->max_used_tokens = (node->num_usedTokens>node->max_used_tokens)?node->num_usedTokens:node->max_used_tokens;
-    *node->remoteAck = tmp_ack & ACK_MASK;
-  }
-
-  node->messagesNotYetAcknowledged = 0;
-
-  AMMASSO_STATS_END(sendAck)
-}
-
-/* NOTE, even in SMP versions, only the communication server should be sending
-   messages out, thus no locking should be necessary */
-/* This function returns the token usable for next communication. If no token is
-   available, it blocks until one becomes available */
-AmmassoToken *getQPSendToken(OtherNode node) {
-  AmmassoToken *token;
-  int i;
-  cc_wc_t wc;
-  ammasso_ack_t newAck;
-  while (node->connectionState != QP_CONN_STATE_CONNECTED ||
-        node->sendTokens == NULL) {
-    // Try to see if an ACK has been sent directly, so we free some tokens The
-    // direct token will never be greater than ACK_MASK (by protocol
-    // definition), so we do not need to wrap around
-    MACHSTATE(3, "getQPSendBuffer() - INFO: No tokens available");
-    if (*node->directAck > node->localAck) {
-      newAck = *node->directAck;
-      for (i=node->localAck; i<newAck; ++i) {
-       LIST_DEQUEUE(node->,usedTokens,token);
-       LIST_ENQUEUE(node->,sendTokens,token);
-      }
-      node->localAck = newAck;
-    }
-
-    CheckRecvBufForMessage(node);
-
-    while (cc_cq_poll(contextBlock->rnic, node->send_cq, &wc) == CC_OK) {
-      MACHSTATE1(3, "getQPSendBuffer() - INFO: Send completed with node %d... now waiting for acknowledge...", node->myNode);
-    }
-  }
-  LIST_DEQUEUE(node->,sendTokens,token);
-  return token;
-}
-
-/*
-int getQPSendBuffer(OtherNode node, char force) {
-
-  int rtnBufIndex, i;
-  cc_wc_t wc;
-
-  AMMASSO_STATS_START(getQPSendBuffer)
-
-  MACHSTATE1(3, "getQPSendBuffer() - Ammasso - Called (send to node %d)...", node->myNode);
-
-  while (1) {
-
-    AMMASSO_STATS_START(getQPSendBuffer_loop)
-
-    rtnBufIndex = -1;
-
-    AMMASSO_STATS_START(getQPSendBuffer_lock)
-
-    MACHSTATE(3, "getQPSendBuffer() - INFO: Pre-sendBufLock");
-    #if CMK_SHARED_VARS_UNAVAILABLE
-      while (node->sendBufLock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-    #endif
-    CmiLock(node->sendBufLock);
-
-    AMMASSO_STATS_END(getQPSendBuffer_lock)
-    
-    // If force is set, let the message use any of the available send buffers.  Otherwise, there can only
-    // be AMMASSO_NUMMSGBUFS_PER_QP message outstanding (haven't gotten an ACK for) so wait for that.
-    // VERY IMPORTANT !!! Only use force for sending ACKs !!!!!!!!  If this is done, it is ensured that there
-    // will always be at least one buffer available when the force code is executed
-    if (node->connectionState == QP_CONN_STATE_CONNECTED) {
-      if (force) {
-        rtnBufIndex = node->send_UseIndex;
-        node->send_UseIndex++;
-        if (node->send_UseIndex >= AMMASSO_NUMMSGBUFS_PER_QP * 2)
-          node->send_UseIndex = 0;
-      } else {
-        if (node->send_InUseCounter < AMMASSO_NUMMSGBUFS_PER_QP) {
-          rtnBufIndex = node->send_UseIndex;
-          node->send_InUseCounter++;
-          node->send_UseIndex++;
-          if (node->send_UseIndex >= AMMASSO_NUMMSGBUFS_PER_QP * 2)
-            node->send_UseIndex = 0;
-        }
-      }
-    }
-    CmiUnlock(node->sendBufLock);
-    MACHSTATE3(3, "getQPSendBuffer() - INFO: Post-sendBufLock - rtnBufIndex = %d, node->connectionState = %d, node->send_UseIndex = %d", rtnBufIndex, node->connectionState, node->send_UseIndex);
-
-    if (rtnBufIndex >= 0) {
-
-      AMMASSO_STATS_END(getQPSendBuffer_loop)
-      break;
-
-    } else {
-
-      //usleep(1);
-
-      AMMASSO_STATS_START(getQPSendBuffer_CEH)
-
-      CheckRecvBufForMessage(node);
-      //CompletionEventHandlerWithAckFlag(contextBlock->rnic, node->recv_cq, node, 1);
-
-      //CompletionEventHandler(contextBlock->rnic, node->send_cq, node);
-      while (cc_cq_poll(contextBlock->rnic, node->send_cq, &wc) == CC_OK) {
-       MACHSTATE1(3, "getQPSendBuffer() - INFO: Send completed with node %d... now waiting for acknowledge...", nextNode);
-      }
-
-
-      AMMASSO_STATS_END(getQPSendBuffer_CEH)
-
-      AMMASSO_STATS_END(getQPSendBuffer_loop)
-    }
-  }
-
-  //// Increment the send_UseIndex counter so another buffer will be used next time
-  //node->send_UseIndex++;
-  //if (node->send_UseIndex >= AMMASSO_NUMMSGBUFS_PER_QP * 2)
-  //  node->send_UseIndex = 0;
-
-  MACHSTATE1(3, "getQPSendBuffer() - Ammasso - Finished (returning buffer index: %d)", rtnBufIndex);
-
-  AMMASSO_STATS_END(getQPSendBuffer)
-
-  return rtnBufIndex;
-
-}
-*/
-
-// NOTE: The force parameter can be thought of as an "is ACK" control message flag (see comments in getQPSendBuffer())
-int sendDataOnQP(char* data, int len, OtherNode node, char flags) {
-
-  AmmassoToken *sendBufToken;
-  AmmassoBuffer *tokenBuf;
-  cc_data_addr_t *tokenSgl;
-  int toSendLength;
-  cc_wc_t wc;
-  cc_uint32_t nWR;
-  char isFirst = 1;
-  char *origMsgStart = data;
-  char *sendBufBegin;
-
-  int origSize = len;
-
-  if (origSize <= 1024) {
-    AMMASSO_STATS_START(sendDataOnQP_1024)
-  } else if (origSize <= 2048) { 
-    AMMASSO_STATS_START(sendDataOnQP_2048)
-  } else if (origSize <= 4096) { 
-    AMMASSO_STATS_START(sendDataOnQP_4096)
-  } else if (origSize <= 16384) { 
-    AMMASSO_STATS_START(sendDataOnQP_16384)
-  } else {
-    AMMASSO_STATS_START(sendDataOnQP_over)
-  }
-
-  AMMASSO_STATS_START(sendDataOnQP)
-
-  //CompletionEventHandler(contextBlock->rnic, node->recv_cq, node);
-  //CompletionEventHandler(contextBlock->rnic, node->send_cq, node);
-  while (cc_cq_poll(contextBlock->rnic, node->send_cq, &wc) == CC_OK) {
-    MACHSTATE1(3, "sendDataOnQP() - INFO: Send completed with node %d... now waiting for acknowledge...", node->myNode);
-  }
-
-
-  MACHSTATE2(2, "sendDataOnQP() - Ammasso - INFO: Called (send to node %d, len = %d)...", node->myNode, len);
-
-  // Assert that control messages will not be fragmented
-  CmiAssert(flags==0 || len<=AMMASSO_BUFSIZE);
-
-  // DMK : For each message that is fragmented, attach another DGRAM header to
-  // it, (keeping in mind that the control messages are no where near large
-  // enough for this to occur).
-
-  while (len > 0) {
-
-    AMMASSO_STATS_START(sendDataOnQP_pre_send)  
-
-    // Get a free send buffer (NOTE: This call will block until a send buffer is free)
-    sendBufToken = getQPSendToken(node);
-    tokenBuf = sendBufToken->localBuf;
-    // Enqueue the token to the used queue immediately, so it is safe to be
-    // interrupted by other calls
-    LIST_ENQUEUE(node->,usedTokens,sendBufToken);
-    node->max_used_tokens = (node->num_usedTokens>node->max_used_tokens)?node->num_usedTokens:node->max_used_tokens;
-
-    // Copy the contents (up to AMMASSO_BUFSIZE worth) of data into the send buffer
-
-    // The toSendLength includes the DGRAM header size. If the chunk sent is not
-    // the first, the initial DGRAM_HEADER_SIZE bytes need to be contructed from
-    // the DGRAM header of the original message (instead of just being copied
-    // together with the message itself)
-
-    if (isFirst) {
-
-      toSendLength = len > AMMASSO_BUFSIZE ? AMMASSO_BUFSIZE : len;  // MIN of len and AMMASSO_BUFSIZE
-      sendBufBegin = tokenBuf->buf + AMMASSO_BUFSIZE - ALIGN8(toSendLength);
-
-      memcpy(sendBufBegin, data, toSendLength);
-
-      MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: Sending 1st Fragment - toSendLength = %d...", toSendLength);
-
-    } else {
-
-      toSendLength = len > (AMMASSO_BUFSIZE - DGRAM_HEADER_SIZE) ? AMMASSO_BUFSIZE : (len+DGRAM_HEADER_SIZE);  // MIN of len and AMMASSO_BUFSIZE
-      sendBufBegin = tokenBuf->buf + AMMASSO_BUFSIZE - ALIGN8(toSendLength);
-
-      memcpy(sendBufBegin+DGRAM_HEADER_SIZE, data, toSendLength-DGRAM_HEADER_SIZE);
-      
-      // This dgram header is the same of the original message, except for the
-      // sequence number, so copy the original and just modify the sequence
-      // number.
-
-      // NOTE: If the message is large enough that fragmentation needs to
-      // happen, the send_next_lock is already owned by the thread executing
-      // this code.
-      memcpy(sendBufBegin, origMsgStart, DGRAM_HEADER_SIZE);
-
-      ((DgramHeader*)sendBufBegin)->seqno = node->send_next;
-      node->send_next = ((node->send_next+1) & DGRAM_SEQNO_MASK);  // Increase the sequence number
-
-      MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: Sending Continuation Fragment - toSendLength = %d...", toSendLength);
-    }
-
-    // Write the size of the message at the end of the buffer, with the ack and
-    // flags
-    tokenBuf->tail.length = toSendLength;
-    node->messagesNotYetAcknowledged = 0;
-    tokenBuf->tail.ack = *node->remoteAck;
-    if (*node->remoteAck > ACK_MASK) {
-      // Rare case of ACK wrapping
-      tokenBuf->tail.ack = 0;
-      // in the rare case that we didn't send an ack with this message because
-      // the ack just wrapped around (rare case), send a full ACK message. It is
-      // safe to send it now, since the queues are consistent between sender and
-      // receiver. The fact that this ack is effectively sent before the regular
-      // message is not important, since on the other side it will be discovered
-      // only after this one is received
-      sendAck(node);
-    }
-    tokenBuf->tail.flags = flags;
-
-    // Setup the local SGL
-    tokenSgl = sendBufToken->wr.wr_u.rdma_write.local_sgl.sge_list;
-    tokenSgl->length = ALIGN8(toSendLength) + sizeof(Tailer);
-    tokenSgl->to = (unsigned long)sendBufBegin;
-    sendBufToken->wr.wr_u.rdma_write.remote_to = (unsigned long)(((char*)sendBufToken->remoteBuf)+AMMASSO_BUFSIZE-ALIGN8(toSendLength));
-
-    // The remote_to and remote_stag are already fixed part of the token
-    AMMASSO_STATS_END(sendDataOnQP_pre_send)  
-    AMMASSO_STATS_START(sendDataOnQP_send)
-
-    MACHSTATE(3, "sendDataOnQP() - Ammasso - INFO: Enqueuing RDMA Write WR...");
-
-    MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: tokenSgl->to = %p", tokenSgl->to);
-    MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: sendBufToken->wr.wr_u.rdma_write.remote_to = %p", sendBufToken->wr.wr_u.rdma_write.remote_to);
-    MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: tail.ack = %d", tokenBuf->tail.ack);
-    MACHSTATE1(1, "sendDataOnQP() - Ammasso - INFO: tail.flags = %d", tokenBuf->tail.flags);
-
-    CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, &sendBufToken->wr, 1, &nWR),node->myNode);
-    
-    MACHSTATE(1, "sendDataOnQP() - Ammasso - INFO: RDMA Write WR Enqueue Completed");
-
-    AMMASSO_STATS_END(sendDataOnQP_send)
-    AMMASSO_STATS_START(sendDataOnQP_post_send)  
-
-    // Update the data and len variables for the next while (if fragmenting is needed)
-    data += toSendLength;
-    len -= toSendLength;
-    if (isFirst == 0) {
-      data -= DGRAM_HEADER_SIZE;
-      len += DGRAM_HEADER_SIZE;
-    }
-    isFirst = 0;
-
-    AMMASSO_STATS_END(sendDataOnQP_post_send)  
-  }
-
-  AMMASSO_STATS_END(sendDataOnQP)
-
-  if (origSize <= 1024) {
-    AMMASSO_STATS_END(sendDataOnQP_1024)
-  } else if (origSize <= 2048) { 
-    AMMASSO_STATS_END(sendDataOnQP_2048)
-  } else if (origSize <= 4096) { 
-    AMMASSO_STATS_END(sendDataOnQP_4096)
-  } else if (origSize <= 16384) { 
-    AMMASSO_STATS_END(sendDataOnQP_16384)
-  } else {
-    AMMASSO_STATS_END(sendDataOnQP_over)
-  }
-
-}
-
-
-/* DeliverViaNetwork()
- *
- */
-void DeliverViaNetwork(OutgoingMsg msg, OtherNode otherNode, int rank, unsigned int broot, int copy) {
-
-  cc_status_t rtn;
-  cc_stag_index_t stag;
-  cc_data_addr_t sgl;
-  cc_sq_wr_t wr;
-  cc_uint32_t WRsPosted;
-
-  AMMASSO_STATS_START(DeliverViaNetwork)
-
-  MACHSTATE(2, "DeliverViaNetwork() - Ammasso - INFO: Called...");
-
-  // We don't need to do this since the message data is being copied into the
-  // send_buf, the OutgoingMsg can be free'd ASAP
-
-  // The lock will be already held by the calling function in machine.c
-  // (CommLock)
-
-  AMMASSO_STATS_START(DeliverViaNetwork_post_lock)
-
-  DgramHeaderMake(msg->data, rank, msg->src, Cmi_charmrun_pid, otherNode->send_next, broot);  // Set DGram Header Fields In-Place
-  otherNode->send_next = ((otherNode->send_next+1) & DGRAM_SEQNO_MASK);  // Increase the sequence number
-
-  MACHSTATE1(1, "DeliverViaNetwork() - INFO: Sending message to  node %d", otherNode->myNode);
-  MACHSTATE1(1, "DeliverViaNetwork() - INFO:                     rank %d", rank);
-  MACHSTATE1(1, "DeliverViaNetwork() - INFO:                    broot %d", broot);
-
-  AMMASSO_STATS_START(DeliverViaNetwork_send)
-
-  sendDataOnQP(msg->data, msg->size, otherNode, 0);
-
-  AMMASSO_STATS_END(DeliverViaNetwork_send)
-
-    //CmiUnlock(otherNode->send_next_lock);
-  MACHSTATE(1, "DeliverViaNetwork() - INFO: Post-send_next_lock");
-
-
-  // DMK : NOTE : I left this in as an example of how to retister the memory with the RNIC on the fly.  Since the ccil
-  //              library has a bug which causes it not to de-pin memory that we unregister, it will probably be a better
-  //              idea to fragment a message that is too large for a single buffer from the buffer pool.
-  /***************************************************************
-  // DMK : TODO : This is an important optimization area.  This is registering the memory where the outgoing message
-  //              is located with the RNIC.  One balancing act that we will need to do is the cost of copying messages
-  //              into memory regions already allocated VS. the cost of registering the memory.  The cost of registering
-  //              memory might be constant as the memory doesn't have to traversed.  If the cost of doing a memcpy on a
-  //              small message, since memcpy traverses the memory range, is less than the cost of registering the
-  //              memory with the RNIC, it would be better to copy the message into a pre-registered memory location.
-  // Start by registering the memory of the outgoing message with the RNIC
-  rtn = cc_nsmr_register_virt(contextBlock->rnic,
-                              CC_ADDR_TYPE_VA_BASED,
-                              msg->data + DGRAM_HEADER_SIZE,
-                              msg->size - DGRAM_HEADER_SIZE,
-                              contextBlock->pd_id,
-                              0, 0,
-                              CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE,
-                              &stag
-                             );
-  if (rtn != CC_OK) {
-    // Let the user know what happened
-    MACHSTATE2(3, "DeliverViaNetwork() - Ammasso - ERROR - Unable to register OutgoingMsg memory with RNIC: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-    return;
-  }
-
-  // Setup the Scatter/Gather List
-  sgl.stag = stag;
-  sgl.to = (cc_uint64_t)(unsigned long)(msg->data + DGRAM_HEADER_SIZE);
-  sgl.length = msg->size - DGRAM_HEADER_SIZE;
-
-  // Create the Work Request
-  wr.wr_id = (cc_uint64_t)(unsigned long)&(wr);
-  wr.wr_type = CC_WR_TYPE_SEND;
-  wr.wr_u.send.local_sgl.sge_count = 1;
-  wr.wr_u.send.local_sgl.sge_list = &sgl;
-  wr.signaled = 1;
-
-  // Post the Work Request to the Send Queue
-  // DMK : TODO : Update this code to handle CCERR_QP_TOO_MANY_WRS_POSTED errors (pause breifly and retry)
-  rtn = cc_qp_post_sq(contextBlock->rnic, otherNode->qp, &(wr), 1, &(WRsPosted));
-  if (rtn != CC_OK || WRsPosted != 1) {
-    // Let the user know what happened
-    MACHSTATE2(3, "DeliverViaNetwork() - Ammasso - ERROR - Unable post Work Request to Send Queue: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-    displayQueueQuery(otherNode->qp, &(otherNode->qp_attrs));
-  }
-  ***************************************************************/
-
-  MACHSTATE(2, "DeliverViaNetwork() - Ammasso - INFO: Completed.");
-
-  AMMASSO_STATS_END(DeliverViaNetwork_post_lock)
-  AMMASSO_STATS_END(DeliverViaNetwork)
-}
-
-
-/****************************************************************************
- *                                                                          
- * CheckSocketsReady
- *
- * Checks both sockets to see which are readable and which are writeable.
- * We check all these things at the same time since this can be done for
- * free with ``select.'' The result is stored in global variables, since
- * this is essentially global state information and several routines need it.
- *
- ***************************************************************************/
-
-int CheckSocketsReady(int withDelayMs)
-{   
-  int nreadable;
-  CMK_PIPE_DECL(withDelayMs);
-
-  CmiStdoutAdd(CMK_PIPE_SUB);
-  if (Cmi_charmrun_fd!=-1) CMK_PIPE_ADDREAD(Cmi_charmrun_fd);
-
-  nreadable=CMK_PIPE_CALL();
-  ctrlskt_ready_read = 0;
-  dataskt_ready_read = 0;
-  dataskt_ready_write = 0;
-  
-  if (nreadable == 0) {
-    MACHSTATE(2, "} CheckSocketsReady (nothing readable)");
-    return nreadable;
-  }
-  if (nreadable==-1) {
-    CMK_PIPE_CHECKERR();
-    MACHSTATE(3, "} CheckSocketsReady (INTERRUPTED!)");
-    return CheckSocketsReady(0);
-  }
-  CmiStdoutCheck(CMK_PIPE_SUB);
-  if (Cmi_charmrun_fd!=-1) 
-          ctrlskt_ready_read = CMK_PIPE_CHECKREAD(Cmi_charmrun_fd);
-  MACHSTATE(3, "} CheckSocketsReady");
-  return nreadable;
-}
-
-/***********************************************************************
- * CommunicationServer()
- * 
- * This function does the scheduling of the tasks related to the
- * message sends and receives. 
- * It first check the charmrun port for message, and poll the gm event
- * for send complete and outcoming messages.
- *
- ***********************************************************************/
-
-// NOTE: Always called from interrupt
-static void ServiceCharmrun_nolock(void) {
-
-  int again = 1;
-
-  MACHSTATE(2, "ServiceCharmrun_nolock begin {");
-
-  while (again) {
-    again = 0;
-
-    CheckSocketsReady(0);
-    if (ctrlskt_ready_read) { ctrl_getone(); again=1; }
-    if (CmiStdoutNeedsService()) { CmiStdoutService(); }
-  }
-
-  MACHSTATE(2, "} ServiceCharmrun_nolock end");
-}
-
-void processAmmassoControlMessage(char* msg, int len, Tailer *tail, OtherNode from) {
-
-  int nodeIndex, ctrlType, i, n, nWR;
-  AmmassoToken *token, *pretoken;
-  AmmassoBuffer *tokenBuf;
-  cc_data_addr_t *tokenSgl;
-  OtherNode node;
-  AmmassoTokenDescription *tokenDesc;
-
-  AMMASSO_STATS_START(processAmmassoControlMessage)
-
-  // Do not check the message, the flags field was set, and this is enough
-
-  // Perform an action based on the control message type
-  switch (tail->flags) {
-
-  case AMMASSO_READY:
-
-    // Decrement the node ready count by one
-    contextBlock->nodeReadyCount--;
-    MACHSTATE1(3, "processAmmassoControlMessage() - Ammasso - INFO: Received READY packet... still waiting for %d more...", contextBlock->nodeReadyCount);
-
-    break;
-
-  case AMMASSO_ALLOCATE: // Sent by the receiver to allocate more tokens
-
-    token = getQPSendToken(from);
-    tokenBuf = token->localBuf;
-
-    tokenBuf->tail.length = 1;
-    tokenBuf->tail.ack = 0;  // do not send any ACK with this message
-    tokenBuf->tail.flags = AMMASSO_ALLOCATED;
-
-    // Setup the local SGL
-    tokenSgl = token->wr.wr_u.rdma_write.local_sgl.sge_list;
-    tokenSgl->length = sizeof(Tailer);
-    tokenSgl->to = (unsigned long)&tokenBuf->tail;
-    token->wr.wr_u.rdma_write.remote_to = (unsigned long)&token->remoteBuf->tail;
-
-    CC_POST_CHECK(cc_qp_post_sq,(contextBlock->rnic, node->qp, &token->wr, 1, &nWR),node->myNode);
-
-    LIST_ENQUEUE(from->,usedTokens,token);
-    from->max_used_tokens = (from->num_usedTokens>from->max_used_tokens)?from->num_usedTokens:from->max_used_tokens;
-    // add the new tokens at the end of usedTokens (right after the one which
-    // which we are sending back the confirmation)
-    n = *((int*)msg);
-    if (contextBlock->num_freeTokens < n) {
-      int quantity = (n - contextBlock->num_freeTokens + 1023) & (~1023);
-      BufferAlloc(quantity);
-    }
-    token = contextBlock->freeTokens;
-    tokenDesc = (AmmassoTokenDescription*)(msg+sizeof(int));
-    for (i=0; i<n; ++i) {
-      token->remoteBuf = (AmmassoBuffer*)tokenDesc[i].to;
-      token->wr.wr_u.rdma_write.remote_stag = tokenDesc[i].stag;
-      pretoken = token;
-      token = token->next;
-    }
-    from->last_usedTokens->next = contextBlock->freeTokens;
-    from->last_usedTokens = pretoken;
-    pretoken->next = NULL;
-    contextBlock->freeTokens = token;
-    from->num_usedTokens += n;
-    contextBlock->num_freeTokens -= n;
-
-    break;
-
-  case AMMASSO_ALLOCATED: // Sent by the sender to conferm the token allocation
-
-    // link the pending buffers at the end of those allocated (right after the
-    // one with which this message came)
-    from->last_recv_buf->next = from->pending;
-    from->last_recv_buf = from->last_pending;
-    from->last_pending = NULL;
-    from->num_recv_buf += from->num_pending;
-    *from->remoteAck += from->num_pending;
-    from->num_pending = 0;
-
-    break;
-
-  case AMMASSO_MOREBUFFERS: // Sent by the sender to ask for more tokens
-
-    /* DYNAMIC ALLOCATOR: Grant what requested */
-    n = *((int*)msg);
-    GrantTokens(from, n);
-    /* DYNAMIC ALLOCATOR END */
-
-    break;
-
-  case AMMASSO_RELEASE: // Sent by the receiver to request back tokens
-
-    /* DYNAMIC ALLOCATOR: Ignore the request of the receiver to return buffers */
-    /* DYNAMIC ALLOCATOR END */
-
-    break;
-
-  case AMMASSO_RELEASED: // Sent by the sender to release tokens
-
-    // This secondLastRecvBuf is set in CheckRecvBufForMessage
-    from->secondLastRecvBuf->next = NULL;
-    tokenBuf = from->last_recv_buf;
-    from->last_recv_buf = from->secondLastRecvBuf;
-    from->num_recv_buf--;
-    LIST_ENQUEUE(contextBlock->,freeRecvBuffers,tokenBuf);
-    n = *((int*)msg);
-    for (i=1; i<n; ++i) {
-      LIST_DEQUEUE(from->,recv_buf,tokenBuf);
-      LIST_ENQUEUE(contextBlock->,freeRecvBuffers,tokenBuf);
-    }
-
-    break;
-
-  case ACK_WRAPPING:
-
-    // The ACK has already been accounted in ProcessMessage, just mask it
-    from->localAck &= ACK_MASK;
-
-    break;
-
-  default:
-    MACHSTATE1(5, "processAmmassoControlMessage() - Ammasso -INFO: Received control message with invalid flags: %d", tail->flags);
-    CmiAbort("Invalid control message received");
-  }
-
-  AMMASSO_STATS_END(processAmmassoControlMessage)
-}
-
-int ProcessMessage(char* msg, int len, Tailer *tail, OtherNode from) {
-
-  int rank, srcPE, seqNum, magicCookie, size, i;
-  unsigned int broot;
-  unsigned char checksum;
-  OtherNode fromNode;
-  char *newMsg;
-  int needAck;
-
-  AMMASSO_STATS_START(ProcessMessage)
-
-  MACHSTATE(2, "ProcessMessage() - INFO: Called...");
-  MACHSTATE2(1, "ProcessMessage() - INFO: tail - ack=%d, flags=%d", tail->ack, tail->flags);
-
-  // Decide whether a direct ACK will be needed or not, based on how many
-  // messages the other side has sent us, and we haven't acknowledged yet
-  if (2*from->messagesNotYetAcknowledged > from->num_recv_buf) {
-    needAck = 1;
-  } else {
-    needAck = 0;
-  }
-
-  // This message contains an ACK as all messages, parse it. Do not worry about
-  // wrap around of the ACK, since when this happen a special control message
-  // ACK_WRAPPING is received
-  if (tail->ack > from->localAck) {
-    AmmassoToken *token;
-    for (i=from->localAck; i<tail->ack; ++i) {
-      LIST_DEQUEUE(from->,usedTokens,token);
-      LIST_ENQUEUE(from->,sendTokens,token);
-    }
-    from->localAck = tail->ack;
-  }
-
-  {
-    MACHSTATE1(1, "ProcessMessage() - INFO: msg = %p", msg);
-    int j;
-    for (j = 0; j < DGRAM_HEADER_SIZE + 24; j++) {
-      MACHSTATE2(1, "ProcessMessage() - INFO: msg[%d] = %02x", j, msg[j]);
-    }
-  }
-
-  if (tail->flags != 0) {
-    processAmmassoControlMessage(msg, len, tail, from);
-    return needAck;
-  }
-
-  // Get the header fields of the message
-  DgramHeaderBreak(msg, rank, srcPE, magicCookie, seqNum, broot);
-
-  MACHSTATE(1, "ProcessMessage() - INFO: Message Contents:");
-  MACHSTATE1(1, "                           rank = %d", rank);
-  MACHSTATE1(1, "                           srcPE = %d", srcPE);
-  MACHSTATE1(1, "                           magicCookie = %d", magicCookie);
-  MACHSTATE1(1, "                           seqNum = %d", seqNum);
-  MACHSTATE1(1, "                           broot = %d", broot);
-
-#ifdef CMK_USE_CHECKSUM
-
-  // Check the checksum
-  checksum = computeCheckSum(msg, len);
-  if (checksum != 0) {
-    MACHSTATE1(5, "ProcessMessage() - Ammasso - ERROR: Received message with bad checksum (%d)... ignoring...", checksum);
-    CmiPrintf("[%d] ProcessMessage() - Ammasso - ERROR: Received message with bad checksum (%d)... ignoring...\n", CmiMyPe(), checksum);
-    return needAck;
-  }
-
-#else
-
-  // Check the magic cookie for correctness
-  if (magicCookie != (Cmi_charmrun_pid & DGRAM_MAGIC_MASK)) {
-    MACHSTATE(5, "ProcessMessage() - Ammasso - ERROR: Received message with a bad magic cookie... ignoring...");
-    CmiPrintf("[%d] ProcessMessage() - Ammasso - ERROR: Received message with a bad magic cookie... ignoring...\n", CmiMyPe());
-
-    {
-      CmiPrintf("ProcessMessage() - INFO: Cmi_charmrun_pid = %d\n", Cmi_charmrun_pid);
-      CmiPrintf("ProcessMessage() - INFO: node->recv_UseIndex = %d\n", nodes_by_pe[srcPE]->recv_UseIndex);
-      CmiPrintf("ProcessMessage() - INFO: msg = %p\n", msg);
-      int j;
-      for (j = 0; j < DGRAM_HEADER_SIZE + 24; j++) {
-        CmiPrintf("ProcessMessage() - INFO: msg[%d] = %02x\n", j, msg[j]);
-      }
-    }
-    return needAck;
-  }
-
-#endif
-
-  // Get the OtherNode structure for the node this message was sent from
-  fromNode = nodes_by_pe[srcPE];
-
-  CmiAssert(fromNode == from);
-
-  MACHSTATE1(1, "ProcessMessage() - INFO: Message from node %d...", fromNode->myNode);
-
-  //MACHSTATE(3, "ProcessMessage() - INFO: Pre-recv_expect_lock");
-  //#if CMK_SHARED_VARS_UNAVAILABLE
-  //  while (fromNode->recv_expect_lock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-  //#endif
-  //CmiLock(fromNode->recv_expect_lock);
-
-  // Check the sequence number of the message
-  if (seqNum == (fromNode->recv_expect)) {
-    // The expected sequence number was received so setup the next one
-    fromNode->recv_expect = ((seqNum+1) & DGRAM_SEQNO_MASK);
-  } else {
-    MACHSTATE(5, "ProcessMessage() - Ammasso - ERROR: Received a message with a bad sequence number... ignoring...");
-    CmiPrintf("[%d] ProcessMessage() - Ammasso - ERROR: Received a message witha bad sequence number... ignoring...\n", CmiMyPe());
-    return needAck;
-  }
-
-  //CmiUnlock(fromNode->recv_expect_lock);
-  //MACHSTATE(3, "ProcessMessage() - INFO: Post-recv_expect_lock");
-
-  newMsg = fromNode->asm_msg;
-
-  // Check to see if this is the start of the message (i.e. - if the message was
-  // fragmented, if this is the first packet of the message) or the entire
-  // message. Only the first packet's header information will be copied into the
-  // asm_buf buffer.
-  if (newMsg == NULL) {
-
-    // Allocate memory to hold the new message
-    size = CmiMsgHeaderGetLength(msg);
-    newMsg = (char*)CmiAlloc(size);
-    _MEMCHECK(newMsg);
-    
-    // Verify the message size
-    if (len > size) {
-      MACHSTATE2(5, "ProcessMessage() - Ammasso - ERROR: Message size mismatch (size: %d != len: %d)", size, len);
-      CmiPrintf("[%d] ProcessMessage() - Ammasso - ERROR: Message size mismatch (size: %d != len: %d)\n", CmiMyPe(), size, len);
-      CmiAbort("Message Size Mismatch");
-    }
-
-    // Copy the message into the memory location and setup the fromNode structure accordingly
-    //memcpy(newMsg, msg + DGRAM_HEADER_SIZE, size);
-    memcpy(newMsg, msg, len);
-    fromNode->asm_rank = rank;
-    fromNode->asm_total = size;
-    fromNode->asm_fill = len;
-    fromNode->asm_msg = newMsg;
-
-  // Otherwise, this packet is a continuation of the overall message so append it to the last packet
-  } else {
-
-    size = len - DGRAM_HEADER_SIZE;
-
-    // Make sure there is enough room in the asm_msg buffer (this should always be true because of the alloc in the true
-    //   portion of this if statement).
-    if (fromNode->asm_fill + size > fromNode->asm_total) {
-      MACHSTATE(5, "ProcessMessage() - Ammasso - ERROR: Message size mismatch");
-      CmiPrintf("[%d] ProcessMessage() - Ammasso - ERROR: Message size mismatch", CmiMyPe());
-      CmiAbort("Message Size Mismatch");
-    }
-
-    // Copy the message into the asm_msg buffer
-    memcpy(newMsg + fromNode->asm_fill, msg + DGRAM_HEADER_SIZE, size);
-    fromNode->asm_fill += size;
-  }
-
-  MACHSTATE2(1, "ProcessMessage() - Ammasso - INFO: Message copied into asm_buf (asm_fill = %d, asm_total = %d)...", fromNode->asm_fill, fromNode->asm_total);
-
-  // Check to see if a full packet has been received
-  if (fromNode->asm_fill == fromNode->asm_total) {
-
-    MACHSTATE(1, "ProcessMessage() - Ammasso - INFO: Pushing message...");
-
-    // React to the message based on its rank
-    switch (rank) {
-
-      case DGRAM_BROADCAST:
-
-        MACHSTATE1(3, "ProcessMessage() - Ammasso - INFO: Broadcast - _Cmi_mynodesize = %d", _Cmi_mynodesize);
-
-        // Make a copy of the message for all the PEs on this node except for zero, zero gets the original
-        for (i = 1; i < _Cmi_mynodesize; i++)
-          CmiPushPE(i, CopyMsg(newMsg, fromNode->asm_total));
-        CmiPushPE(0, newMsg);
-        break;
-
-#if CMK_NODE_QUEUE_AVAILABLE
-      case DGRAM_NODEBROADCAST:
-      case DGRAM_NODEMESSAGE:
-        CmiPushNode(newMsg);
-        break;
-#endif
-
-      default:
-        CmiPushPE(rank, newMsg);
-    }
-
-    MACHSTATE(1, "ProcessMessage() - Ammasso - INFO: NULLing asm_msg...");
-
-    // Clear the message buffer
-    fromNode->asm_msg = NULL;
-  }
-
-  MACHSTATE(1, "ProcessMessage() - Ammasso - INFO: Checking for re-broadcast");
-
-  // If this packet is part of a broadcast, pass it on to the next nodes
-  #if CMK_BROADCAST_SPANNING_TREE
-
-      if (rank == DGRAM_BROADCAST
-    #if CMK_NODE_QUEUE_AVAILABLE
-          || rank == DGRAM_NODEBROADCAST
-    #endif
-        ) SendSpanningChildren(NULL, 0, len, msg, broot, rank);
-  
-  #elif CMK_BROADCAST_HYPERCUBE
-
-      if (rank == DGRAM_BROADCAST
-    #if CMK_NODE_QUEUE_AVAILABLE
-          || rank == DGRAM_NODEBROADCAST
-    #endif
-        ) {
-             MACHSTATE(3, "ProcessMessage() - INFO: Calling SendHypercube()...");
-             SendHypercube(NULL, 0, len, msg, broot, rank);
-           }
-  #endif
-
-  AMMASSO_STATS_END(ProcessMessage)
-  return needAck;
-}
-
-// DMK : NOTE : I attempted to put this is a single function to be called by many places but it was getting
-//              way too messy... I'll leave the code here for now in hopes of being able to do this in the future.
-/*************************************************
-// NOTE: Return non-zero if a message was received, zero otherwise
-int PollForMessage(cc_cq_handle_t cq) {
-
-  cc_status_t rtn;
-  cc_wc_t wc;
-  int tmp, fromNode;
-  cc_qp_query_attrs_t qpAttr;
-
-  // Poll the Completion Queue for Work Completions
-  rtn = cc_cq_poll(contextBlock->rnic, cq, &wc);
-
-  // Just return 0 if there are no Work Completions
-  if (rtn == CCERR_CQ_EMPTY) return 0;
-
-  // Let the user know if there was an error
-  if (rtn != CC_OK) {
-    MACHSTATE2(3, "PollForMessage() - Ammasso - ERROR: Unable to poll Completion Queue: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-  }
-
-  // Let the user know if there was an error
-  if (wc.status != CC_OK) {
-    MACHSTATE2(3, "PollForMessage() - Ammasso - ERROR: Work Completion Status: %d, \"%s\"", wc.status, cc_status_to_string(wc.status));
-  }
-
-  // Depending on the WC.type, react accordingly
-  switch (wc.wr_type) {
-
-    // wc.wr_type == CC_WR_TYPE_RDMA_READ
-    case CC_WR_TYPE_RDMA_READ:
-
-      // DMK : TODO : Fill this in
-
-      break;
-
-    // wc.wr_type == CC_WR_TYPE_RDMA_WRITE
-    case CC_WR_TYPE_RDMA_WRITE:
-
-      // DMK : TODO : Fill this in
-
-      break;
-
-    // wc.wr_type == CC_WR_TYPE_SEND  (Message was sent)
-    case CC_WR_TYPE_SEND:
-
-      // This will be called as a result of a cc_qp_post_sq()'s Send Work Request in DeliverViaNetwork finishing
-      // Free the STag that was created
-      rtn = cc_stag_dealloc(contextBlock->rnic, wc.stag);
-
-      break;
-
-    // wc.wr_type == CC_WR_TYPE_RECV  (Message was received)
-    case CC_WR_TYPE_RECV:
-
-      // Check for CCERR_FLUSHED which means "we're shutting down" according to the example code
-      if (wc.status == CCERR_FLUSHED)
-        break;
-
-      //// Make sure that node is defined
-      //if (node == NULL) {
-      //  MACHSTATE(3, "PollForMessage() - WARNING: Received message but node == NULL... ignoring...");
-      //  break;
-      //}
-
-      // HORRIBLE HACK!!! -- In the OtherNode structure, a myNode was placed directly after the rq_wr structure.  This
-      //   was done so this function could tell from which node the received message came.
-      fromNode = *((int*)((char*)wc.wr_id + sizeof(cc_rq_wr_t)));
-      //fromNode = node->myNode;
-
-      {
-        int j;
-        MACHSTATE1(3, "PollForMessage() - INFO: fromNode = %d", fromNode);
-        MACHSTATE1(3, "PollForMessage() - INFO: wc.status = %d", wc.status);
-        MACHSTATE1(3, "PollForMessage() - INFO: wc.bytes_rcvd = %d", wc.bytes_rcvd);
-        MACHSTATE1(3, "PollForMessage() - INFO: wc.wr_id = %d", wc.wr_id);
-        //MACHSTATE1(3, "PollForMessage() - INFO: &(nodes[0].rq_wr) = %p", &(nodes[0].rq_wr));
-        //MACHSTATE1(3, "PollForMessage() - INFO: &(nodes[1].rq_wr) = %p", &(nodes[1].rq_wr));
-        //MACHSTATE(3, "PollForMessage() - INFO: Raw Message Data:");
-        //for (j = 0; j < wc.bytes_rcvd; j++) {
-        //  MACHSTATE1(3, "                         [%02x]", nodes[fromNode].recv_buf[j]);
-        //}
-      }
-
-      // Process the Message
-      ProcessMessage(nodes[fromNode].recv_buf, wc.bytes_rcvd);
-      // Re-Post the receive buffer
-      tmp = 0;
-      rtn = cc_qp_post_rq(contextBlock->rnic, nodes[fromNode].qp, &(nodes[fromNode].rq_wr), 1, &tmp);
-      if (rtn != CC_OK) {
-        // Let the user know what happened
-        MACHSTATE2(3, "PollForMessage() - Ammasso - ERROR: Unable to Post Work Request to Queue Pair: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      }
-
-      break;
-
-    // default
-    default:
-      MACHSTATE1(3, "PollForMessage() - Ammasso - WARNING - Unknown WC.wr_type: %d", wc.wr_type);
-      break;
-
-  } // end switch (wc.wr_type)
-
-  return 1;
-}
-****************************************************/
-
-
-static void CommunicationServer_nolock(int withDelayMs) {
-
-  int i;
-
-  MACHSTATE(2, "CommunicationServer_nolock start {");
-
-  // DMK : TODO : In spare time (here), check for messages and/or completions and/or errors
-  //while(PollForMessage(contextBlock->send_cq));  // Clear all the completed sends
-  //while(PollForMessage(contextBlock->recv_cq));  // Keep looping while there are still messages that have been received
-
-  for (i = 0; i < contextBlock->numNodes; i++) {
-    if (i == contextBlock->myNode) continue;
-    //CompletionEventHandlerWithAckFlag(contextBlock->rnic, nodes[i].recv_cq, &(nodes[i]), 1);
-    //CompletionEventHandler(contextBlock->rnic, nodes[i].send_cq, &(nodes[i]));
-    CheckRecvBufForMessage(&(nodes[i]));
-  }
-
-  MACHSTATE(2, "}CommunicationServer_nolock end");
-}
-
-
-/* CommunicationServer()
- * where:
- *   0: from smp thread
- *   1: from interrupt
- *   2: from worker thread
- */
-static void CommunicationServer(int withDelayMs, int where) {
-
-  //// Check to see if we are running in standalone mode... if so, do nothing
-  //if (Cmi_charmrun_pid == 0)
-  //  return;
-
-  AMMASSO_STATS_START(CommunicationServer)
-
-  MACHSTATE2(2, "CommunicationServer(%d) from %d {",withDelayMs, where);
-
-  // Check to see if this call is from an interrupt
-  if (where == 1) {
-    
-    // Don't service charmrum if converse exits, this fixed a hang bug
-    if (!machine_initiated_shutdown)
-      ServiceCharmrun_nolock();
-    
-    // Don't process any further for interrupts
-    return;
-  }
-
-  CmiCommLock();
-  inProgress[CmiMyRank()] += 1;
-  CommunicationServer_nolock(withDelayMs);
-  CmiCommUnlock();
-  inProgress[CmiMyRank()] -= 1;
-
-#if CMK_IMMEDIATE_MSG
-  if (where == 0)
-  CmiHandleImmediate();
-#endif
-
-  MACHSTATE(2,"} CommunicationServer");
-
-  AMMASSO_STATS_END(CommunicationServer)
-}
-
-
-
-/* CmiMachineExit()
- *
- */
-void CmiMachineExit(void) {
-
-  char buf[128];
-  cc_status_t rtn;
-  int i;
-
-  MACHSTATE(2, "CmiMachineExit() - INFO: Called...");
-
-  // DMK - This is a sleep to help keep the output from the stat displays below separated in the program output
-  if (contextBlock->myNode)
-    usleep(10000*contextBlock->myNode);
-
-  AMMASSO_STATS_DISPLAY(MachineInit)
-
-  AMMASSO_STATS_DISPLAY(AmmassoDoIdle)
-
-  AMMASSO_STATS_DISPLAY(DeliverViaNetwork)
-  AMMASSO_STATS_DISPLAY(DeliverViaNetwork_pre_lock)
-  AMMASSO_STATS_DISPLAY(DeliverViaNetwork_lock)
-  AMMASSO_STATS_DISPLAY(DeliverViaNetwork_post_lock)
-  AMMASSO_STATS_DISPLAY(DeliverViaNetwork_send)
-
-  AMMASSO_STATS_DISPLAY(getQPSendBuffer)
-  AMMASSO_STATS_DISPLAY(getQPSendBuffer_lock)
-  AMMASSO_STATS_DISPLAY(getQPSendBuffer_CEH)
-  AMMASSO_STATS_DISPLAY(getQPSendBuffer_loop)
-
-  AMMASSO_STATS_DISPLAY(sendDataOnQP)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_pre_send)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_send)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_post_send)
-
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_1024)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_2048)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_4096)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_16384)
-  AMMASSO_STATS_DISPLAY(sendDataOnQP_over)
-
-  AMMASSO_STATS_DISPLAY(AsynchronousEventHandler)
-  AMMASSO_STATS_DISPLAY(CompletionEventHandler)
-  AMMASSO_STATS_DISPLAY(ProcessMessage)
-  AMMASSO_STATS_DISPLAY(processAmmassoControlMessage)
-
-  AMMASSO_STATS_DISPLAY(sendAck)
-
-  // TODO: It would probably be a good idea to make a "closing connection" control message so all of the nodes
-  //       can agree to close the connections in a graceful way when they are all done.  Similar to the READY packet
-  //       but for closing so the closing is graceful (and the other node does not try to reconnect).  This barrier
-  //       would go here.
-
-  // Check to see if in stand-alone mode
-  if (Cmi_charmrun_pid != 0 && contextBlock->rnic != -1) {
-
-    // Do Clean-up for each of the OtherNode structures (the connections and related data)
-    for (i = 0; i < contextBlock->numNodes; i++) {
-
-      // Close all the connections and destroy all the QPs (and related data structures)
-      closeQPConnection(nodes + i, 1);  // The program is closing so destroy the QPs
-
-      // Destroy the locks
-      CmiDestroyLock(nodes[i].sendBufLock);
-      CmiDestroyLock(nodes[i].send_next_lock);
-      CmiDestroyLock(nodes[i].recv_expect_lock);
-    }
-
-    // DMK : TODO : Clean up the buffer pool here (unregister the memory from the RNIC and free it)
-
-    //// Close the RNIC interface
-    rtn = cc_rnic_close(contextBlock->rnic);
-    if (rtn != CC_OK) {
-      MACHSTATE2(5, "CmiMachineExit() - ERROR: Unable to close the RNIC: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      sprintf(buf, "CmiMachineExit() - ERROR: Unable to close the RNIC: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      CmiAbort(buf);
-    }
-
-    MACHSTATE(2, "CmiMachineExit() - INFO: RNIC Closed.");
-  }
-}
-
-
-
-OtherNode getNodeFromQPId(cc_qp_id_t qp_id) {
-
-  int i;
-
-  // DMK : FIXME : This is a horrible linear search through the nodes table to find the node structure that has this
-  //               qp_id but currently this is the fastest way that I know how to do this... In the future to speed it up.
-  for (i = 0; i < contextBlock->numNodes; i++)
-    if (nodes[i].qp_id == qp_id)
-      return (nodes + i);
-
-  return NULL;
-}
-
-OtherNode getNodeFromQPHandle(cc_qp_handle_t qp) {
-
-  int i;
-
-  // DMK : FIXME : This is a horrible linear search through the nodes table to find the node structure that has this
-  //               qp_id but currently this is the fastest way that I know how to do this... In the future to speed it up.
-  for (i = 0; i < contextBlock->numNodes; i++)
-    if (nodes[i].qp == qp)
-      return (nodes + i);
-
-  return NULL;
-}
-
-
-void AsynchronousEventHandler(cc_rnic_handle_t rnic, cc_event_record_t *er, void *cb) {
-
-  int nodeNumber, i;
-  OtherNode node;
-  cc_ep_handle_t connReqEP;
-  cc_status_t rtn;
-  char buf[64];
-  cc_qp_modify_attrs_t modAttrs;
-  AmmassoPrivateData *priv;
-  AmmassoToken *token;
-
-  AMMASSO_STATS_START(AsynchronousEventHandler)
-
-  MACHSTATE2(2, "AsynchronousEventHandler() - INFO: Called... event_id = %d, \"%s\"", er->event_id, cc_event_id_to_string(er->event_id));
-
-  // Do a couple of checks... the reasons for these stem from some example code
-  if (er->rnic_handle != contextBlock->rnic) {
-    MACHSTATE(5, "AsynchronousEventHandler() - WARNING: er->rnic_handle != contextBlock->rnic");
-  }
-  if (er->rnic_user_context != contextBlock) {
-    MACHSTATE(5, "AsynchronousEventHandler() - WARNING: er->rnic_user_context != contextBlock");
-  }
-
-  // Based on the er->event_id, do something about it
-  switch (er->event_id) {
-
-    // er->event_id == CCAE_LLP_CLOSE_COMPLETE
-    case CCAE_LLP_CLOSE_COMPLETE:
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Connection Closed.");
-
-      // Get the OtherNode structure for the other node
-      MACHSTATE2(1, "AsynchronousEventHandler() - INFO: er->resource_indicator = %d (CC_RES_IND_QP: %d)", er->resource_indicator, CC_RES_IND_QP);
-      node = getNodeFromQPId(er->resource_id.qp_id);
-      if (node == NULL) {
-        MACHSTATE(5, "AsynchronousEventHandler() - ERROR: Unable to find QP from QP ID... Unable to create/recover connection");
-        break;
-      }
-
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Pre-sendBufLock");
-      #if CMK_SHARED_VARS_UNAVAILABLE
-        while (node->sendBufLock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-      #endif
-      CmiLock(node->sendBufLock);
-
-      // Set the state of the connection to closed
-      node->connectionState = QP_CONN_STATE_CONNECTION_CLOSED;
-  
-      CmiUnlock(node->sendBufLock);
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Post-sendBufLock");
-    
-      break;
-
-    // er->event_id == CCAE_CONNECTION_REQUEST
-    case CCAE_CONNECTION_REQUEST:
-
-      MACHSTATE2(1, "AsynchronousEventHandler() - INFO: Incomming Connection Request -> %s:%d",
-                    inet_ntoa(*(struct in_addr*) &(er->event_data.connection_request.laddr)),
-                    ntohs(er->event_data.connection_request.lport)
-                );
-
-      connReqEP = er->event_data.connection_request.cr_handle;
-
-      priv = (AmmassoPrivateData*)er->event_data.connection_request.private_data;
-
-      nodeNumber = priv->node;
-      MACHSTATE1(3, "AsynchronousEventHandler() - INFO: Connection Request from node %d", nodeNumber);
-
-      if (nodeNumber < 0 || nodeNumber >= contextBlock->numNodes) {
-
-        // Refuse the connection and log the rejection
-        MACHSTATE1(1, "AsynchronousEventHandler() - WARNING: Unknown entity attempting to connect (node %d)... rejecting connection.", nodeNumber);
-        cc_cr_reject(contextBlock->rnic, connReqEP);
-
-      } else {
-
-
-        { int j;
-          for (j = 0; j < 16; j++) {
-            MACHSTATE2(3, "                                      private_data[%d] = %02X", j, ((char*)er->event_data.connection_request.private_data)[j]);
-         }
-       }
-
-        // Grab the remote stag and to, by the protocol, all the tokens are
-        // consecutive in memory
-       for (i=0; i<(AMMASSO_INITIAL_BUFFERS/(contextBlock->numNodes-1)); ++i) {
-         LIST_DEQUEUE(contextBlock->,freeTokens,token);
-         token->wr.wr_u.rdma_write.remote_stag = priv->stag;
-         token->wr.wr_u.rdma_write.remote_to = priv->to + (i * sizeof(AmmassoBuffer));
-         token->remoteBuf = (AmmassoBuffer*)(priv->to + (i * sizeof(AmmassoBuffer)));
-         LIST_ENQUEUE(nodes[nodeNumber].,sendTokens,token);
-       }
-
-       nodes[nodeNumber].ack_sq_wr->wr_u.rdma_write.remote_to = priv->ack_to;
-       nodes[nodeNumber].ack_sq_wr->wr_u.rdma_write.remote_stag = priv->stag;
-
-        MACHSTATE2(1, "AsynchronousEventHandler() - INFO: tokens starting from %p, stag = %d",priv->to,priv->stag);
-
-        // Keep a copy of the end point handle
-        nodes[nodeNumber].cr = connReqEP;
-
-        // Accept the connection
-        priv = (AmmassoPrivateData*)buf;
-        priv->node = contextBlock->myNode;
-        priv->stag = nodes[nodeNumber].recv_buf->stag;
-        priv->to = (cc_uint64_t)nodes[nodeNumber].recv_buf;
-       priv->ack_to = (cc_uint64_t)nodes[nodeNumber].directAck;
-
-        MACHSTATE1(1, "                                          node = %d", priv->node);
-        MACHSTATE1(1, "                                          stag = %d", priv->stag);
-        MACHSTATE1(1, "                                          to = %p", priv->to);
-        
-        { int j;
-          MACHSTATE2(3, "                                      buf = %p, priv = %p", buf, priv);
-          for (j = 0; j < 16; j++) {
-            MACHSTATE2(3, "                                      ((char*)priv)[%d] = %02X", j, ((char*)priv)[j]);
-         }
-       }
-
-        MACHSTATE(1, "AsynchronousEventHandler() - Ammasso - INFO: Accepting Connection...");
-
-        rtn = cc_cr_accept(contextBlock->rnic, connReqEP, nodes[nodeNumber].qp, sizeof(AmmassoPrivateData), (cc_uint8_t*)priv);
-        if (rtn != CC_OK) {
-
-          // Let the user know what happened
-          MACHSTATE1(3, "AsynchronousEventHandler() - Ammasso - WARNING: Unable to accept connection from node %d", nodeNumber);
-
-       } else {
-  
-          MACHSTATE1(3, "AsynchronousEventHandler() - Ammasso - INFO: Accepted Connection from node %d", nodeNumber);
-
-          MACHSTATE(1, "AsynchronousEventHandler() - INFO: Pre-sendBufLock");
-          #if CMK_SHARED_VARS_UNAVAILABLE
-            while (nodes[nodeNumber].sendBufLock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-          #endif
-          CmiLock(nodes[nodeNumber].sendBufLock);
-
-          // Indicate that this connection has been made only if is it the first time the connection was made (don't count re-connects)
-          if (nodes[nodeNumber].connectionState == QP_CONN_STATE_PRE_CONNECT)
-            (contextBlock->outstandingConnectionCount)--;
-          nodes[nodeNumber].connectionState = QP_CONN_STATE_CONNECTED;
-
-          CmiUnlock(nodes[nodeNumber].sendBufLock);
-          MACHSTATE(1, "AsynchronousEventHandler() - INFO: Post-sendBufLock");
-
-          MACHSTATE1(1, "AsynchronousEventHandler() - Connected to node %d", nodes[nodeNumber].myNode);
-       }
-      }
-
-      break;
-
-    // er->event_id == CCAE_ACTIVE_CONNECT_RESULTS
-    case CCAE_ACTIVE_CONNECT_RESULTS:
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Connection Results");
-
-      // Get the OtherNode structure for the other node
-      MACHSTATE2(1, "AsynchronousEventHandler() - INFO: er->resource_indicator = %d (CC_RES_IND_QP: %d)", er->resource_indicator, CC_RES_IND_QP);
-      node = getNodeFromQPId(er->resource_id.qp_id);
-      if (node == NULL) {
-        MACHSTATE(5, "AsynchronousEventHandler() - ERROR: Unable to find QP from QP ID... Unable to create/recover connection");
-        break;
-      }
-
-      // Check to see if the connection was established or not
-      if (er->event_data.active_connect_results.status != CC_CONN_STATUS_SUCCESS) {
-
-        MACHSTATE(5, "                                     Connection Failed.");
-        MACHSTATE1(5, "                                      - status: \"%s\"", cc_connect_status_to_string(er->event_data.active_connect_results.status));
-        MACHSTATE1(5, "                                      - private_data_length = %d", er->event_data.active_connect_results.private_data_length);
-        displayQueueQuery(node->qp, &(node->qp_attrs));
-
-        // Attempt to reconnect (try again... don't give up... you can do it!)
-        reestablishQPConnection(node);
-
-      } else { // Connection was a success
-
-        MACHSTATE(3, "                                     Connection Success...");
-        MACHSTATE2(1, "                                     l -> %s:%d", inet_ntoa(*(struct in_addr*) &(er->event_data.active_connect_results.laddr)), ntohs(er->event_data.active_connect_results.lport));
-        MACHSTATE2(1, "                                     r -> %s:%d", inet_ntoa(*(struct in_addr*) &(er->event_data.active_connect_results.raddr)), ntohs(er->event_data.active_connect_results.rport));
-        MACHSTATE4(1, "                                     private_data_length = %d (%d, %d, %d)", er->event_data.active_connect_results.private_data_length, sizeof(int), sizeof(cc_stag_t), sizeof(cc_uint64_t));
-
-        priv = (AmmassoPrivateData*)((char*)er->event_data.active_connect_results.private_data);
-
-        { int j;
-         MACHSTATE2(1, "                                      private_data = %p, priv = %p", er->event_data.active_connect_results.private_data, priv);
-          for (j = 0; j < 16; j++) {
-            MACHSTATE3(1, "                                      private_data[%d] = %02X (priv:%02X)", j, ((char*)(er->event_data.active_connect_results.private_data))[j], ((char*)priv)[j]);
-         }
-       }
-
-        // Grab the remote stag and to, by the protocol, all the tokens are
-        // consecutive in memory
-       for (i=0; i<(AMMASSO_INITIAL_BUFFERS/(contextBlock->numNodes-1)); ++i) {
-         LIST_DEQUEUE(contextBlock->,freeTokens,token);
-         token->wr.wr_u.rdma_write.remote_stag = priv->stag;
-         token->wr.wr_u.rdma_write.remote_to = priv->to + (i * sizeof(AmmassoBuffer));
-         token->remoteBuf = (AmmassoBuffer*)(priv->to + (i * sizeof(AmmassoBuffer)));
-         LIST_ENQUEUE(node->,sendTokens,token);
-       }
-
-       node->ack_sq_wr->wr_u.rdma_write.remote_to = priv->ack_to;
-       node->ack_sq_wr->wr_u.rdma_write.remote_stag = priv->stag;
-
-       MACHSTATE1(3, "                                          node = %d", priv->node);
-       MACHSTATE1(3, "                                          stag = %d", priv->stag);
-       MACHSTATE1(3, "                                          to = %p", priv->to);
-        
-        MACHSTATE2(1, "AsynchronousEventHandler() - INFO: tokens from %p, stag = %d",priv->to,priv->stag);
-
-        MACHSTATE(1, "AsynchronousEventHandler() - INFO: Pre-sendBufLock");
-        #if CMK_SHARED_VARS_UNAVAILABLE
-          while (node->sendBufLock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-        #endif
-        CmiLock(node->sendBufLock);
-
-        // Indicate that this connection has been made only if it is the first time the connection was made (don't count re-connects)
-        if (node->connectionState == QP_CONN_STATE_PRE_CONNECT)
-          (contextBlock->outstandingConnectionCount)--;
-        node->connectionState = QP_CONN_STATE_CONNECTED;
-
-        CmiUnlock(node->sendBufLock);
-        MACHSTATE(1, "AsynchronousEventHandler() - INFO: Post-sendBufLock");
-
-        MACHSTATE1(1, "AsynchronousEventHandler() - Connected to node %d", node->myNode);
-      }
-
-      break;
-
-    // er->event_id is set to a value that indicates the QP will transition into the TERMINATE state
-      // Remotely Detected Errors
-    case CCAE_TERMINATE_MESSAGE_RECEIVED:
-      // LLP Errors
-    case CCAE_LLP_SEGMENT_SIZE_INVALID:
-    case CCAE_LLP_INVALID_CRC:
-    case CCAE_LLP_BAD_FPDU:
-      // Remote Operation Errors
-    case CCAE_INVALID_DDP_VERSION:
-    case CCAE_INVALID_RDMA_VERSION:
-    case CCAE_UNEXPECTED_OPCODE:
-    case CCAE_INVALID_DDP_QUEUE_NUMBER:
-    case CCAE_RDMA_READ_NOT_ENABLED:
-    case CCAE_NO_L_BIT:
-      // Remote Protection Errors (not associated with RQ)
-    case CCAE_TAGGED_INVALID_STAG:
-    case CCAE_TAGGED_BASE_BOUNDS_VIOLATION:
-    case CCAE_TAGGED_ACCESS_RIGHTS_VIOLATION:
-    case CCAE_TAGGED_INVALID_PD:
-    case CCAE_WRAP_ERROR:
-      // Remote Closing Error
-    case CCAE_BAD_LLP_CLOSE:
-      // Remote Protection Errors (associated with RQ)
-    case CCAE_INVALID_MSN_RANGE:
-    case CCAE_INVALID_MSN_GAP:
-      // IRRQ Proection Errors
-    case CCAE_IRRQ_INVALID_STAG:
-    case CCAE_IRRQ_BASE_BOUNDS_VIOLATION:
-    case CCAE_IRRQ_ACCESS_RIGHTS_VIOLATION:
-    case CCAE_IRRQ_INVALID_PD:
-    case CCAE_IRRQ_WRAP_ERROR:  // For these, the VERBS
-    case CCAE_IRRQ_OVERFLOW:    // spec is 100% clear as to
-    case CCAE_IRRQ_MSN_GAP:     // what to do... but I think
-    case CCAE_IRRQ_MSN_RANGE:   // this is correct... DMK
-      // Local Errors - ??? Not 100% sure about these (they are written differently in cc_ae.h than in the verbs spec... but I think they go here)
-    case CCAE_CQ_SQ_COMPLETION_OVERFLOW:
-    case CCAE_CQ_RQ_COMPLETION_ERROR:
-    case CCAE_QP_SRQ_WQE_ERROR:
-    
-    // er->event_id is set to a value that indicates the QP will transition into the ERROR state
-      // LLP Errors
-    case CCAE_LLP_CONNECTION_LOST:
-    case CCAE_LLP_CONNECTION_RESET:
-      // Remote Closing Error
-    case CCAE_BAD_CLOSE:
-      // Local Errors
-    case CCAE_QP_LOCAL_CATASTROPHIC_ERROR:
-      MACHSTATE3(5, "AsynchronousEventHandler() - WARNING: Connection Error \"%s\" - er->resource_indicator = %d (CC_RES_IND_QP: %d)", cc_event_id_to_string(er->event_id), er->resource_indicator, CC_RES_IND_QP);
-      CmiPrintf("AsynchronousEventHandler() - WARNING: Connection Error \"%s\" - er->resource_indicator = %d (CC_RES_IND_QP: %d)\n", cc_event_id_to_string(er->event_id), er->resource_indicator, CC_RES_IND_QP);
-
-      // Figure out which QP went down
-      node = getNodeFromQPId(er->resource_id.qp_id);
-      if (node == NULL) {
-        MACHSTATE(5, "AsynchronousEventHandler() - ERROR: Unable to find QP from QP ID... Unable to recover connection");
-        break;
-      }
-
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Pre-sendBufLock");
-      #if CMK_SHARED_VARS_UNAVAILABLE
-        while (node->sendBufLock != 0) { usleep(1); } // Since CmiLock() is not really a lock, actually wait
-      #endif
-      CmiLock(node->sendBufLock);
-
-      // Indicate that the connection was lost or will be lost in the very near future (depending on the er->event_id)
-      node->connectionState = QP_CONN_STATE_CONNECTION_LOST;
-
-      CmiUnlock(node->sendBufLock);
-      MACHSTATE(1, "AsynchronousEventHandler() - INFO: Post-sendBufLock");
-
-      MACHSTATE1(1, "AsynchronousEventHandler() -        Connection ERROR Occured - node %d", node->myNode);
-      displayQueueQuery(node->qp, &(node->qp_attrs));
-
-      // Attempt to bring the connection back to life
-      reestablishQPConnection(node);
-
-      break;
-
-    // er->event_id == ???
-    default:
-      MACHSTATE1(5, "AsynchronousEventHandler() - WARNING - Unknown/Unexpected Asynchronous Event: er->event_id = %d", er->event_id);
-      break;
-
-  } // end switch (er->event_id)
-
-  AMMASSO_STATS_END(AsynchronousEventHandler)
-}
-
-void CheckRecvBufForMessage(OtherNode node) {
-
-  int needAck;
-  unsigned int len;
-  AmmassoBuffer *current;
-
-  MACHSTATE1(2, "CheckRecvBufForMessage() - INFO: Called... (node->recv_buf = %p)...", node->recv_buf);
-
-  // Process all messages, identified by a length not zero
-  while ((len = node->recv_buf->tail.length) != 0) {
-
-    MACHSTATE1(2, "                                           (len = %d)...", len);
-
-    // Start by zero-ing out the length of the message so it isn't picked up again
-    node->recv_buf->tail.length = 0;
-    (*node->remoteAck) ++;
-    node->messagesNotYetAcknowledged ++;
-
-    current = node->recv_buf;
-    // Move the recv_buf back at the end of the receiving queue. This works also
-    // if the queue if formed by a single element
-    node->last_recv_buf->next = node->recv_buf;
-    node->last_recv_buf = node->recv_buf;
-    node->recv_buf = node->recv_buf->next;
-    node->last_recv_buf->next = NULL;
-
-    // Process the messsage, NOTE that the message start is aligned to 8 bytes!
-    needAck = ProcessMessage(&(current->buf[AMMASSO_BUFSIZE - ALIGN8(len)]), len, &current->tail, node);
-
-    // If an ACK is needed in response to this message, send one
-    if (needAck) sendAck(node);
-  }
-
-}
-
-/*
-void CompletionEventHandler(cc_rnic_handle_t rnic, cc_cq_handle_t cq, void *cb) {
-  CompletionEventHandlerWithAckFlag(rnic, cq, cb, 0);
-}
-
-void CompletionEventHandlerWithAckFlag(cc_rnic_handle_t rnic, cc_cq_handle_t cq, void *cb, int breakOnAck) {
-
-  OtherNode node = (OtherNode)cb;
-  cc_status_t rtn;
-  cc_wc_t wc;
-  int tmp;
-  cc_rq_wr_t *rq_wr;
-  char* recvBuf;
-  char needAck;
-
-  AMMASSO_STATS_START(CompletionEventHandler)
-
-  //MACHSTATE(3, "CompletionEventHandler() - Called...");
-
-  //// Reset the request notification type
-  //rtn = cc_cq_request_notification(contextBlock->rnic, cq, CC_CQ_NOTIFICATION_TYPE_NEXT);
-  //if (rtn != CC_OK) {
-  //  // Let the user know what happened
-  //  MACHSTATE2(3, "CompletionEventHandler() - Ammasso - WARNING - Unable to reset CQ request notification type: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-  //}
-
-  // Keep polling the Completion Queue until it is empty of Work Completions
-  while (1) {
-
-    //if (PollForMessage(cq) == 0) break;
-
-    // Poll the Completion Queue
-    rtn = cc_cq_poll(contextBlock->rnic, cq, &wc);
-    //if (rtn == CCERR_CQ_EMPTY) break;
-    if (rtn != CC_OK) break;
-
-    // Let the user know if there was an error
-    if (wc.status != CC_OK) {
-      MACHSTATE2(3, "CompletionEventHandler() - Ammasso - WARNING - WC status not CC_OK: %d, \"%s\"", wc.status, cc_status_to_string(wc.status));
-
-      // DMK : TODO : Add code here that will recover from a Work Request Gone Wrong... i.e. - if a connection is lost, the
-      //              WRs that are still pending in the queues will complete in failure with a FLUSHED error status... these
-      //              WRs will have to be reposed after the QP is back into the IDLE state in reestablishQPConnection (or later).
-    }
-
-    // Depending on the WC.type, react accordingly
-    switch (wc.wr_type) {
-
-      // wc.wr_type == CC_WR_TYPE_RDMA_READ
-      case CC_WR_TYPE_RDMA_READ:
-        // DMK : TODO : Fill this in
-        break;
-
-      // wc.wr_type == CC_WR_TYPE_RDMA_WRITE
-      case CC_WR_TYPE_RDMA_WRITE:
-        // DMK : TODO : Fill this in
-        break;
-
-      // wc.wr_type == CC_WR_TYPE_SEND
-      case CC_WR_TYPE_SEND:
-
-        // This will be called as a result of a cc_qp_post_sq()'s Send Work Request in DeliverViaNetwork finishing
-
-        // DMK : NOTE : I left this here as an example (if message memory is registered with the RNIC on the fly
-        //              in DeliverViaNetwork(), the memory would have to be un-registered/un-pinned here).
-        // Free the STag that was created
-        //rtn = cc_stag_dealloc(contextBlock->rnic, wc.stag);
-
-        // Unlock the send_buf so it can be used again
-        // DMK : TODO : Make this a real lock
-        //MACHSTATE1(3, "CompletionEventHandler() - INFO: Send completed... clearing sendBufLock for next send (%p)", node);
-        //CmiUnlock(node->sendBufLock);
-
-        MACHSTATE1(3, "CompletionEventHandler() - INFO: Send completed with node %d... still waiting for acknowledge...", node->myNode);
-
-        break;
-
-      // wc.wr_type == CC_WR_TYPE_RECV
-      case CC_WR_TYPE_RECV:
-
-        // Check for CCERR_FLUSHED which mean "we're shutting down" according to the example code
-        if (wc.status == CCERR_FLUSHED) {
-          displayQueueQuery(node->qp, &(node->qp_attrs));
-          break;
-       }
-
-        // Make sure that node is defined
-        if (node == NULL) {
-          MACHSTATE(3, "CompletionEventHandler() - WARNING: Received message but node == NULL... ignoring...");
-          break;
-        }
-
-        displayQueueQuery(node->qp, &(node->qp_attrs));
-
-        {
-          int j;
-          MACHSTATE1(3, "CompletionEventHandler() - INFO: fromNode = %d", node->myNode);
-          MACHSTATE1(3, "CompletionEventHandler() - INFO: wc.status = %d", wc.status);
-          MACHSTATE1(3, "CompletionEventHandler() - INFO: wc.bytes_rcvd = %d", wc.bytes_rcvd);
-          MACHSTATE1(3, "CompletionEventHandler() - INFO: node = %p", node);
-          //MACHSTATE1(3, "CompletionEventHandler() - INFO: &(nodes[0]) = %p", &(nodes[0]));
-          //MACHSTATE1(3, "CompletionEventHandler() - INFO: &(nodes[1]) = %p", &(nodes[1]));
-          //MACHSTATE1(3, "CompletionEventHandler() - INFO: nodes[0].recv_buf = %p", nodes[0].recv_buf);
-          //MACHSTATE1(3, "CompletionEventHandler() - INFO: nodes[1].recv_buf = %p", nodes[1].recv_buf);
-          //MACHSTATE(3, "CompletionEventHandler() - INFO: Raw Message Data:");
-          //for (j = 0; j < DGRAM_HEADER_SIZE; j++) {
-          //  //MACHSTATE1(3, "                         [%02x]", node->recv_buf[j]);
-          //  MACHSTATE2(3, "                         [0:%02x, 1:%02x]", nodes[0].recv_buf[j], nodes[1].recv_buf[j]);
-         //}
-        }
-
-        // Get the address of the receive buffer used
-        rq_wr = (cc_rq_wr_t*)wc.wr_id;
-        
-        // Process the Message
-        ProcessMessage((char*)(rq_wr->local_sgl.sge_list[0].to), wc.bytes_rcvd, &needAck);
-        //ProcessMessage(node->recv_buf, wc.bytes_rcvd);
-
-        // Re-Post the receive buffer
-        // DMK : NOTE : I'm doing this after processing the message because I'm not 100% sure if the RNIC is "smart" enought
-        //              to not reuse the buffer if still in this event handler.
-        tmp = 0;
-        rtn = cc_qp_post_rq(contextBlock->rnic, node->qp, rq_wr, 1, &tmp);
-        if (rtn != CC_OK || tmp != 1) {
-          // Let the user know what happened
-          MACHSTATE2(3, "CompletionEventHandler() - Ammasso - ERROR: Unable to Post Work Request to Queue Pair: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-        }
-
-        // Send and ACK to indicate this node is ready to receive another message
-        if (needAck)
-          sendAck(node);
-
-        // Check to see if the function should return (stop polling for messages)
-        if (breakOnAck && (!needAck))   // NOTE: Should only not need an ACK if ACK arrived or error
-          return;
-
-        break;
-
-      // default
-      default:
-       MACHSTATE1(3, "CompletionEventHandler() - Ammasso - WARNING - Unknown WC.wr_type: %d", wc.wr_type);
-       break;
-
-    } // end switch (wc.wr_type)
-  } // end while (1)
-
-  AMMASSO_STATS_END(CompletionEventHandler)
-}
-*/
-
-// NOTE: DMK: The code here follows from open_tcp_sockets() in machine-tcp.c.
-void CmiAmmassoOpenQueuePairs(void) {
-
-  char buf[128];
-  int i, myNode, numNodes, keepWaiting;
-  int buffersPerNode;
-  cc_qp_create_attrs_t qpCreateAttrs;
-  cc_status_t rtn;
-  cc_inet_addr_t address;
-  cc_inet_port_t port;
-  AmmassoBuffer *sendBuffer;
-  AmmassoBuffer *bufferScanner;
-  AmmassoToken *newTokens, *tokenScanner;
-  cc_data_addr_t *newSgls;
-  cc_stag_index_t newStagIndex;
-  ammasso_ack_t *ack_location;
-
-
-  MACHSTATE1(2, "CmiAmmassoOpenQueuePairs() - INFO: Called... (Cmi_charmrun_pid = %d)", Cmi_charmrun_pid);
-
-  // Check for stand-alone mode... no connections needed
-  if (Cmi_charmrun_pid == 0) return;
-
-  if (nodes == NULL) {
-    MACHSTATE(5, "CmiAmmassoOpenQueuePairs() - WARNING: nodes = NULL");
-    return;
-  }
-  MACHSTATE1(1, "CmiAmmassoOpenQueuePairs() - INFO: nodes = %p (remove this line)", nodes);
-
-  // DMK : FIXME : At this point, CmiMyNode() seems to be returning 0 on any node while _Cmi_mynode is
-  // !!!!!!!!!!!   returning the correct value.  However, _Cmi_mynode and _Cmi_numnodes may not work with
-  // !!!!!!!!!!!   SMP.  Resolve this issue and fix this code.  For now, using _Cmi_mynode and _Cmi_numnodes.
-  //myNode = CmiMyNode();
-  //numNodes = CmiNumNodes();
-  contextBlock->myNode = myNode = _Cmi_mynode;
-  contextBlock->numNodes = numNodes = _Cmi_numnodes;
-  contextBlock->outstandingConnectionCount = contextBlock->numNodes - 1;  // No connection with self
-  contextBlock->nodeReadyCount = contextBlock->numNodes - 1;              // No ready packet from self
-  contextBlock->conditionRegistered = 0;
-
-  MACHSTATE2(1, "CmiAmmassoOpenQueuePairs() - INFO: myNode = %d, numNodes = %d", myNode, numNodes);
-
-  CmiAssert(sizeof(AmmassoBuffer) == (sizeof(AmmassoBuffer)&(~63)));
-
-  // Try to allocate the memory for the receiving buffers
-  contextBlock->freeRecvBuffers = (AmmassoBuffer*) CmiAlloc(AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t));
-
-  ack_location = (ammasso_ack_t*)&(contextBlock->freeRecvBuffers[AMMASSO_INITIAL_BUFFERS]);
-  if (contextBlock->freeRecvBuffers == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for RECV buffers");
-    sprintf(buf, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for RECV buffers");
-    CmiAbort(buf);
-  }
-
-  contextBlock->pinnedMemory = AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t);
-  CC_CHECK(cc_nsmr_register_virt,(contextBlock->rnic,
-                                 CC_ADDR_TYPE_VA_BASED,
-                                 (cc_byte_t*)contextBlock->freeRecvBuffers,
-                                 AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t),
-                                 contextBlock->pd_id,
-                                 0, 0,
-                                 CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE | CC_ACF_REMOTE_WRITE,
-                                 &newStagIndex)
-          );
-
-  for (i=0; i<AMMASSO_INITIAL_BUFFERS; ++i) {
-    contextBlock->freeRecvBuffers[i].tail.length = 0;
-    contextBlock->freeRecvBuffers[i].next = &(contextBlock->freeRecvBuffers[i+1]);
-    contextBlock->freeRecvBuffers[i].stag = newStagIndex;
-  }
-  contextBlock->freeRecvBuffers[AMMASSO_INITIAL_BUFFERS-1].next = NULL;
-  contextBlock->last_freeRecvBuffers = &contextBlock->freeRecvBuffers[AMMASSO_INITIAL_BUFFERS-1];
-
-  buffersPerNode = AMMASSO_INITIAL_BUFFERS / (contextBlock->numNodes-1);
-
-  // distribute all the buffers allocated to the different processors, together
-  // the the buffer where to receive the directly sent ACK
-  bufferScanner = contextBlock->freeRecvBuffers;
-  contextBlock->freeRecvBuffers = contextBlock->freeRecvBuffers[(contextBlock->numNodes-1)*buffersPerNode-1].next;
-  contextBlock->num_freeRecvBuffers = AMMASSO_INITIAL_BUFFERS - (contextBlock->numNodes-1)*buffersPerNode;
-  for (i=0; i<contextBlock->numNodes; ++i) {
-    if (i == contextBlock->myNode) continue;
-    nodes[i].num_recv_buf = buffersPerNode;
-    nodes[i].recv_buf = bufferScanner;
-    bufferScanner[buffersPerNode-1].next = NULL;
-    nodes[i].last_recv_buf = &(bufferScanner[buffersPerNode-1]);
-    nodes[i].pending = NULL;
-    bufferScanner += buffersPerNode;  // move forward of buffersPerNode buffers (of size sizeof(AmmassoBuffer))
-    nodes[i].directAck = ack_location;
-    ack_location++;
-  }
-
-  // Try to allocate the memory for the sending buffers, together with the
-  // buffers from where the direct ACK will be sent
-  sendBuffer = (AmmassoBuffer*) CmiAlloc(AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t));
-
-  if (sendBuffer == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    sprintf(buf, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    CmiAbort(buf);
-  }
-
-  contextBlock->pinnedMemory += AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t);
-  CC_CHECK(cc_nsmr_register_virt,(contextBlock->rnic,
-                                 CC_ADDR_TYPE_VA_BASED,
-                                 (cc_byte_t*)sendBuffer,
-                                 AMMASSO_INITIAL_BUFFERS*sizeof(AmmassoBuffer) + (contextBlock->numNodes-1)*sizeof(ammasso_ack_t),
-                                 contextBlock->pd_id,
-                                 0, 0,
-                                 CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE,
-                                 &newStagIndex)
-          );
-
-  contextBlock->freeTokens = NULL;
-  contextBlock->num_freeTokens = 0;
-
-  // Allocate the send tokens, together with the tokens for the ACK buffers
-  newTokens = (AmmassoToken*) CmiAlloc((AMMASSO_INITIAL_BUFFERS+contextBlock->numNodes-1)*ALIGN8(sizeof(AmmassoToken)));
-
-  if (newTokens == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    sprintf(buf, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    CmiAbort(buf);
-  }
-
-  newSgls = (cc_data_addr_t*) CmiAlloc((AMMASSO_INITIAL_BUFFERS+contextBlock->numNodes-1)*ALIGN8(sizeof(cc_data_addr_t)));
-
-  if (newSgls == NULL) {
-
-    // Attempt to close the RNIC
-    cc_rnic_close(contextBlock->rnic);
-
-    // Let the user know what happened and bail
-    MACHSTATE(5, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    sprintf(buf, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to allocate memory for SEND buffers");
-    CmiAbort(buf);
-  }
-
-  contextBlock->num_freeTokens = 0;
-  contextBlock->last_freeTokens = NULL;
-  contextBlock->freeTokens = NULL;
-  tokenScanner = newTokens;
-  for (i=0; i<AMMASSO_INITIAL_BUFFERS; ++i) {
-    newSgls->stag = newStagIndex;
-    newSgls->length = AMMASSO_BUFSIZE + sizeof(Tailer);
-    newSgls->to = (unsigned long)&(sendBuffer[i]);
-    tokenScanner->wr.wr_id = (unsigned long)tokenScanner;
-    tokenScanner->wr.wr_type = CC_WR_TYPE_RDMA_WRITE;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_count = 1;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_list = newSgls;
-    tokenScanner->wr.signaled = 1;
-    tokenScanner->localBuf = (AmmassoBuffer*)&(sendBuffer[i]);
-    LIST_ENQUEUE(contextBlock->,freeTokens,tokenScanner);
-    newSgls = (cc_data_addr_t*)(((char*)newSgls)+ALIGN8(sizeof(cc_data_addr_t)));
-    tokenScanner = (AmmassoToken*)(((char*)tokenScanner)+ALIGN8(sizeof(AmmassoToken)));
-  }
-
-  /* At this point, newSgls, tokenScanner, and ack_location point to the first
-     element to be used for the ACK buffers */
-
-  // Setup the ack_sq_wr for all nodes
-  for (i=0; i<contextBlock->numNodes; ++i) {
-    if (i == contextBlock->myNode) continue;
-    newSgls->stag = newStagIndex;
-    newSgls->length = sizeof(ammasso_ack_t);
-    newSgls->to = (unsigned long)ack_location;
-    nodes[i].remoteAck = ack_location;
-    tokenScanner->wr.wr_id = (unsigned long)tokenScanner;
-    tokenScanner->wr.wr_type = CC_WR_TYPE_RDMA_WRITE;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_count = 1;
-    tokenScanner->wr.wr_u.rdma_write.local_sgl.sge_list = newSgls;
-    tokenScanner->wr.signaled = 1;
-    nodes[i].ack_sq_wr = &tokenScanner->wr;
-    newSgls = (cc_data_addr_t*)(((char*)newSgls)+ALIGN8(sizeof(cc_data_addr_t)));
-    tokenScanner = (AmmassoToken*)(((char*)tokenScanner)+ALIGN8(sizeof(AmmassoToken)));
-    ack_location++;
-  }
-
-  // Loop through all of the PEs
-  //   Begin setting up the Queue Pairs (common code for both "server" and "client")
-  //   For nodes with a lower PE, set up a "server" connection (accept)
-  //   For nodes with a higher PE, connect as "client" (connect)
-
-  // Loop through all the nodes in the setup
-  for (i = 0; i < numNodes; i++) {
-
-    // Setup any members of this nodes OtherNode structure that need setting up
-    nodes[i].myNode = i;
-    nodes[i].connectionState = QP_CONN_STATE_PRE_CONNECT;
-    nodes[i].messagesNotYetAcknowledged = 0;
-    nodes[i].usedTokens = NULL;
-    nodes[i].last_usedTokens = NULL;
-    nodes[i].num_usedTokens = 0;
-    nodes[i].localAck = 0;
-    nodes[i].sendBufLock = CmiCreateLock();
-    nodes[i].send_next_lock = CmiCreateLock();
-    nodes[i].recv_expect_lock = CmiCreateLock();
-    nodes[i].max_used_tokens = 0;
-    //nodes[i].send_UseIndex = 0;
-    //nodes[i].send_InUseCounter = 0;
-    //nodes[i].recv_UseIndex = 0;
-
-    // If you walk around talking to yourself people will look at you all funny-like.  Try not to do that.
-    if (i == myNode) continue;
-
-    *nodes[i].remoteAck = 0;
-
-    // Establish the Connection
-    establishQPConnection(nodes + i, 0); // Don't reuse the QP (there isn't one yet) 
-
-  } // end for (i < numNodes)
-
-
-  // Need to block here until all the connections for this node are made
-  MACHSTATE(2, "CmiAmmassoOpenQueuePairs() - INFO: Waiting for all connections to be established...");
-  while (contextBlock->outstandingConnectionCount > 0) {
-
-    usleep(1000);
-
-    for (i = 0; i < contextBlock->numNodes; i++) {
-      if (i == contextBlock->myNode) continue;
-      //CompletionEventHandler(contextBlock->rnic, nodes[i].recv_cq, &(nodes[i]));
-      //CompletionEventHandler(contextBlock->rnic, nodes[i].send_cq, &(nodes[i]));
-      CheckRecvBufForMessage(&(nodes[i]));
-    }
-  }
-  MACHSTATE(1, "CmiAmmassoOpenQueuePairs() - INFO: All Connections have been established... Continuing");
-
-  // Pause a little so both ends of the connection have time to receive and process the asynchronous events
-  //usleep(800000); // 800ms
-  sleep(1);
-
-  MACHSTATE(1, "CmiAmmassoOpenQueuePairs() - INFO: Sending ready to all neighboors...");
-
-  // Send all the ready packets
-  for (i = 0; i < numNodes; i++) {
-    int tmp;
-    char buf[24];
-
-    if (i == myNode) continue;  // Skip self
-
-    MACHSTATE1(1, "CmiAmmassoOpenQueuePairs() - INFO: Sending READY to node %d", i);
-
-    // Send a READY control message to the node, give a non-null length
-    sendDataOnQP(buf, 1, &(nodes[i]), AMMASSO_READY);
-
-    /*
-    // Set the sendBufLock
-    CmiLock(nodes[i].sendBufLock);
-
-    // Setup the message
-    *(  (int*)(nodes[i].send_buf    )) = Cmi_charmrun_pid;   // Send the charmrun PID
-    *((short*)(nodes[i].send_buf + 4)) = myNode;             // Send this node's number
-
-    // Post the send
-    nodes[i].send_sgl.length = 6;   // DMK : TODO : FIXME : Change this later when multiple buffers are supported
-    rtn = cc_qp_post_sq(contextBlock->rnic, nodes[i].qp, &(nodes[i].sq_wr), 1, &tmp);
-    if (rtn != CC_OK || tmp != 1) {
-      // Free the sendBufLock
-      CmiUnlock(nodes[i].sendBufLock);
-
-      // Let the user know what happened
-      MACHSTATE1(3, "CmiAmmassoOpenQueuePairs() - ERROR: Unable to send READY packet to node %d.", i);
-    }
-    */
-
-    // Note: The code in the completion event handler will unlock sendBufLock again after the packet has actually been sent
-  }
-
-  // DMK : NOTE : I don't think this is really needed... (leaving for now just incase I find out it really is)
-  //// Wait until all the ready packets have been sent
-  //while (keepWaiting) {
-  //  int j;
-  //
-  //  // Assume we won't find a lock that is set
-  //  keepWaiting = 0;
-  //
-  //  // Check all the locks, if one is set, sleep and try again
-  //  for (j = 0; i < numNodes; j++)
-  //    if (nodes[j].sendBufLock) {
-  //      keepWaiting = 1;
-  //      usleep(10000); // sleep 10ms
-  //      break;
-  //    }
-  //}
-
-  MACHSTATE(1, "CmiAmmassoOpenQueuePairs() - INFO: All ready packets sent to neighboors...");
-
-  // Need to block here until all of the ready packets have been received
-  // NOTE : Because this is a fully connection graph of connections between the nodes, this will block all the nodes
-  //        until all the nodes are ready (and all the PEs since there is a node barrier in the run pe function that
-  //        all the threads execute... the thread executing this is one of those so it has to reach that node barrier
-  //        before any of the other can start doing much of anything).
-  MACHSTATE(2, "CmiAmmassoOpenQueuePairs() - INFO: Waiting for all neighboors to be ready...");
-  while (contextBlock->nodeReadyCount > 0) {
-    usleep(10000);  // Sleep 10ms
-    
-    for (i = 0; i < contextBlock->numNodes; i++) {
-      if (i == contextBlock->myNode) continue;
-      //CompletionEventHandler(contextBlock->rnic, nodes[i].recv_cq, &(nodes[i]));
-      //CompletionEventHandler(contextBlock->rnic, nodes[i].send_cq, &(nodes[i]));
-      CheckRecvBufForMessage(&(nodes[i]));
-    }
-  }
-  MACHSTATE(1, "CmiAmmassoOpenQueuePairs() - INFO: All neighboors ready...");
-
-  MACHSTATE(2, "CmiAmmassoOpenQueuePairs() - INFO: Finished.");
-}
-
-
-
-// NOTE: When reestablishing a connection, the QP can be reused so don't recreate a new one (reuseQPFlag = 1).
-//       When openning the connection for the first time, there is no QP so create one (reuseQPFlag = 0).
-// DMK : TODO : Fix the comment and parameter (I've been playing with what reuseQPFlag actually does and got
-//              tired of updating comments)... update the comment when this is finished).
-void establishQPConnection(OtherNode node, int reuseQPFlag) {
-
-  cc_qp_create_attrs_t qpCreateAttrs;
-  cc_status_t rtn;
-  int i;
-  cc_uint32_t numWRsPosted;
-
-  MACHSTATE1(2, "establishQPConnection() - INFO: Called for node %d...", node->myNode);
-
-  ///// Shared "Client" and "Server" Code /////
-
-  MACHSTATE(1, "establishQPConnection() - INFO: (PRE-RECV-CQ-CREATE)");
-
-  // Create the Completion Queue, just create a fake one since with rdma writes it is not used
-  node->recv_cq_depth = 1;
-  CC_CHECK(cc_cq_create,(contextBlock->rnic, &(node->recv_cq_depth), contextBlock->eh_id, node, &(node->recv_cq)));
-
-  MACHSTATE(1, "establishQPConnection() - INFO: (PRE-SEND-CQ-CREATE)");
-
-  // Create the Completion Queue
-  //node->send_cq_depth = AMMASSO_NUMMSGBUFS_PER_QP * 4;
-  node->send_cq_depth = AMMASSO_BUFFERS_INFLY;
-  CC_CHECK(cc_cq_create,(contextBlock->rnic, &(node->send_cq_depth), contextBlock->eh_id, node, &(node->send_cq)));
-
-  MACHSTATE(1, "establishQPConnection() - INFO: (PRE-QP-CREATE)");
-
-  // Create the Queue Pair
-  // Set some initial Create Queue Pair Attributes that will be reused for all Queue Pairs Created
-  qpCreateAttrs.sq_cq = node->send_cq;   // Set the Send Queue's Completion Queue
-  qpCreateAttrs.rq_cq = node->recv_cq;   // Set the Request Queue's Completion Queue
-  qpCreateAttrs.sq_depth = node->send_cq_depth;
-  qpCreateAttrs.rq_depth = node->recv_cq_depth;
-  qpCreateAttrs.srq = 0;
-  qpCreateAttrs.rdma_read_enabled = 1;
-  qpCreateAttrs.rdma_write_enabled = 1;
-  qpCreateAttrs.rdma_read_response_enabled = 1;
-  qpCreateAttrs.mw_bind_enabled = 0;
-  qpCreateAttrs.zero_stag_enabled = 0;
-  qpCreateAttrs.send_sgl_depth = 1;
-  qpCreateAttrs.recv_sgl_depth = 1;
-  qpCreateAttrs.rdma_write_sgl_depth = 1;
-  qpCreateAttrs.ord = 1;
-  qpCreateAttrs.ird = 1;
-  qpCreateAttrs.pdid = contextBlock->pd_id;      // Set the Protection Domain
-  qpCreateAttrs.user_context = node;             // Set the User Context Block that will be passed into function calls
-
-  CC_CHECK(cc_qp_create,(contextBlock->rnic, &qpCreateAttrs, &(node->qp), &(node->qp_id)));
-
-  // Since the QP was just created (or re-created), reset the sequence number and any other variables that need reseting
-  //node->send_InUseCounter = 0;
-  //node->send_UseIndex = 0;
-  node->sendBufLock = 0;
-  node->send_next = 0;
-  node->send_next_lock = CmiCreateLock();
-  node->recv_expect = 0;
-  node->recv_expect_lock = CmiCreateLock();
-  node->recv_UseIndex = 0;
-
-  if (!reuseQPFlag) {
-
-    MACHSTATE(1, "establishQPConnection() - INFO: (PRE-NSMR-REGISTER-VIRT QP-QUERY-ATTRS)");
-
-    // Attempt to register the qp_attrs member of the OtherNode structure with the RNIC so the Queue Pair's state can be queried
-    contextBlock->pinnedMemory += sizeof(cc_qp_query_attrs_t);
-    CC_CHECK(cc_nsmr_register_virt,(contextBlock->rnic,
-                                   CC_ADDR_TYPE_VA_BASED,
-                                   (cc_byte_t*)(&(node->qp_attrs)),
-                                   sizeof(cc_qp_query_attrs_t),
-                                   contextBlock->pd_id,
-                                   0, 0,
-                                   CC_ACF_LOCAL_READ | CC_ACF_LOCAL_WRITE,
-                                   &(node->qp_attrs_stag_index))
-            );
-  }
-
-  ///// "Server" Specific /////
-  if (node->myNode < contextBlock->myNode) {
-
-    int count = 64;
-    char value[64];
-    int j;
-
-    MACHSTATE(1, "establishQPConnection() - INFO: Starting \"Server\" Code...");
-
-    // Setup the address
-    CC_CHECK(cc_rnic_getconfig,(contextBlock->rnic, CC_GETCONFIG_ADDRS, &count, &value));
-
-    //MACHSTATE1(3, "establishQPConnection() - count = %4d", count);
-    //for (j = 0; j < count; j++) {
-    //  MACHSTATE2(3, "establishQPConnection() - value[%d] = %4d", j, (int)value[j]);
-    //}
-      
-    // Setup the Address
-    // DMK : TODO : FIXME : Fix this code so that it handles host-network/big-little endian ordering
-    *(((char*)&(node->address)) + 0) = value[0];
-    *(((char*)&(node->address)) + 1) = value[1];
-    *(((char*)&(node->address)) + 2) = value[2];
-    *(((char*)&(node->address)) + 3) = value[3];
-
-    // Setup the Port
-    node->port = htons(AMMASSO_PORT + node->myNode);
-
-    MACHSTATE4(1, "establishQPConnection() - Using Address (Hex) 0x%02X 0x%02X 0x%02X 0x%02X", ((node->address >> 24) & 0xFF), ((node->address >> 16) & 0xFF), ((node->address >> 8) & 0xFF), (node->address & 0xFF));
-    MACHSTATE4(1, "                                        (Dec) %4d %4d %4d %4d", ((node->address >> 24) & 0xFF), ((node->address >> 16) & 0xFF), ((node->address >> 8) & 0xFF), (node->address & 0xFF));
-    MACHSTATE2(1, "                                   Port (Hex) 0x%02X 0x%02X", ((node->port >> 8) & 0xFF), (node->port & 0xFF));
-
-    /* Listen for an incomming connection (NOTE: This call will return
-       immediately; when a connection attempt is made by a "client", the
-       asynchronous handler will be called.) */
-    CC_CHECK(cc_ep_listen_create,(contextBlock->rnic, node->address, &(node->port), 3, contextBlock, &(node->ep)));
-
-    MACHSTATE(1, "establishQPConnection() - Listening...");
-  }
-
-
-  ///// "Client" Specific /////
-  if (node->myNode > contextBlock->myNode) {
-    int first;
-    AmmassoPrivateData priv;
-
-    // A one-time sleep that should give the passive side QPs time to post the listens before the active sides start trying to connect
-    if (node->myNode == contextBlock->myNode + 1 || reuseQPFlag)  // Only do once if all the connections are being made for the first time, do this for all
-      usleep(400000);  // Sleep 400ms                             // connections if reconnecting so the other RNIC has time to setup the listen
-
-    MACHSTATE(1, "establishQPConnection() - INFO: Starting \"Client\" Code...");
-
-    // Setup the Address
-    // DMK : TODO : FIXME : Fix this code so that it handles host-network/big-little endian ordering
-    *(((char*)&(node->address)) + 0) =  *(((char*)&(node->addr.sin_addr.s_addr)) + 0);
-    *(((char*)&(node->address)) + 1) =  *(((char*)&(node->addr.sin_addr.s_addr)) + 1);
-    *(((char*)&(node->address)) + 2) =  *(((char*)&(node->addr.sin_addr.s_addr)) + 2);
-    *(((char*)&(node->address)) + 3) = (*(((char*)&(node->addr.sin_addr.s_addr)) + 3)) - 1;
-
-    // Setup the Port
-    node->port = htons(AMMASSO_PORT + contextBlock->myNode);
-
-    MACHSTATE4(1, "establishQPConnection() - Using Address (Hex) 0x%02X 0x%02X 0x%02X 0x%02X", ((node->address >> 24) & 0xFF), ((node->address >> 16) & 0xFF), ((node->address >> 8) & 0xFF), (node->address & 0xFF));
-    MACHSTATE4(1, "                                        (Dec) %4d %4d %4d %4d", ((node->address >> 24) & 0xFF), ((node->address >> 16) & 0xFF), ((node->address >> 8) & 0xFF), (node->address & 0xFF));
-    MACHSTATE2(1, "                                   Port (Hex) 0x%02X 0x%02X", ((node->port >> 8) & 0xFF), (node->port & 0xFF));
-
-    /* Attempt to make a connection to a "server" (NOTE: This call will return
-       immediately; when the connection to the "server" is established, the
-       asynchronous handler will be called.) */
-
-    // by allocation protocol, the stags will be the same
-    priv.node = contextBlock->myNode;
-    priv.stag = node->recv_buf->stag;
-    priv.to = (cc_uint64_t)node->recv_buf;
-    priv.ack_to = (cc_uint64_t)node->directAck;
-
-    CC_CHECK(cc_qp_connect,(contextBlock->rnic, node->qp, node->address, node->port, sizeof(AmmassoPrivateData), (cc_uint8_t*)&priv));
-
-  }
-}
-
-
-// NOTE: This will be called in the event of a connection error
-void reestablishQPConnection(OtherNode node) {
-
-  cc_status_t rtn;
-  char buf[16];
-  cc_qp_modify_attrs_t modAttrs;
-  cc_wc_t wc;
-
-  MACHSTATE1(2, "reestablishQPConnection() - INFO: For node %d: Clearing Outstanding WRs...", node->myNode);
-
-  // Drain the RECV completion Queue (if a connection is lost, all pending Work Requests are completed with a FLUSHED error)
-  while (1) {
-    // Pool for a message
-    rtn = cc_cq_poll(contextBlock->rnic, node->recv_cq, &wc);
-    if (rtn == CCERR_CQ_EMPTY) break;
-
-    // DMK : TODO : FIXME : Something should be done with the WRs that are pulled off so they can be reissued
-  }  
-
-  // Drain the SEND completion Queue (if a connection is lost, all pending Work Requests are completed with a FLUSHED error)
-  while (1) {
-    // Pool for a message
-    rtn = cc_cq_poll(contextBlock->rnic, node->send_cq, &wc);
-    if (rtn == CCERR_CQ_EMPTY) break;
-
-    // DMK : TODO : FIXME : Something should be done with the WRs that are pulled off so they can be reissued
-  }  
-
-  MACHSTATE1(1, "reestablishQPConnection() - INFO: For node %d: Waiting for QP to enter ERROR state...", node->myNode);
-
-  do {
-
-    // Query the QP's state
-    rtn = cc_qp_query(contextBlock->rnic, node->qp, &(node->qp_attrs));
-    if (rtn != CC_OK) {
-      MACHSTATE2(5, "AsynchronousEventHandler() - ERROR: Unable to Query Queue Pair (l): %d, \"%s\"", rtn, cc_status_to_string(rtn));
-      break;
-    }
-
-    // Check to see if the state is ERROR, if so, break from the loop... otherwise, keep waiting, it will be soon
-    if (node->qp_attrs.qp_state == CC_QP_STATE_ERROR)
-      break;
-    else
-      usleep(1000); // 1ms
-
-  } while (1);
-
-  MACHSTATE2(1, "reestablishQPConnection() - INFO: Finished waiting node %d: QP state = \"%s\"...", node->myNode, cc_qp_state_to_string(node->qp_attrs.qp_state));
-  MACHSTATE1(1, "reestablishQPConnection() - INFO: Attempting to transition QP into IDLE state for node %d", node->myNode);
-
-  // Transition the Queue Pair from ERROR into IDLE state
-  modAttrs.llp_ep = node->ep;
-  modAttrs.next_qp_state = CC_QP_STATE_IDLE;
-  modAttrs.ord = CC_QP_NO_ATTR_CHANGE;
-  modAttrs.ird = CC_QP_NO_ATTR_CHANGE;
-  modAttrs.sq_depth = CC_QP_NO_ATTR_CHANGE;
-  modAttrs.rq_depth = CC_QP_NO_ATTR_CHANGE;
-  modAttrs.stream_message_buffer = NULL;
-  modAttrs.stream_message_length = 0;
-  rtn = cc_qp_modify(contextBlock->rnic, node->qp, &modAttrs);
-  if (rtn != CC_OK) {
-    // Let the user know what happened
-    MACHSTATE2(5, "reestablishQPConnection() - ERROR: Unable to Modify QP State: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-  }
-
-  rtn = cc_qp_query(contextBlock->rnic, node->qp, &(node->qp_attrs));
-  if (rtn != CC_OK) {
-    MACHSTATE2(5, "reestablishQPConnection() - ERROR: Unable to Query Queue Pair (1): %d, \"%s\"", rtn, cc_status_to_string(rtn));
-  }
-  MACHSTATE2(1, "reestablishQPConnection() - INFO: Transition results for node %d: QP state = \"%s\"...", node->myNode, cc_qp_state_to_string(node->qp_attrs.qp_state));
-
-  closeQPConnection(node, 0);      // Close the connection but do not destroy the QP
-  establishQPConnection(node, 1);  // Reopen the connection and reuse the QP that has already been created
-}
-
-
-// NOTE: When reestablishing a connection, the QP can be reused so don't destroy it and create a new one (destroyQPFlag = 0).
-//       When closing the connection because the application is terminating, destroy the QP (destroyQPFlat != 0).
-// DMK : TODO : Fix the comment and parameter (I've been playing with what destroyQPFlag actually does and got
-//              tired of updating comments)... update the comment when this is finished).
-void closeQPConnection(OtherNode node, int destroyQPFlag) {
-
-  MACHSTATE(2, "closeQPConnection() - INFO: Called...");
-
-  /*
-  // Close the Completion Queues
-  cc_qp_destroy(contextBlock->rnic, node->qp);
-  cc_cq_destroy(contextBlock->rnic, node->send_cq);
-  cc_cq_destroy(contextBlock->rnic, node->recv_cq);
-
-  if (destroyQPFlag) {
-    // De-Register Memory with the RNIC
-    cc_stag_dealloc(contextBlock->rnic, node->qp_attrs_stag_index);
-    cc_stag_dealloc(contextBlock->rnic, node->recv_stag_index);
-    cc_stag_dealloc(contextBlock->rnic, node->send_stag_index);
-    //cc_stag_dealloc(contextBlock->rnic, node->rdma_stag_index);
-
-    CmiFree(node->recv_buf);
-    CmiFree(node->rq_wr);
-    CmiFree(node->recv_sgl);
-
-    CmiFree(node->send_buf);
-    CmiFree(node->sq_wr);
-    CmiFree(node->send_sgl);
-    //CmiFree(node->send_bufFree);
-  }
-  */
-}
-
-
-char* cc_status_to_string(cc_status_t errorCode) {
-
-  switch (errorCode) {
-
-    case CC_OK: return "OK";
-    case CCERR_INSUFFICIENT_RESOURCES:       return "Insufficient Resources";
-    case CCERR_INVALID_MODIFIER:             return "Invalid Modifier";
-    case CCERR_INVALID_MODE:                 return "Invalid Mode";
-    case CCERR_IN_USE:                       return "In Use";
-    case CCERR_INVALID_RNIC:                 return "Invalid RNIC";
-    case CCERR_INTERRUPTED_OPERATION:        return "Interrupted Operation";
-    case CCERR_INVALID_EH:                   return "Invalid EH";
-    case CCERR_INVALID_CQ:                   return "Invalid CQ";
-    case CCERR_CQ_EMPTY:                     return "CQ Empty";
-    case CCERR_NOT_IMPLEMENTED:              return "Not Implemented";
-    case CCERR_CQ_DEPTH_TOO_SMALL:           return "CQ Depth Too Small";
-    case CCERR_PD_IN_USE:                    return "PD In Use";
-    case CCERR_INVALID_PD:                   return "Invalid PD";
-    case CCERR_INVALID_SRQ:                  return "Invalid SRQ";
-    case CCERR_INVALID_ADDRESS:              return "Invalid Address";
-    case CCERR_INVALID_NETMASK:              return "Invalid Netmask";
-    case CCERR_INVALID_QP:                   return "Invalid QP";
-    case CCERR_INVALID_QP_STATE:             return "Invalid QP State";
-    case CCERR_TOO_MANY_WRS_POSTED:          return "Too Many WRs Posted";
-    case CCERR_INVALID_WR_TYPE:              return "Invalid WR Type";
-    case CCERR_INVALID_SGL_LENGTH:           return "Invalid SGL Length";
-    case CCERR_INVALID_SQ_DEPTH:             return "Invalid SQ Depth";
-    case CCERR_INVALID_RQ_DEPTH:             return "Invalid RQ Depth";
-    case CCERR_INVALID_ORD:                  return "Invalid ORD";
-    case CCERR_INVALID_IRD:                  return "Invalid IRD";
-    case CCERR_QP_ATTR_CANNOT_CHANGE:        return "QP_ATTR_CANNON_CHANGE";
-    case CCERR_INVALID_STAG:                 return "Invalid STag";
-    case CCERR_QP_IN_USE:                    return "QP In Use";
-    case CCERR_OUTSTANDING_WRS:              return "Outstanding WRs";
-    // case CCERR_MR_IN_USE:   NOTE : "CCERR_MR_IN_USE = CCERR_STAG_IN_USE" in "cc_status.h"
-    case CCERR_STAG_IN_USE:                  return "STag In Use";
-    case CCERR_INVALID_STAG_INDEX:           return "Invalid STag Index";
-    case CCERR_INVALID_SGL_FORMAT:           return "Invalid SGL Format";
-    case CCERR_ADAPTER_TIMEOUT:              return "Adapter Timeout";
-    case CCERR_INVALID_CQ_DEPTH:             return "Invalid CQ Depth";
-    case CCERR_INVALID_PRIVATE_DATA_LENGTH:  return "Invalid Private Data Length";
-    case CCERR_INVALID_EP:                   return "Invalid EP";
-    case CCERR_FLUSHED:                      return "Flushed";
-    case CCERR_INVALID_WQE:                  return "Invalid WQE";
-    case CCERR_LOCAL_QP_CATASTROPHIC_ERROR:  return "Local QP Catastrophic Error";
-    case CCERR_REMOTE_TERMINATION_ERROR:     return "Remote Termination Error";
-    case CCERR_BASE_AND_BOUNDS_VIOLATION:    return "Base and Bounds Violation";
-    case CCERR_ACCESS_VIOLATION:             return "Access Violation";
-    case CCERR_INVALID_PD_ID:                return "Invalid PD ID";
-    case CCERR_WRAP_ERROR:                   return "Wrap Error";
-    case CCERR_INV_STAG_ACCESS_ERROR:        return "Invalid STag Access Error";
-    case CCERR_ZERO_RDMA_READ_RESOURCES:     return "Zero RDMA Read Resources";
-    case CCERR_QP_NOT_PRIVILEGED:            return "QP Not Privileged";
-    case CCERR_STAG_STATE_NOT_INVALID:       return "STag State Not Invalid";  // ???
-    case CCERR_INVALID_PAGE_SIZE:            return "Invalid Page Size";
-    case CCERR_INVALID_BUFFER_SIZE:          return "Invalid Buffer Size";
-    case CCERR_INVALID_PBE:                  return "Invalid PBE";
-    case CCERR_INVALID_FBO:                  return "Invalid FBO";
-    case CCERR_INVALID_LENGTH:               return "Invalid Length";
-    case CCERR_INVALID_ACCESS_RIGHTS:        return "Invalid Access Rights";
-    case CCERR_PBL_TOO_BIG:                  return "PBL Too Big";
-    case CCERR_INVALID_VA:                   return "Invalid VA";
-    case CCERR_INVALID_REGION:               return "Invalid Region";
-    case CCERR_INVALID_WINDOW:               return "Invalid Window";
-    case CCERR_TOTAL_LENGTH_TOO_BIG:         return "Total Length Too Big";
-    case CCERR_INVALID_QP_ID:                return "Invalid QP ID";
-    case CCERR_ADDR_IN_USE:                  return "Address In Use";
-    case CCERR_ADDR_NOT_AVAIL:               return "Address Not Available";
-    case CCERR_NET_DOWN:                     return "Network Down";
-    case CCERR_NET_UNREACHABLE:              return "Network Unreachable";
-    case CCERR_CONN_ABORTED:                 return "Connection Aborted";
-    case CCERR_CONN_RESET:                   return "Connection Reset";
-    case CCERR_NO_BUFS:                      return "No Buffers";
-    case CCERR_CONN_TIMEDOUT:                return "Connection Timed-Out";
-    case CCERR_CONN_REFUSED:                 return "Connection Refused";
-    case CCERR_HOST_UNREACHABLE:             return "Host Unreachable";
-    case CCERR_INVALID_SEND_SGL_DEPTH:       return "Invalid Send SGL Depth";
-    case CCERR_INVALID_RECV_SGL_DEPTH:       return "Invalid Receive SGL Depth";
-    case CCERR_INVALID_RDMA_WRITE_SGL_DEPTH: return "Ivalid RDMA Write SGL Depth";
-    case CCERR_INSUFFICIENT_PRIVILEGES:      return "Insufficient Privileges";
-    case CCERR_STACK_ERROR:                  return "Stack Error";
-    case CCERR_INVALID_VERSION:              return "Invalid Version";
-    case CCERR_INVALID_MTU:                  return "Invalid MTU";
-    case CCERR_INVALID_IMAGE:                return "Invalid Image";
-    case CCERR_PENDING:                      return "(PENDING: Internal to Adapter... Hopefully you aren't reading this...)";   /* not an error; user internally by adapter */
-    case CCERR_DEFER:                        return "(DEFER: Internal to Adapter... Hopefully you aren't reading this...)";     /* not an error; used internally by adapter */
-    case CCERR_FAILED_WRITE:                 return "Failed Write";
-    case CCERR_FAILED_ERASE:                 return "Failed Erase";
-    case CCERR_FAILED_VERIFICATION:          return "Failed Verification";
-    case CCERR_NOT_FOUND:                    return "Not Found";
-    default:                                 return "Unknown Error Code";
-  }
-}
-
-// NOTE: Letting these string be separate incase different information should be
-//   returned that what cc_status_to_string() would return
-char* cc_conn_error_to_string(cc_connect_status_t errorCode) {
-
-  switch (errorCode) {
-    case CC_CONN_STATUS_SUCCESS:          return "Success";
-    case CC_CONN_STATUS_NO_MEM:           return "No Memory";
-    case CC_CONN_STATUS_TIMEDOUT:         return "Timed-Out";
-    case CC_CONN_STATUS_REFUSED:          return "Refused";
-    case CC_CONN_STATUS_NETUNREACH:       return "Network Unreachable";
-    case CC_CONN_STATUS_HOSTUNREACH:      return "Host Unreachable";
-    case CC_CONN_STATUS_INVALID_RNIC:     return "Invalid RNIC";
-    case CC_CONN_STATUS_INVALID_QP:       return "Invalid QP";
-    case CC_CONN_STATUS_INVALID_QP_STATE: return "Invalid QP State";
-    case CC_CONN_STATUS_REJECTED:         return "Rejected";
-    default:                              return (cc_status_to_string((cc_status_t)errorCode));
-  }
-}
-
-void displayQueueQuery(cc_qp_handle_t qp, cc_qp_query_attrs_t *attrs) {
-
-  //cc_qp_query_attrs_t attr;
-  cc_status_t rtn;
-  char buf[1024];
-
-  OtherNode node = getNodeFromQPHandle(qp);
-  if (node != NULL) {
-    MACHSTATE1(2, "displayQueueQuery() - Called for node %d", node->myNode);
-  } else {
-    MACHSTATE(2, "displayQueueQuery() - Called for unknown node");
-  }
-
-  // Query the Queue for its Attributes
-  rtn = cc_qp_query(contextBlock->rnic, qp, attrs);
-  if (rtn != CC_OK) {
-    // Let the user know what happened
-    MACHSTATE2(5, "displayQueueQuery() - ERROR: Unable to query queue: %d, \"%s\"", rtn, cc_status_to_string(rtn));
-    return;
-  }
-
-  // Output the results of the Query
-  // DMK : TODO : For now I'm only putting in the ones that I care about... add more later or as needed
-  MACHSTATE2(1, "displayQueueQuery() - qp_state = %d, \"%s\"", attrs->qp_state, cc_qp_state_to_string(attrs->qp_state));
-  if (attrs->terminate_message_length > 0) {
-    memcpy(buf, attrs->terminate_message, attrs->terminate_message_length);
-    buf[attrs->terminate_message_length] = '\0';
-    MACHSTATE1(1, "displayQueueQuery() - terminate_message = \"%s\"", buf);
-  } else {
-    MACHSTATE(1, "displayQueueQuery() - terminate_message = NULL");
-  }
-}
-
-char* cc_qp_state_to_string(cc_qp_state_t qpState) {
-
-  switch (qpState) {
-    case CC_QP_STATE_IDLE:       return "IDLE";
-    case CC_QP_STATE_CONNECTING: return "CONNECTED";
-    case CC_QP_STATE_RTS:        return "RTS";
-    case CC_QP_STATE_CLOSING:    return "CLOSING";
-    case CC_QP_STATE_TERMINATE:  return "TERMINATE";
-    case CC_QP_STATE_ERROR:      return "ERROR";
-    default:                     return "unknown";
-  }
-}
-
-char* cc_event_id_to_string(cc_event_id_t id) {
-
-  switch(id) {
-    case CCAE_REMOTE_SHUTDOWN:                 return "Remote Shutdown";
-    case CCAE_ACTIVE_CONNECT_RESULTS:          return "Active Connect Results";
-    case CCAE_CONNECTION_REQUEST:              return "Connection Request";
-    case CCAE_LLP_CLOSE_COMPLETE:              return "LLP Close Complete";
-    case CCAE_TERMINATE_MESSAGE_RECEIVED:      return "Terminate Message Received";
-    case CCAE_LLP_CONNECTION_RESET:            return "LLP Connection Reset";
-    case CCAE_LLP_CONNECTION_LOST:             return "LLP Connection Lost";
-    case CCAE_LLP_SEGMENT_SIZE_INVALID:        return "Segment Size Invalid";
-    case CCAE_LLP_INVALID_CRC:                 return "LLP Invalid CRC";
-    case CCAE_LLP_BAD_FPDU:                    return "LLP Bad FPDU";
-    case CCAE_INVALID_DDP_VERSION:             return "Invalid DDP Version";
-    case CCAE_INVALID_RDMA_VERSION:            return "Invalid RMDA Version";
-    case CCAE_UNEXPECTED_OPCODE:               return "Unexpected Opcode";
-    case CCAE_INVALID_DDP_QUEUE_NUMBER:        return "Invalid DDP Queue Number";
-    case CCAE_RDMA_READ_NOT_ENABLED:           return "RDMA Read Not Enabled";
-    case CCAE_RDMA_WRITE_NOT_ENABLED:          return "RDMA Write Not Enabled";
-    case CCAE_RDMA_READ_TOO_SMALL:             return "RDMA Read Too Small";
-    case CCAE_NO_L_BIT:                        return "No L Bit";
-    case CCAE_TAGGED_INVALID_STAG:             return "Tagged Invalid STag";
-    case CCAE_TAGGED_BASE_BOUNDS_VIOLATION:    return "Tagged Base Bounds Violation";
-    case CCAE_TAGGED_ACCESS_RIGHTS_VIOLATION:  return "Tagged Access Rights Violation";
-    case CCAE_TAGGED_INVALID_PD:               return "Tagged Invalid PD";
-    case CCAE_WRAP_ERROR:                      return "Wrap Error";
-    case CCAE_BAD_CLOSE:                       return "Bad Close";
-    case CCAE_BAD_LLP_CLOSE:                   return "Bad LLP Close";
-    case CCAE_INVALID_MSN_RANGE:               return "Invalid MSN Range";
-    case CCAE_INVALID_MSN_GAP:                 return "Invalid MSN Gap";
-    case CCAE_IRRQ_OVERFLOW:                   return "IRRQ Overflow";
-    case CCAE_IRRQ_MSN_GAP:                    return "IRRQ MSG Gap";
-    case CCAE_IRRQ_MSN_RANGE:                  return "IRRQ MSN Range";
-    case CCAE_IRRQ_INVALID_STAG:               return "IRRQ Invalid STag";
-    case CCAE_IRRQ_BASE_BOUNDS_VIOLATION:      return "IRRQ Base Bounds Violation";
-    case CCAE_IRRQ_ACCESS_RIGHTS_VIOLATION:    return "IRRQ Access Rights Violation";
-    case CCAE_IRRQ_INVALID_PD:                 return "IRRQ Invalid PD";
-    case CCAE_IRRQ_WRAP_ERROR:                 return "IRRQ Wrap Error";
-    case CCAE_CQ_SQ_COMPLETION_OVERFLOW:       return "CQ SQ Completion Overflow";
-    case CCAE_CQ_RQ_COMPLETION_ERROR:          return "CQ RQ Completion Overflow";
-    case CCAE_QP_SRQ_WQE_ERROR:                return "QP SRQ WQE Error";
-    case CCAE_QP_LOCAL_CATASTROPHIC_ERROR:     return "QP Local Catastrophic Error";
-    case CCAE_CQ_OVERFLOW:                     return "CQ Overflow";
-    case CCAE_CQ_OPERATION_ERROR:              return "CQ Operation Error";
-    case CCAE_SRQ_LIMIT_REACHED:               return "SRQ Limit Reached";
-    case CCAE_QP_RQ_LIMIT_REACHED:             return "QP RQ Limit Reached";
-    case CCAE_SRQ_CATASTROPHIC_ERROR:          return "SRQ Catastrophic Error";
-    case CCAE_RNIC_CATASTROPHIC_ERROR:         return "RNIC Catastrophic Error";
-    default:                                   return "Unknown Event ID";
-  }
-}
-
-char* cc_connect_status_to_string(cc_connect_status_t status) {
-
-  switch (status) {
-    case CC_CONN_STATUS_SUCCESS:     return "Success";
-    case CC_CONN_STATUS_TIMEDOUT:    return "Timedout";
-    case CC_CONN_STATUS_REFUSED:     return "Refused";
-    case CC_CONN_STATUS_NETUNREACH:  return "Network Unreachable";
-    default:                         return "Unknown";
-  }
-}
diff --git a/src/arch/net/machine-ammasso.h b/src/arch/net/machine-ammasso.h
deleted file mode 100644 (file)
index 046546a..0000000
+++ /dev/null
@@ -1,282 +0,0 @@
-/**
- ** Ammasso implementation of Converse NET version Contains Ammasso specific
- ** code for: CmiMachineInit() CmiNotifyIdle() DeliverViaNetwork()
- ** CommunicationServer() CmiMachineExit()
- **
- ** Written By:    Isaac Dooley      idooley2@uiuc.edu
- ** 03/12/05       Esteban Pauli     etpauli2@uiuc.edu
- **                Filippo Gioachin  gioachin@uiuc.edu
- **                David Kunzman     kunzman2@uiuc.edu
- **
- ** Change Log:
- **   03/12/05 : DMK : Initial Version
- **   04/30/05 : Filippo : Revised Version
- **
- ** Todo List:
- **
- **/
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Defines, Types, etc. ////////////////////////////////////////////////////////////////////////////
-
-#define AMMASSO_PORT        2583
-
-/* The size of the initial allocated buffers is AMMASSO_BUFSIZE*AMMASSO_INITIAL_BUFFERS
-   with 8500, the buffers become aligned to 64 bytes (16 bytes per buffer are support) */
-#define AMMASSO_BUFSIZE            8488
-#define AMMASSO_INITIAL_BUFFERS    1024
-#define AMMASSO_BUFFERS_INFLY       128
-
-/*
-#define AMMASSO_CTRLMSG_LEN        7
-#define CtrlHeader_Construct(buf, ctrlType)  { *((int*)buf) = Cmi_charmrun_pid;                               \
-                                               *((short*)((char*)buf + sizeof(int))) = contextBlock->myNode;  \
-                                               *((char*)buf + sizeof(int) + sizeof(short)) = ctrlType;        \
-                                             }
-
-#define CtrlHeader_GetCharmrunPID(buf)   (*((int*)buf))
-#define CtrlHeader_GetNode(buf)          (*((short*)((char*)buf + sizeof(int))))
-#define CtrlHeader_GetCtrlType(buf)      (*((char*)buf + sizeof(int) + sizeof(short)))
-*/
-
-#define ACK_MASK               (1<<31)
-
-/* ACK_WRAPPING is used when the ACK overflows, rare condition */
-#define ACK_WRAPPING           0x01
-/* AMMASSO_ALLOCATE is used by the receiving side to allocate more buffers on
-   the sender. The sender will reply with a AMMASSO_ALLOCATED message,
-   specifying so where they have been allocated. AMMASSO_MOREBUFFERS is instead
-   a message sent from the sender to the receiver asking for more buffers. The
-   receiver can decide to allocate more buffers or just ignore the message. */
-#define AMMASSO_ALLOCATE       0x02
-#define AMMASSO_ALLOCATED      0x04
-#define AMMASSO_MOREBUFFERS    0x08
-/* AMMASSO_RELEASE is used by the receiver to ask the sender to release a
-   certain amount of buffers which are allocated to it. The sender can answer
-   with an AMMASSO_RELEASED message specifying how many buffers have been
-   released. The sender can also decide to ignore an AMMASSO_RELEASE message, or
-   to release some buffers on its own idea. */
-#define AMMASSO_RELEASE        0x10
-#define AMMASSO_RELEASED       0x20
-/* AMMASSO_READY is used at the beginning of the program to synchronize all
-   processors */
-#define AMMASSO_READY          0x40
-
-typedef struct __cmi_idle_state {
-  char none;
-} CmiIdleState;
-
-typedef struct __ammasso_private_data {
-  int node;
-  cc_stag_t stag;
-  cc_uint64_t to;
-  cc_uint64_t ack_to;
-} AmmassoPrivateData;
-
-typedef struct __ammasso_token_description {
-  cc_stag_t stag;
-  cc_uint64_t to;
-} AmmassoTokenDescription;
-
-typedef CmiUInt4 ammasso_ack_t;
-
-/* Structures to deal with the buffering */
-typedef struct __ammasso_tailer {
-  ammasso_ack_t ack;
-  char flags;
-  char pad;
-  CmiUInt2 length;
-} PACKED Tailer;
-
-/* AMMASSO_BUFSIZE should be updated if this structure size is changed */
-typedef struct __ammasso_buffer {
-  char buf[AMMASSO_BUFSIZE];
-  Tailer tail;
-  union {
-    struct {
-      cc_stag_t stag;
-      struct __ammasso_buffer *next;
-    };
-    char pad[12];
-  };
-} PACKED AmmassoBuffer;
-
-typedef struct __ammasso_token {
-  AmmassoBuffer *localBuf;
-  AmmassoBuffer *remoteBuf;
-  struct __ammasso_token *next;
-  cc_sq_wr_t wr;
-} AmmassoToken;
-
-// if "name" is null, then "num_##name"==0 and "last_##name" is not defined
-#define LIST_DEFINE(type, name) \
-    type *name; \
-    type *last_ ## name; \
-    int    num_ ## name
-
-// NOTE: in order to use LIST_*, no space has to be present in the parenthesis
-#define LIST_ENQUEUE(prefix, suffix, newtoken) \
-    prefix num_ ## suffix ++; \
-    newtoken ->next = NULL; \
-    if (prefix suffix != NULL) { \
-      prefix last_ ## suffix ->next = newtoken; \
-    } else { \
-      prefix suffix = newtoken; \
-    } \
-    prefix last_ ## suffix = newtoken
-
-#define LIST_DEQUEUE(prefix, suffix, thetoken) \
-    CmiAssert(prefix num_ ## suffix > 0); \
-    prefix num_ ## suffix --; \
-    thetoken = prefix suffix; \
-    prefix suffix = thetoken ->next
-
-// DMK : This is copied from "qp_rping.c" in Ammasso's Example Code (modify as needed for our machine layer).
-// User Defined Context that will be sent to function handlers for asynchronous events.
-typedef struct __context_block {
-
-  cc_rnic_handle_t       rnic;     // The RNIC Handle
-  cc_eh_ce_handler_id_t  eh_id;    // Event Handler ID
-  cc_pdid_t              pd_id;    // Protection Domain Handle
-
-  // DMK : TODO : When there is a global buffer pool, the send and recv (or just single) completion queues' handles can go here
-  //              instead of in the OtherNode structure.  All of the memory buffers will move here also.
-
-  // Extra Useful Info
-  int                    numNodes;                    // The number of nodes total
-  int                    myNode;                      // "My" node index for this node
-  int                    outstandingConnectionCount;  // Synch. variable used to block until all connections to other nodes are made
-  int                    nodeReadyCount;              // Synch. variable used to block until all other nodes are ready
-
-  // Free Receive Buffer Pool, typically empty when allocated to processors
-  LIST_DEFINE(AmmassoBuffer,freeRecvBuffers);
-  //CmiNodeLock bufferPoolLock;
-  LIST_DEFINE(AmmassoToken,freeTokens);
-
-  int pinnedMemory;
-  int conditionRegistered;
-} mycb_t;
-
-// Global instance of the mycb_t structure to be used throughout this machine layer
-mycb_t *contextBlock = NULL;
-
-// NOTE: I couldn't find functions similar to these in the CCIL API, if found, use the API ones instead.  They
-//   are basically just utility functions.
-char* cc_status_to_string(cc_status_t errorCode);
-char* cc_conn_error_to_string(cc_connect_status_t errorCode);
-void displayQueueQuery(cc_qp_handle_t qp, cc_qp_query_attrs_t *attrs);
-char* cc_qp_state_to_string(cc_qp_state_t qpState);
-char* cc_event_id_to_string(cc_event_id_t id);
-char* cc_connect_status_to_string(cc_connect_status_t status);
-
-
-#define AMMASSO_STATS   0
-#if AMMASSO_STATS
-
-#define AMMASSO_STATS_VARS(event)   double event ## _start;    \
-                                    double event ## _end;      \
-                                    double event ## _total;    \
-                                    long event ## _count;
-
-typedef struct __ammasso_stats {
-
-  AMMASSO_STATS_VARS(MachineInit)
-
-  AMMASSO_STATS_VARS(AmmassoDoIdle)
-
-  AMMASSO_STATS_VARS(DeliverViaNetwork)
-  AMMASSO_STATS_VARS(DeliverViaNetwork_pre_lock)
-  AMMASSO_STATS_VARS(DeliverViaNetwork_lock)
-  AMMASSO_STATS_VARS(DeliverViaNetwork_post_lock)
-  AMMASSO_STATS_VARS(DeliverViaNetwork_send)
-
-  AMMASSO_STATS_VARS(getQPSendBuffer)
-  AMMASSO_STATS_VARS(getQPSendBuffer_lock)
-  AMMASSO_STATS_VARS(getQPSendBuffer_CEH)
-  AMMASSO_STATS_VARS(getQPSendBuffer_loop)
-
-  AMMASSO_STATS_VARS(sendDataOnQP)
-  AMMASSO_STATS_VARS(sendDataOnQP_pre_send)
-  AMMASSO_STATS_VARS(sendDataOnQP_send)
-  AMMASSO_STATS_VARS(sendDataOnQP_post_send)
-
-  AMMASSO_STATS_VARS(sendDataOnQP_1024)
-  AMMASSO_STATS_VARS(sendDataOnQP_2048)
-  AMMASSO_STATS_VARS(sendDataOnQP_4096)
-  AMMASSO_STATS_VARS(sendDataOnQP_16384)
-  AMMASSO_STATS_VARS(sendDataOnQP_over)
-
-  AMMASSO_STATS_VARS(AsynchronousEventHandler)
-  AMMASSO_STATS_VARS(CompletionEventHandler)
-  AMMASSO_STATS_VARS(ProcessMessage)
-  AMMASSO_STATS_VARS(processAmmassoControlMessage)
-
-  AMMASSO_STATS_VARS(sendAck)
-
-  AMMASSO_STATS_VARS(CommunicationServer)
-
-} AmmassoStats;
-
-AmmassoStats __stats;
-
-#define AMMASSO_STATS_INIT_AUX(event)  { __stats.event ## _total = 0.0; __stats.event ## _count = 0; }
-#define AMMASSO_STATS_INIT   {                                                  \
-                               AMMASSO_STATS_INIT_AUX(MachineInit)                  \
-                               AMMASSO_STATS_INIT_AUX(AmmassoDoIdle)                \
-                               AMMASSO_STATS_INIT_AUX(DeliverViaNetwork)            \
-                               AMMASSO_STATS_INIT_AUX(DeliverViaNetwork_pre_lock)   \
-                               AMMASSO_STATS_INIT_AUX(DeliverViaNetwork_lock)       \
-                               AMMASSO_STATS_INIT_AUX(DeliverViaNetwork_post_lock)  \
-                               AMMASSO_STATS_INIT_AUX(DeliverViaNetwork_send)       \
-                               AMMASSO_STATS_INIT_AUX(getQPSendBuffer)              \
-                               AMMASSO_STATS_INIT_AUX(getQPSendBuffer_lock)         \
-                               AMMASSO_STATS_INIT_AUX(getQPSendBuffer_CEH)          \
-                               AMMASSO_STATS_INIT_AUX(getQPSendBuffer_loop)         \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP)                 \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_pre_send)        \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_send)            \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_post_send)       \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_1024)            \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_2048)            \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_4096)            \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_16384)           \
-                               AMMASSO_STATS_INIT_AUX(sendDataOnQP_over)            \
-                               AMMASSO_STATS_INIT_AUX(AsynchronousEventHandler)     \
-                               AMMASSO_STATS_INIT_AUX(CompletionEventHandler)       \
-                               AMMASSO_STATS_INIT_AUX(ProcessMessage)               \
-                               AMMASSO_STATS_INIT_AUX(processAmmassoControlMessage) \
-                               AMMASSO_STATS_INIT_AUX(sendAck)                      \
-                               AMMASSO_STATS_INIT_AUX(CommunicationServer)          \
-                            }
-
-#define TO_NS  ((double)1000000000.0)
-
-#define AMMASSO_STATS_START(event) { __stats.event ## _start = CmiWallTimer();     \
-                                   }
-
-#define AMMASSO_STATS_END(event)   { __stats.event ## _end = CmiWallTimer();         \
-                                     __stats.event ## _count++;                      \
-                                     __stats.event ## _total += (__stats.event ## _end - __stats.event ## _start);   \
-                                   }
-
-#define AMMASSO_STATS_DISPLAY_VERBOSE(event) { char buf[128];                        \
-                                               CmiPrintf("[%d] Ammasso Stats: event -> " #event "_count = %d\n", CmiMyPe(), __stats.event ## _count);   \
-                                               CmiPrintf("[%d]                event -> " #event "_total = %.3fns\n", CmiMyPe(), __stats.event ## _total * TO_NS);   \
-                                               CmiPrintf("[%d]                                                    " #event " average: %.3fns\n", CmiMyPe(), (((double)__stats.event ## _total)/(__stats.event ## _count)) * TO_NS); \
-                                             }
-
-#define AMMASSO_STATS_DISPLAY(event) {                                               \
-                                       CmiPrintf("[%d] " #event " average: %.3fns\n", CmiMyPe(), (((double)__stats.event ## _total)/(__stats.event ## _count)) * TO_NS); \
-                                    }
-
-#else
-
-#define AMMASSO_STATS_INIT_AUX(event)
-#define AMMASSO_STATS_INIT
-#define AMMASSO_STATS_START(event)
-#define AMMASSO_STATS_END(event)
-#define AMMASSO_STATS_DISPLAY(event)
-
-#endif
-
diff --git a/src/arch/net/machine-dgram.c b/src/arch/net/machine-dgram.c
deleted file mode 100644 (file)
index 9b1df0c..0000000
+++ /dev/null
@@ -1,838 +0,0 @@
-/** @file
- * Datagram implementation of Converse NET version
- * @ingroup NET
- * converse basic message header:<br>
- * d0 d1 d2 d3:  DgramHeader<br>
- * d4 d5:        msg length (32-bit number)<br>
- * hdl:          handler<br>
- * xhdl:         extended handler<br>
-
-  moved from machine.c by 
-  Orion Sky Lawlor, olawlor@acm.org, 7/25/2001
-*/
-
-/**
- * @addtogroup NET
- * @{
- */
-
-#if CMK_USE_AMMASSO
-#include "machine-ammasso.h"
-#endif
-
-#if CMK_USE_IBVERBS | CMK_USE_IBUD
-#include <infiniband/verbs.h>
-#endif
-
-#define DGRAM_HEADER_SIZE 8
-
-#define CmiMsgHeaderSetLength(msg, len) (((int*)(msg))[2] = (len))
-#define CmiMsgHeaderGetLength(msg)      (((int*)(msg))[2])
-#define CmiMsgNext(msg) (*((void**)(msg)))
-
-#define DGRAM_ROOTPE_MASK   (0xFFFFu)
-#define DGRAM_SRCPE_MASK    (0xFFFF)
-#define DGRAM_MAGIC_MASK    (0xFF)
-#define DGRAM_SEQNO_MASK    (0xFFFFu)
-
-#if CMK_NODE_QUEUE_AVAILABLE
-#define DGRAM_NODEBROADCAST (0xFA)
-#define DGRAM_NODEMESSAGE   (0xFB)
-#endif
-#define DGRAM_DSTRANK_MAX   (0xFC)
-#define DGRAM_SIMPLEKILL    (0xFD)
-#define DGRAM_BROADCAST     (0xFE)
-#define DGRAM_ACKNOWLEDGE   (0xFF)
-
-/* DgramHeader overlays the first 4 fields of the converse CMK_MSG_HEADER_BASIC,
-   defined in conv-common.h.  As such, its size and alignment are critical. */
-typedef struct {
-        unsigned int seqno:16;  /* seq number in send window */
-        unsigned int srcpe:16;  /* CmiMyPe of the sender */
-        unsigned int dstrank:8; /* rank of destination processor */
-        unsigned int magic:8;   /* Low 8 bits of charmrun PID */
-        unsigned int rootpe:16; /* broadcast root processor */
-} DgramHeader;
-
-
-/* the window size needs to be Cmi_window_size + sizeof(unsigned int) bytes) */
-typedef struct { DgramHeader head; char window[1024]; } DgramAck;
-
-extern unsigned char computeCheckSum(unsigned char *data, int len);
-
-#define DgramHeaderMake(ptr, dstrank_, srcpe_, magic_, seqno_, root_) { \
-   DgramHeader *header = (DgramHeader *)(ptr); \
-   header->seqno = seqno_; \
-   header->srcpe = srcpe_; \
-   header->dstrank = dstrank_; \
-   header->magic = magic_ & DGRAM_MAGIC_MASK; \
-   header->rootpe = root_; \
-}
-
-#define DgramHeaderBreak(ptr, dstrank_, srcpe_, magic_, seqno_, root_) { \
-   DgramHeader *header = (DgramHeader *)(ptr); \
-   seqno_ = header->seqno; \
-   srcpe_ = header->srcpe; \
-   dstrank_ = header->dstrank; \
-   magic_ = header->magic; \
-   root_ = header->rootpe; \
-}
-
-#ifdef CMK_RANDOMLY_CORRUPT_MESSAGES
-static void randomCorrupt(char *data, int len)
-{
-  if (0==(rand()%CMK_RANDOMLY_CORRUPT_MESSAGES))
-  { /* insert one random bit flip into this message: */
-    int badByte=rand()%len;
-    int badBit=rand()%8;
-    data[badByte]^=(1<<badBit);
-  } 
-}
-#endif
-
-#define PE_BROADCAST_OTHERS (-101)
-#define PE_BROADCAST_ALL    (-102)
-
-#if CMK_NODE_QUEUE_AVAILABLE
-#define NODE_BROADCAST_OTHERS (-201)
-#define NODE_BROADCAST_ALL    (-202)
-#endif
-
-/********* Startup and Command-line args ********/
-static int    Cmi_max_dgram_size;
-static int    Cmi_os_buffer_size;
-static int    Cmi_window_size;
-static int    Cmi_half_window;
-static double Cmi_delay_retransmit;
-static double Cmi_ack_delay;
-static int    Cmi_dgram_max_data;
-static int    Cmi_comm_periodic_delay;
-static int    Cmi_comm_clock_delay;
-static int writeableAcks,writeableDgrams;/*Write-queue counts (to know when to sleep)*/
-
-static void setspeed_atm(void)
-{
-  Cmi_max_dgram_size   = 2048;
-  Cmi_os_buffer_size   = 50000;
-  Cmi_window_size      = 16;       /*20;*/
-  Cmi_delay_retransmit = 0.0150;
-  Cmi_ack_delay        = 0.0035;
-}
-
-static void setspeed_eth(void)
-{
-  Cmi_max_dgram_size   = 1400;
-  Cmi_window_size      = 32;       /*40*/
-  Cmi_os_buffer_size   = Cmi_window_size*Cmi_max_dgram_size;
-  Cmi_delay_retransmit = 0.0400;
-  Cmi_ack_delay        = 0.0050;
-}
-
-static void setspeed_gigabit(void)
-{
-  /* for gigabit net */
-  Cmi_max_dgram_size   = 9000;
-  Cmi_window_size      = 8;
-  Cmi_os_buffer_size   = 200000;
-  Cmi_delay_retransmit = 0.020;
-  Cmi_ack_delay        = 0.018;
-}
-
-static void extract_args(char **argv)
-{
-  int ms;
-  setspeed_gigabit();
-  if (CmiGetArgFlagDesc(argv,"+atm","Tune for a low-latency ATM network"))
-    setspeed_atm();
-  if (CmiGetArgFlagDesc(argv,"+eth","Tune for an ethernet network"))
-    setspeed_eth();
-  if (CmiGetArgFlagDesc(argv,"+giga","Tune for a gigabit network"))
-    setspeed_gigabit();
-  CmiGetArgIntDesc(argv,"+max_dgram_size",&Cmi_max_dgram_size,"Size of each UDP packet");
-  CmiGetArgIntDesc(argv,"+window_size",&Cmi_window_size,"Number of unacknowledged packets");
-  /* must divide for window protocol to work */
-  if ( (DGRAM_SEQNO_MASK+1)%Cmi_window_size != 0)
-    CmiAbort("Invalid window size!");
-  CmiGetArgIntDesc(argv,"+os_buffer_size",&Cmi_os_buffer_size, "UDP socket's SO_RCVBUF/SO_SNDBUF");
-  if (CmiGetArgIntDesc(argv,"+delay_retransmit",&ms, "Milliseconds to wait before retransmit"))
-         Cmi_delay_retransmit=0.001*ms;
-  if (CmiGetArgIntDesc(argv,"+ack_delay",&ms, "Milliseconds to wait before ack'ing"))
-         Cmi_ack_delay=0.001*ms;
-  extract_common_args(argv);
-  Cmi_dgram_max_data = Cmi_max_dgram_size - DGRAM_HEADER_SIZE;
-  Cmi_half_window = Cmi_window_size >> 1;
-  if ((Cmi_window_size * Cmi_max_dgram_size) > Cmi_os_buffer_size)
-    KillEveryone("Window size too big for OS buffer.");
-  Cmi_comm_periodic_delay=(int)(1000*Cmi_delay_retransmit);
-  if (Cmi_comm_periodic_delay>60) Cmi_comm_periodic_delay=60;
-  Cmi_comm_clock_delay=(int)(1000*Cmi_ack_delay);
-  if (sizeof(DgramHeader)!=DGRAM_HEADER_SIZE) {
-    CmiAbort("DatagramHeader in machine-dgram.c is the wrong size!\n");
-  }
-}
-
-/* Compare seqnos using modular arithmetic-- currently unused
-static int seqno_in_window(unsigned int seqno,unsigned int winStart)
-{
-  return ((DGRAM_SEQNO_MASK&(seqno-winStart)) < Cmi_window_size);
-}
-static int seqno_lt(unsigned int seqA,unsigned int seqB)
-{
-  unsigned int del=seqB-seqA;
-  return (del>0u) && (del<(DGRAM_SEQNO_MASK/2));
-}
-static int seqno_le(unsigned int seqA,unsigned int seqB)
-{
-  unsigned int del=seqB-seqA;
-  return (del>=0u) && (del<(DGRAM_SEQNO_MASK/2));
-}
-*/
-
-
-/*****************************************************************************
- *
- * Communication Structures
- *
- *****************************************************************************/
-
-typedef struct OutgoingMsgStruct
-{
-  struct OutgoingMsgStruct *next;
-  int   src, dst;
-  int   size;
-  char *data;
-  int   refcount;
-  int   freemode;
-}
-*OutgoingMsg;
-
-typedef struct ExplicitDgramStruct
-{
-  struct ExplicitDgramStruct *next;
-  int  srcpe, rank, seqno, broot;
-  unsigned int len, dummy; /* dummy to fix bug in rs6k alignment */
-  double data[1];
-}
-*ExplicitDgram;
-
-typedef struct ImplicitDgramStruct
-{
-  struct ImplicitDgramStruct *next;
-  struct OtherNodeStruct *dest;
-  int srcpe, rank, seqno, broot;
-  char  *dataptr;
-  int    datalen;
-  OutgoingMsg ogm;
-}
-*ImplicitDgram;
-
-struct PendingMsgStruct;
-
-
-#if CMK_USE_IBUD
-struct infiOtherNodeData;
-struct infiOtherNodeData *initinfiData(int node,int lid,int qpn,int psn);
-#endif
-#if CMK_USE_IBVERBS
-struct infiOtherNodeData;
-struct infiOtherNodeData *initInfiOtherNodeData(int node,int addr[3]);
-void   infiPostInitialRecvs(void);
-#endif
-
-#if CMK_USE_SYSVSHM
-inline void CommunicationServerSysvshm(void);
-#endif
-#if CMK_USE_PXSHM
-inline void CommunicationServerPxshm(void);
-#endif
-
-#if CMK_USE_AMMASSO
-
-/*
- * State Machine for Queue Pair Connection State (machine layer state for QP)
- *  PRE_CONNECT  --->  CONNECTED  ---> CONNECTION_LOST
- *                        |    /|\              |
- *                       \|/    \---------------/
- *           CONNECTION_CLOSED
- */
-
-typedef enum __qp_connection_state {
-  QP_CONN_STATE_PRE_CONNECT = 1,     /* Connection is being attempted and no successful connection has been made yet           */
-  QP_CONN_STATE_CONNECTED,           /* Connection has be established                                                          */
-  QP_CONN_STATE_CONNECTION_LOST,     /* Connection is being attempted and there has been an established connection in the past */
-  QP_CONN_STATE_CONNECTION_CLOSED    /* Connection closed                                                                      */
-} qp_connection_state_t;
-#endif
-  
-typedef struct FutureMessageStruct {
-  char *msg;
-  int len;
-} *FutureMessage;
-
-typedef struct OtherNodeStruct
-{
-  int nodestart, nodesize;
-  skt_ip_t IP;
-  unsigned int mach_id;
-  unsigned int dataport;
-  struct sockaddr_in addr;
-#if CMK_USE_TCP 
-  SOCKET       sock;           /* for TCP */
-#endif
-#if CMK_USE_MX
-  CmiUInt8 nic_id;
-  mx_endpoint_addr_t       endpoint_addr;
-  CdsFifo                  futureMsgs;   /* out-of-order */
-#endif
-
-  unsigned int             send_last;    /* seqno of last dgram sent */
-  ImplicitDgram           *send_window;  /* datagrams sent, not acked */
-  ImplicitDgram            send_queue_h; /* head of send queue */
-  ImplicitDgram            send_queue_t; /* tail of send queue */
-  unsigned int             send_next;    /* next seqno to go into queue */
-  unsigned int             send_good;    /* last acknowledged seqno */
-  double                   send_primer;  /* time to send retransmit */
-  unsigned int             send_ack_seqno; /* next ack seqno to send */
-  int                      retransmit_leash; /*Maximum number of packets to retransmit*/
-#if CMK_USE_GM
-  struct PendingMsgStruct *sendhead, *sendtail;  /* gm send queue */
-  int                     disable;
-  int                     gm_pending;
-#endif
-
-#if CMK_USE_IBVERBS | CMK_USE_IBUD
-       struct infiOtherNodeData *infiData;
-#endif
-
-#if CMK_USE_AMMASSO
-  /* DMK : TODO : If any of these can be shared, then they can be moved to mycb_t in "machine-ammasso.c"  */
-
-  cc_uint32_t            recv_cq_depth;
-  cc_cq_handle_t         recv_cq;
-  cc_uint32_t            send_cq_depth;
-  cc_cq_handle_t         send_cq;
-  cc_qp_id_t             qp_id;      /* Queue Pair ID      */
-  cc_qp_handle_t         qp;         /* Queue Pair Handle  */
-
-  int                    myNode;
-
-  // list of buffers currently allocated to this paricular node
-  LIST_DEFINE(AmmassoBuffer,recv_buf);
-  // This is used when deallocating buffers: it is set to the buffer previous to
-  // the last in the list of recv_buf, so that the list can be broken, and the
-  // buffers released without the usage of either a double linked list or a
-  // linear search
-  AmmassoBuffer          *secondLastRecvBuf;
-
-  // when allocating new buffers, they remain here until the other node conferm
-  // their allocation
-  LIST_DEFINE(AmmassoBuffer,pending);
-
-  // number of the next ACK to be sent to this node, it is piggybacked on every
-  // message
-  ammasso_ack_t          *remoteAck;
-
-  // how many messages have been received since the last ACK was sent
-  int                    messagesNotYetAcknowledged;
-
-  // the following is used to send the ACK without a message
-  cc_sq_wr_t             *ack_sq_wr;
-
-  // the following is a pointer to where the ACK will arrive (if directly sent)
-  ammasso_ack_t          *directAck;
-
-  LIST_DEFINE(AmmassoToken,sendTokens); // linked list of available tokens
-  LIST_DEFINE(AmmassoToken,usedTokens); // they are waiting for an ACK
-
-  // number of ACK received from this node, it is compared against the incoming
-  // ACK to release send buffers
-  ammasso_ack_t          localAck;
-
-  // NOTE: this locks are probably not needed, try to see if they can be deleted
-  CmiNodeLock            sendBufLock;
-  CmiNodeLock            send_next_lock;
-  CmiNodeLock            recv_expect_lock;
-
-  cc_ep_handle_t         ep;
-  cc_ep_handle_t         cr;
-
-  cc_qp_query_attrs_t      qp_attrs;
-  cc_stag_index_t          qp_attrs_stag_index;
-
-  int                    posted;
-
-  cc_inet_addr_t         address;  /* local if passive side of connection, remote if active side of connection */
-  cc_inet_port_t         port;     /* local if passive side of connection, remote if active side of connection */
-  qp_connection_state_t  connectionState;  /* State of the connection (connected, lost, etc) */
-
-  cc_stag_t              remote_recv_stag;
-  cc_uint64_t            remote_starting_to;
-  int                    recv_UseIndex;
-
-  /* Used by DYNAMIC ALLOCATOR */
-  int  max_used_tokens;
-
-#endif
-
-  int                      asm_rank;
-  int                      asm_total;
-  int                      asm_fill;
-  char                    *asm_msg;
-  
-  int                      recv_ack_cnt; /* number of unacked dgrams */
-  double                   recv_ack_time;/* time when ack should be sent */
-  unsigned int             recv_expect;  /* next dgram to expect */
-  ExplicitDgram           *recv_window;  /* Packets received, not integrated */
-  int                      recv_winsz;   /* Number of packets in recv window */
-  unsigned int             recv_next;    /* Seqno of first missing packet */
-  unsigned int             recv_ack_seqno; /* last ack seqno received */
-
-  unsigned int             stat_total_intr; /* Total Number of Interrupts */
-  unsigned int             stat_proc_intr;  /* Processed Interrupts */
-  unsigned int             stat_send_pkt;   /* number of packets sent */
-  unsigned int             stat_resend_pkt; /* number of packets resent */
-  unsigned int             stat_send_ack;   /* number of acks sent */
-  unsigned int             stat_recv_pkt;   /* number of packets received */
-  unsigned int             stat_recv_ack;   /* number of acks received */
-  unsigned int             stat_ack_pkts;   /* packets acked */
-  unsigned int             stat_consec_resend; /*Packets retransmitted since last ack*/ 
-
-  unsigned int sent_msgs;
-  unsigned int recd_msgs;
-  unsigned int sent_bytes;
-  unsigned int recd_bytes;
-}
-*OtherNode;
-
-static void OtherNode_init(OtherNode node)
-{
-    int i;
-    node->send_primer = 1.0e30; /*Don't retransmit until needed*/
-    node->retransmit_leash = 1; /*Start with short leash*/
-    node->send_last=0;
-    node->send_window =
-      (ImplicitDgram*)malloc(Cmi_window_size*sizeof(ImplicitDgram));
-    for (i=0;i<Cmi_window_size;i++) node->send_window[i]=NULL;
-    node->send_queue_h=node->send_queue_t=NULL;
-    node->send_next=0;
-    node->send_good=(unsigned int)(-1);
-    node->send_ack_seqno=0;
-#if CMK_USE_GM
-    node->sendhead = node->sendtail = NULL;
-    node->disable = 0;
-    node->gm_pending = 0;
-#endif
-#if CMK_USE_MX
-    node->futureMsgs = CdsFifo_Create();
-#endif
-
-    /*
-    TODO: The initial values of the Ammasso related members will be set by the machine layer
-          as the QPs are being created (along with any initial values).  After all the details
-          of the layer are figured out, put some defaults here just so they are initialized to
-          known values.  (Though, it should not be a problem that they are not initialized here yet.)
-    */
-
-    node->asm_rank=0;
-    node->asm_total=0;
-    node->asm_fill=0;
-    node->asm_msg=0;
-    
-    node->recv_ack_cnt=0;
-    node->recv_ack_time=1.0e30;
-    node->recv_ack_seqno=0;
-    node->recv_expect=0;
-    node->recv_window =
-      (ExplicitDgram*)malloc(Cmi_window_size*sizeof(ExplicitDgram));
-    for (i=0;i<Cmi_window_size;i++) node->recv_window[i]=NULL;    
-    node->recv_winsz=0;
-    node->recv_next=0;
-
-    node->stat_total_intr=0;
-    node->stat_proc_intr=0;
-    node->stat_send_pkt=0;
-    node->stat_resend_pkt=0;
-    node->stat_send_ack=0; 
-    node->stat_recv_pkt=0;      
-    node->stat_recv_ack=0;        
-    node->stat_ack_pkts=0;
-
-    node->sent_msgs = 0;
-    node->recd_msgs = 0;
-    node->sent_bytes = 0;
-    node->recd_bytes = 0;
-}
-
-static OtherNode *nodes_by_pe;  /* OtherNodes indexed by processor number */
-static OtherNode  nodes;        /* Indexed only by ``node number'' */
-
-#ifdef CMK_USE_SPECIAL_MESSAGE_QUEUE_CHECK
-/** Return 1 if our outgoing message queue 
-   for this node is longer than this many bytes. */
-int CmiLongSendQueue(int forNode,int longerThan) {
-       int ret=0;
-       ImplicitDgram dg;
-       CmiCommLock();
-       dg=nodes[forNode].send_queue_h;
-       while (longerThan>0 && dg) {
-               longerThan-=dg->datalen;
-               dg=dg->next;
-       }
-       CmiCommUnlock();
-       return ret;
-}
-#endif
-
-extern void CmiGmConvertMachineID(unsigned int *mach_id);
-extern void CmiAmmassoNodeAddressesStoreHandler(int pe, struct sockaddr_in *addr, int port);
-
-/* initnode node table reply format:
- +------------------------------------------------------- 
- | 4 bytes  |   Number of nodes n                       ^
- |          |   (big-endian binary integer)       4+12*n bytes
- +-------------------------------------------------     |
- ^  |        (one entry for each node)            ^     |
- |  | 4 bytes  |   Number of PEs for this node    |     |
- n  | 4 bytes  |   IP address of this node   12*n bytes |
- |  | 4 bytes  |   Data (UDP) port of this node   |     |
- v  |          |   (big-endian binary integers)   v     v
- ---+----------------------------------------------------
-*/
-static void node_addresses_store(ChMessage *msg)
-{
-  ChMessageInt_t *n32=(ChMessageInt_t *)msg->data;
-  ChNodeinfo *d=(ChNodeinfo *)(n32+1);
-  int nodestart;
-  int i,j,n;
-#if CMK_USE_IBVERBS
-  ChInfiAddr *remoteInfiAddr;
-#endif
-  MACHSTATE(1,"node_addresses_store {");       
-  _Cmi_numnodes=ChMessageInt(n32[0]);
-
-#if CMK_USE_IBVERBS
-  remoteInfiAddr = (ChInfiAddr *) (&msg->data[sizeof(ChMessageInt_t)+sizeof(ChNodeinfo)*_Cmi_numnodes]);
-  if (Cmi_charmrun_fd == -1) {
-    d = &((ChSingleNodeinfo*)n32)->info;
-  }
-  else if ((sizeof(ChMessageInt_t)+sizeof(ChNodeinfo)*_Cmi_numnodes +sizeof(ChInfiAddr)*_Cmi_numnodes )
-         !=(unsigned int)msg->len)
-    {printf("Node table has inconsistent length!");machine_exit(1);}
-
-#else
-
-  if ((sizeof(ChMessageInt_t)+sizeof(ChNodeinfo)*_Cmi_numnodes)
-         !=(unsigned int)msg->len)
-    {printf("Node table has inconsistent length!");machine_exit(1);}
-#endif /*CMK_USE_IBVERBS*/
-  nodes = (OtherNode)malloc(_Cmi_numnodes * sizeof(struct OtherNodeStruct));
-  nodestart=0;
-  for (i=0; i<_Cmi_numnodes; i++) {
-    nodes[i].nodestart = nodestart;
-    nodes[i].nodesize  = ChMessageInt(d[i].nPE);
-    MACHSTATE2(3,"node %d nodesize %d",i,nodes[i].nodesize);
-    nodes[i].mach_id = ChMessageInt(d[i].mach_id);
-#if CMK_USE_MX
-    nodes[i].nic_id = ChMessageLong(d[i].nic_id);
-#endif
-
-#if CMK_USE_IBUD
-    nodes[i].infiData=initinfiData(i,ChMessageInt(d[i].qp.lid),ChMessageInt(d[i].qp.qpn),ChMessageInt(d[i].qp.psn));
-#endif
-
-#if CMK_USE_GM
-    CmiGmConvertMachineID(& nodes[i].mach_id);
-#endif
-    nodes[i].IP=d[i].IP;
-    if (i==_Cmi_mynode) {
-      Cmi_nodestart=nodes[i].nodestart;
-      _Cmi_mynodesize=nodes[i].nodesize;
-      Cmi_self_IP=nodes[i].IP;
-    }
-
-#if CMK_USE_IBVERBS 
-    if(i != _Cmi_mynode){
-       int addr[3];
-       addr[0] =ChMessageInt(remoteInfiAddr[i].lid);
-       addr[1] =ChMessageInt(remoteInfiAddr[i].qpn);
-       addr[2] =ChMessageInt(remoteInfiAddr[i].psn);
-       nodes[i].infiData = initInfiOtherNodeData(i,addr);
-    }
-#else
-    nodes[i].dataport = ChMessageInt(d[i].dataport);
-    nodes[i].addr = skt_build_addr(nodes[i].IP,nodes[i].dataport);
-#endif
-
-#if CMK_USE_TCP
-    nodes[i].sock = INVALID_SOCKET;
-#endif
-    nodestart+=nodes[i].nodesize;
-
-#if CMK_USE_AMMASSO
-    CmiAmmassoNodeAddressesStoreHandler(nodes[i].nodestart, &(nodes[i].addr), nodes[i].dataport);
-#endif
-
-  }
-  _Cmi_numpes=nodestart;
-  n = _Cmi_numpes;
-#ifdef CMK_CPV_IS_SMP
-  n += _Cmi_numnodes;
-#endif
-  nodes_by_pe = (OtherNode*)malloc(n * sizeof(OtherNode));
-  _MEMCHECK(nodes_by_pe);
-  for (i=0; i<_Cmi_numnodes; i++) {
-    OtherNode node = nodes + i;
-    OtherNode_init(node);
-    for (j=0; j<node->nodesize; j++)
-      nodes_by_pe[j + node->nodestart] = node;
-  }
-#ifdef CMK_CPV_IS_SMP
-  /* index for communication threads */
-  for (i=_Cmi_numpes; i<_Cmi_numpes+_Cmi_numnodes; i++) {
-    OtherNode node = nodes + i-_Cmi_numpes;
-    nodes_by_pe[i] = node;
-  }
-#endif
-#if CMK_USE_IBVERBS
-  infiPostInitialRecvs();
-#endif
-  MACHSTATE(1,"} node_addresses_store");
-}
-
-/**
- * Printing Net Statistics -- milind
- */
-static char statstr[10000];
-
-void printNetStatistics(void)
-{
-  char tmpstr[1024];
-  OtherNode myNode;
-  int i;
-  unsigned int send_pkt=0, resend_pkt=0, recv_pkt=0, send_ack=0;
-  unsigned int recv_ack=0, ack_pkts=0;
-
-  myNode = nodes+CmiMyNode();
-  sprintf(tmpstr, "***********************************\n");
-  strcpy(statstr, tmpstr);
-  sprintf(tmpstr, "Net Statistics For Node %u\n", CmiMyNode());
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "Interrupts: %u \tProcessed: %u\n",
-                  myNode->stat_total_intr, myNode->stat_proc_intr);
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "Total Msgs Sent: %u \tTotal Bytes Sent: %u\n",
-                  myNode->sent_msgs, myNode->sent_bytes);
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "Total Msgs Recv: %u \tTotal Bytes Recv: %u\n",
-                  myNode->recd_msgs, myNode->recd_bytes);
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "***********************************\n");
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "[Num]\tSENDTO\tRESEND\tRECV\tACKSTO\tACKSFRM\tPKTACK\n");
-  strcat(statstr,tmpstr);
-  sprintf(tmpstr, "=====\t======\t======\t====\t======\t=======\t======\n");
-  strcat(statstr,tmpstr);
-  for(i=0;i<CmiNumNodes();i++) {
-    OtherNode node = nodes+i;
-    sprintf(tmpstr, "[%u]\t%u\t%u\t%u\t%u\t%u\t%u\n",
-                     i, node->stat_send_pkt, node->stat_resend_pkt,
-                    node->stat_recv_pkt, node->stat_send_ack,
-                    node->stat_recv_ack, node->stat_ack_pkts);
-    strcat(statstr, tmpstr);
-    send_pkt += node->stat_send_pkt;
-    recv_pkt += node->stat_recv_pkt;
-    resend_pkt += node->stat_resend_pkt;
-    send_ack += node->stat_send_ack;
-    recv_ack += node->stat_recv_ack;
-    ack_pkts += node->stat_ack_pkts;
-  }
-  sprintf(tmpstr, "[TOTAL]\t%u\t%u\t%u\t%u\t%u\t%u\n",
-                     send_pkt, resend_pkt,
-                    recv_pkt, send_ack,
-                    recv_ack, ack_pkts);
-  strcat(statstr, tmpstr);
-  sprintf(tmpstr, "***********************************\n");
-  strcat(statstr, tmpstr);
-  CmiPrintf(statstr);
-}
-
-
-/************** free list management *****************/
-
-static ExplicitDgram Cmi_freelist_explicit;
-static ImplicitDgram Cmi_freelist_implicit;
-/*static OutgoingMsg   Cmi_freelist_outgoing;*/
-
-#define FreeImplicitDgram(dg) {\
-  ImplicitDgram d=(dg);\
-  d->next = Cmi_freelist_implicit;\
-  Cmi_freelist_implicit = d;\
-}
-
-#define MallocImplicitDgram(dg) {\
-  ImplicitDgram d = Cmi_freelist_implicit;\
-  if (d==0) {d = ((ImplicitDgram)malloc(sizeof(struct ImplicitDgramStruct)));\
-             _MEMCHECK(d);\
-  } else Cmi_freelist_implicit = d->next;\
-  dg = d;\
-}
-
-#define FreeExplicitDgram(dg) {\
-  ExplicitDgram d=(dg);\
-  d->next = Cmi_freelist_explicit;\
-  Cmi_freelist_explicit = d;\
-}
-
-#define MallocExplicitDgram(dg) {\
-  ExplicitDgram d = Cmi_freelist_explicit;\
-  if (d==0) { d = ((ExplicitDgram)malloc \
-                  (sizeof(struct ExplicitDgramStruct) + Cmi_max_dgram_size));\
-              _MEMCHECK(d);\
-  } else Cmi_freelist_explicit = d->next;\
-  dg = d;\
-}
-
-/* Careful with these next two, need concurrency control */
-
-#define FreeOutgoingMsg(m) (free(m))
-#define MallocOutgoingMsg(m)\
-    {(m=(OutgoingMsg)malloc(sizeof(struct OutgoingMsgStruct))); _MEMCHECK(m);}
-
-/****************************************************************************
- *                                                                          
- * CheckSocketsReady
- *
- * Checks both sockets to see which are readable and which are writeable.
- * We check all these things at the same time since this can be done for
- * free with ``select.'' The result is stored in global variables, since
- * this is essentially global state information and several routines need it.
- *
- ***************************************************************************/
-
-static int ctrlskt_ready_read;
-static int dataskt_ready_read;
-static int dataskt_ready_write;
-
-/******************************************************************************
- *
- * Transmission Code
- *
- *****************************************************************************/
-
-void GarbageCollectMsg(OutgoingMsg ogm)
-{
-  MACHSTATE2(3,"GarbageCollectMsg called on ogm %p refcount %d",ogm,ogm->refcount);
-       if (ogm->refcount == 0) {
-    if (ogm->freemode == 'A') {
-      ogm->freemode = 'X';
-    } else {
-      if (ogm->freemode != 'G') CmiFree(ogm->data);
-      FreeOutgoingMsg(ogm);
-    }
-  }
-}
-
-void DiscardImplicitDgram(ImplicitDgram dg)
-{
-  OutgoingMsg ogm;
-  ogm = dg->ogm;
-  ogm->refcount--;
-  GarbageCollectMsg(ogm);
-  FreeImplicitDgram(dg);
-}
-
-/*
- Check the real-time clock and perform periodic tasks.
- Must be called with comm. lock held.
- */
-static void CommunicationsClock(void)
-{
-  static double Cmi_ack_last, Cmi_check_last;
-  MACHSTATE(1,"CommunicationsClock");
-  Cmi_clock = GetClock();
-  if (Cmi_clock > Cmi_ack_last + 0.5*Cmi_ack_delay) {
-    MACHSTATE(2,"CommunicationsClock timing out acks");    
-    Cmi_ack_last=Cmi_clock;
-    writeableAcks=1;
-    writeableDgrams=1;
-  }
-  
-  if (Cmi_clock > Cmi_check_last + Cmi_check_delay) {
-    MACHSTATE(4,"CommunicationsClock pinging charmrun");       
-    Cmi_check_last = Cmi_clock; 
-    ctrl_sendone_nolock("ping",NULL,0,NULL,0); /*Charmrun may have died*/
-  }
-}
-
-#if CMK_SHARED_VARS_UNAVAILABLE
-static void CommunicationsClockCaller(void *ignored)
-{
-  CmiCommLock();
-  CommunicationsClock();
-  CmiCommUnlock();
-  CcdCallFnAfter((CcdVoidFn)CommunicationsClockCaller,NULL,Cmi_comm_clock_delay);  
-}
-
-static void CommunicationPeriodic(void) 
-{ /*Poll on the communications server*/
-#if CMK_USE_SYSVSHM
-       CommunicationServerSysvshm();
-#endif
-#if CMK_USE_PXSHM
-       CommunicationServerPxshm();
-#endif
-  CommunicationServer(0, COMM_SERVER_FROM_SMP);
-}
-
-static void CommunicationPeriodicCaller(void *ignored)
-{
-  CommunicationPeriodic();
-  CcdCallFnAfter((CcdVoidFn)CommunicationPeriodicCaller,NULL,Cmi_comm_periodic_delay);
-}
-#endif
-
-/* common hardware dependent API */
-/*void EnqueueOutgoingDgram(OutgoingMsg ogm, char *ptr, int dlen, OtherNode node, int rank, int broot);*/
-void DeliverViaNetwork(OutgoingMsg ogm, OtherNode node, int rank, unsigned int broot, int copy);
-
-void SendSpanningChildren(OutgoingMsg ogm, int root, int size, char *msg, unsigned int startpe, int nodesend);
-void SendHypercube(OutgoingMsg ogm, int root, int size, char *msg, unsigned int curcycle, int nodesend);
-
-#if CMK_USE_GM
-
-#include "machine-gm.c"
-
-#elif CMK_USE_MX
-
-#include "machine-mx.c"
-
-#elif CMK_USE_AMMASSO
-
-#include "machine-ammasso.c"
-
-#elif CMK_USE_TCP
-
-#include "machine-tcp.c"
-
-#elif CMK_USE_IBVERBS
-
-#include "machine-ibverbs.c"
-
-#elif CMK_USE_IBUD
-#include "machine-ibud.c"
-
-#else
-
-#include "machine-eth.c"
-
-#endif
-
-#if CMK_USE_SYSVSHM
-#include "machine-sysvshm.c"
-#endif
-#if CMK_USE_PXSHM
-#include "machine-pxshm.c"
-#endif
-
-
-/*@}*/
diff --git a/src/arch/net/machine-eth.c b/src/arch/net/machine-eth.c
deleted file mode 100644 (file)
index cfe6dfc..0000000
+++ /dev/null
@@ -1,832 +0,0 @@
-/** @file
- * UDP implementation of Converse NET version
- * @ingroup NET
- * contains only UDP specific code for:
- * - CmiMachineInit()
- * - CmiCommunicationInit()
- * - CmiNotifyIdle()
- * - DeliverViaNetwork()
- * - CommunicationServer()
-
-  moved from machine.c by 
-  Gengbin Zheng, gzheng@uiuc.edu  4/22/2001
-*/
-
-/**
- * @addtogroup NET
- * @{
- */
-
-/******************************************************************************
- *
- * CmiNotifyIdle()-- wait until a packet comes in
- *
- *****************************************************************************/
-
-typedef struct {
-  int sleepMs; /*Milliseconds to sleep while idle*/
-  int nIdles; /*Number of times we've been idle in a row*/
-  CmiState cs; /*Machine state*/
-} CmiIdleState;
-
-static CmiIdleState *CmiNotifyGetState(void)
-{
-  CmiIdleState *s=(CmiIdleState *)malloc(sizeof(CmiIdleState));
-  s->sleepMs=0;
-  s->nIdles=0;
-  s->cs=CmiGetState();
-  return s;
-}
-
-static void CmiNotifyBeginIdle(CmiIdleState *s)
-{
-  s->sleepMs=0;
-  s->nIdles=0;
-
-  MACHSTATE(3,"begin idle")
-}
-
-static void CmiNotifyStillIdle(CmiIdleState *s)
-{
-#if CMK_SHARED_VARS_UNAVAILABLE
-  /*No comm. thread-- listen on sockets for incoming messages*/
-  MACHSTATE(1,"idle commserver {")
-  CommunicationServer(Cmi_idlepoll?0:10, COMM_SERVER_FROM_SMP);
-  MACHSTATE(1,"} idle commserver")
-#else
-#if CMK_SHARED_VARS_POSIX_THREADS_SMP
-  if(_Cmi_sleepOnIdle ){
-#endif
-    int nSpins=20; /*Number of times to spin before sleeping*/
-    s->nIdles++;
-    if (s->nIdles>nSpins) { /*Start giving some time back to the OS*/
-      s->sleepMs+=2;
-      if (s->sleepMs>10) s->sleepMs=10;
-    }
-    /*Comm. thread will listen on sockets-- just sleep*/
-    if (s->sleepMs>0) {
-      MACHSTATE1(3,"idle lock(%d) {",CmiMyPe())
-      CmiIdleLock_sleep(&s->cs->idle,s->sleepMs);
-      CsdResetPeriodic();              /* check ccd callbacks when I am awakened */
-      MACHSTATE1(3,"} idle lock(%d)",CmiMyPe())
-    }
-#if CMK_SHARED_VARS_POSIX_THREADS_SMP
-  }
-#endif
-#endif
-}
-
-void CmiNotifyIdle(void) {
-  CmiIdleState s;
-  s.sleepMs=5; 
-  CmiNotifyStillIdle(&s);
-}
-
-/****************************************************************************
- *                                                                          
- * CheckSocketsReady
- *
- * Checks both sockets to see which are readable and which are writeable.
- * We check all these things at the same time since this can be done for
- * free with ``select.'' The result is stored in global variables, since
- * this is essentially global state information and several routines need it.
- *
- ***************************************************************************/
-
-int CheckSocketsReady(int withDelayMs)
-{   
-  int nreadable,dataWrite=writeableDgrams || writeableAcks;
-  CMK_PIPE_DECL(withDelayMs);
-
-
-#if CMK_USE_KQUEUE && 0
-  // This implementation doesn't yet work, but potentially is much faster
-
-  /* Only setup the CMK_PIPE structures the first time they are used. 
-     This makes the kqueue implementation much faster.
-  */
-  static int first = 1;
-  if(first){
-    first = 0;
-    CmiStdoutAdd(CMK_PIPE_SUB);
-    if (Cmi_charmrun_fd!=-1) { CMK_PIPE_ADDREAD(Cmi_charmrun_fd); }
-    else return 0; /* If there's no charmrun, none of this matters. */
-    if (dataskt!=-1) {
-      CMK_PIPE_ADDREAD(dataskt); 
-      CMK_PIPE_ADDWRITE(dataskt);
-    }
-  }
-
-#else  
-  CmiStdoutAdd(CMK_PIPE_SUB);  
-  if (Cmi_charmrun_fd!=-1) { CMK_PIPE_ADDREAD(Cmi_charmrun_fd); }  
-  else return 0; /* If there's no charmrun, none of this matters. */  
-  if (dataskt!=-1) {  
-    { CMK_PIPE_ADDREAD(dataskt); }  
-    if (dataWrite)  
-      CMK_PIPE_ADDWRITE(dataskt);  
-  }  
-#endif 
-
-  nreadable=CMK_PIPE_CALL();
-  ctrlskt_ready_read = 0;
-  dataskt_ready_read = 0;
-  dataskt_ready_write = 0;
-
-  if (nreadable == 0) {
-    MACHSTATE(1,"} CheckSocketsReady (nothing readable)")
-    return nreadable;
-  }
-  if (nreadable==-1) {
-    CMK_PIPE_CHECKERR();
-    MACHSTATE(2,"} CheckSocketsReady (INTERRUPTED!)")
-    return CheckSocketsReady(0);
-  }
-  
-  CmiStdoutCheck(CMK_PIPE_SUB);
-  if (Cmi_charmrun_fd!=-1) 
-         ctrlskt_ready_read = CMK_PIPE_CHECKREAD(Cmi_charmrun_fd);
-  if (dataskt!=-1) {
-       dataskt_ready_read = CMK_PIPE_CHECKREAD(dataskt);
-       if (dataWrite)
-               dataskt_ready_write = CMK_PIPE_CHECKWRITE(dataskt);
-  }
-  return nreadable;
-}
-
-/***********************************************************************
- * TransmitAckDatagram
- *
- * This function sends the ack datagram, after setting the window
- * array to show which of the datagrams in the current window have been
- * received. The sending side will then use this information to resend
- * packets, mark packets as received, etc. This system also prevents
- * multiple retransmissions/acks when acks are lost.
- ***********************************************************************/
-void TransmitAckDatagram(OtherNode node)
-{
-  DgramAck ack; int i, seqno, slot; ExplicitDgram dg;
-  int retval;
-  
-  seqno = node->recv_next;
-  MACHSTATE2(3,"  TransmitAckDgram [seq %d to 'pe' %d]",seqno,node->nodestart)
-  DgramHeaderMake(&ack, DGRAM_ACKNOWLEDGE, Cmi_nodestart, Cmi_net_magic, seqno, 0);
-  LOG(Cmi_clock, Cmi_nodestart, 'A', node->nodestart, seqno);
-  for (i=0; i<Cmi_window_size; i++) {
-    slot = seqno % Cmi_window_size;
-    dg = node->recv_window[slot];
-    ack.window[i] = (dg && (dg->seqno == seqno));
-    seqno = ((seqno+1) & DGRAM_SEQNO_MASK);
-  }
-  memcpy(&ack.window[Cmi_window_size], &(node->send_ack_seqno), 
-          sizeof(unsigned int));
-  node->send_ack_seqno = ((node->send_ack_seqno + 1) & DGRAM_SEQNO_MASK);
-  retval = (-1);
-#ifdef CMK_USE_CHECKSUM
-  DgramHeader *head = (DgramHeader *)(&ack);
-  head->magic ^= computeCheckSum((unsigned char*)&ack, DGRAM_HEADER_SIZE + Cmi_window_size + sizeof(unsigned int));
-#endif
-  while(retval==(-1))
-    retval = sendto(dataskt, (char *)&ack,
-        DGRAM_HEADER_SIZE + Cmi_window_size + sizeof(unsigned int), 0,
-        (struct sockaddr *)&(node->addr),
-        sizeof(struct sockaddr_in));
-  node->stat_send_ack++;
-}
-
-
-/***********************************************************************
- * TransmitImplicitDgram
- * TransmitImplicitDgram1
- *
- * These functions do the actual work of sending a UDP datagram.
- ***********************************************************************/
-void TransmitImplicitDgram(ImplicitDgram dg)
-{
-  char *data; DgramHeader *head; int len; DgramHeader temp;
-  OtherNode dest;
-  int retval;
-  
-  MACHSTATE3(3,"  TransmitImplicitDgram (%d bytes) [seq %d to 'pe' %d]",
-            dg->datalen,dg->seqno,dg->dest->nodestart)
-  len = dg->datalen;
-  data = dg->dataptr;
-  head = (DgramHeader *)(data - DGRAM_HEADER_SIZE);
-  temp = *head;
-  dest = dg->dest;
-  DgramHeaderMake(head, dg->rank, dg->srcpe, Cmi_net_magic, dg->seqno, dg->broot);
-#ifdef CMK_USE_CHECKSUM
-  head->magic ^= computeCheckSum((unsigned char*)head, len + DGRAM_HEADER_SIZE);
-#endif
-  LOG(Cmi_clock, Cmi_nodestart, 'T', dest->nodestart, dg->seqno);
-  retval = (-1);
-  while(retval==(-1))
-    retval = sendto(dataskt, (char *)head, len + DGRAM_HEADER_SIZE, 0,
-             (struct sockaddr *)&(dest->addr), sizeof(struct sockaddr_in));
-  *head = temp;
-  dest->stat_send_pkt++;
-}
-
-void TransmitImplicitDgram1(ImplicitDgram dg)
-{
-  char *data; DgramHeader *head; int len; DgramHeader temp;
-  OtherNode dest;
-  int retval;
-
-  MACHSTATE3(4,"  RETransmitImplicitDgram (%d bytes) [seq %d to 'pe' %d]",
-            dg->datalen,dg->seqno,dg->dest->nodestart)
-  len = dg->datalen;
-  data = dg->dataptr;
-  head = (DgramHeader *)(data - DGRAM_HEADER_SIZE);
-  temp = *head;
-  dest = dg->dest;
-  DgramHeaderMake(head, dg->rank, dg->srcpe, Cmi_net_magic, dg->seqno, dg->broot);
-#ifdef CMK_USE_CHECKSUM
-  head->magic ^= computeCheckSum((unsigned char *)head, len + DGRAM_HEADER_SIZE);
-#endif
-  LOG(Cmi_clock, Cmi_nodestart, 'P', dest->nodestart, dg->seqno);
-  retval = (-1);
-  while (retval == (-1))
-    retval = sendto(dataskt, (char *)head, len + DGRAM_HEADER_SIZE, 0,
-             (struct sockaddr *)&(dest->addr), sizeof(struct sockaddr_in));
-  *head = temp;
-  dest->stat_resend_pkt++;
-}
-
-
-/***********************************************************************
- * TransmitAcknowledgement
- *
- * This function sends the ack datagrams, after checking to see if the 
- * Recv Window is atleast half-full. After that, if the Recv window size 
- * is 0, then the count of un-acked datagrams, and the time at which
- * the ack should be sent is reset.
- ***********************************************************************/
-int TransmitAcknowledgement(void)
-{
-  int skip; static int nextnode=0; OtherNode node;
-  for (skip=0; skip<_Cmi_numnodes; skip++) {
-    node = nodes+nextnode;
-    nextnode = (nextnode + 1) % _Cmi_numnodes;
-    if (node->recv_ack_cnt) {
-      if ((node->recv_ack_cnt > Cmi_half_window) ||
-         (Cmi_clock >= node->recv_ack_time)) {
-       TransmitAckDatagram(node);
-       if (node->recv_winsz) {
-         node->recv_ack_cnt  = 1;
-         node->recv_ack_time = Cmi_clock + Cmi_ack_delay;
-       } else {
-         node->recv_ack_cnt  = 0;
-         node->recv_ack_time = 0.0;
-       }
-       return 1;
-      }
-    }
-  }
-  return 0;
-}
-
-
-/***********************************************************************
- * TransmitDatagram()
- *
- * This function fills up the Send Window with the contents of the
- * Send Queue. It also sets the node->send_primer variable, which
- * indicates when a retransmission will be attempted.
- ***********************************************************************/
-int TransmitDatagram(void)
-{
-  ImplicitDgram dg; OtherNode node;
-  static int nextnode=0; int skip, count, slot;
-  unsigned int seqno;
-  
-  for (skip=0; skip<_Cmi_numnodes; skip++) {
-    node = nodes+nextnode;
-    nextnode = (nextnode + 1) % _Cmi_numnodes;
-    dg = node->send_queue_h;
-    if (dg) {
-      seqno = dg->seqno;
-      slot = seqno % Cmi_window_size;
-      if (node->send_window[slot] == 0) {
-       node->send_queue_h = dg->next;
-       node->send_window[slot] = dg;
-       TransmitImplicitDgram(dg);
-       if (seqno == ((node->send_last+1)&DGRAM_SEQNO_MASK))
-         node->send_last = seqno;
-       node->send_primer = Cmi_clock + Cmi_delay_retransmit;
-       return 1;
-      }
-    }
-    if (Cmi_clock > node->send_primer) {
-      slot = (node->send_last % Cmi_window_size);
-      for (count=0; count<Cmi_window_size; count++) {
-       dg = node->send_window[slot];
-       if (dg) break;
-       slot = ((slot+Cmi_window_size-1) % Cmi_window_size);
-      }
-      if (dg) {
-       TransmitImplicitDgram1(node->send_window[slot]);
-       node->send_primer = Cmi_clock + Cmi_delay_retransmit;
-       return 1;
-      }
-    }
-  }
-  return 0;
-}
-
-/***********************************************************************
- * EnqueOutgoingDgram()
- *
- * This function enqueues the datagrams onto the Send queue of the
- * sender, after setting appropriate data values into each of the
- * datagrams. 
- ***********************************************************************/
-void EnqueueOutgoingDgram
-        (OutgoingMsg ogm, char *ptr, int len, OtherNode node, int rank, int broot)
-{
-  int seqno, dst, src; ImplicitDgram dg;
-  src = ogm->src;
-  dst = ogm->dst;
-  seqno = node->send_next;
-  node->send_next = ((seqno+1)&DGRAM_SEQNO_MASK);
-  MallocImplicitDgram(dg);
-  dg->dest = node;
-  dg->srcpe = src;
-  dg->rank = rank;
-  dg->seqno = seqno;
-  dg->broot = broot;
-  dg->dataptr = ptr;
-  dg->datalen = len;
-  dg->ogm = ogm;
-  ogm->refcount++;
-  dg->next = 0;
-  if (node->send_queue_h == 0) {
-    node->send_queue_h = dg;
-    node->send_queue_t = dg;
-  } else {
-    node->send_queue_t->next = dg;
-    node->send_queue_t = dg;
-  }
-}
-
-
-/***********************************************************************
- * DeliverViaNetwork()
- *
- * This function is responsible for all non-local transmission. This
- * function takes the outgoing messages, splits it into datagrams and
- * enqueues them into the Send Queue.
- ***********************************************************************/
-void DeliverViaNetwork(OutgoingMsg ogm, OtherNode node, int rank, unsigned int broot, int copy)
-{
-  int size; char *data;
-  OtherNode myNode = nodes+CmiMyNode();
-
-  MACHSTATE2(3,"DeliverViaNetwork %d-byte message to pe %d",
-            ogm->size,node->nodestart+rank);
-  size = ogm->size - DGRAM_HEADER_SIZE;
-  data = ogm->data + DGRAM_HEADER_SIZE;
-  writeableDgrams++;
-  while (size > Cmi_dgram_max_data) {
-    EnqueueOutgoingDgram(ogm, data, Cmi_dgram_max_data, node, rank, broot);
-    data += Cmi_dgram_max_data;
-    size -= Cmi_dgram_max_data;
-  }
-  EnqueueOutgoingDgram(ogm, data, size, node, rank, broot);
-
-  myNode->sent_msgs++;
-  myNode->sent_bytes += ogm->size;
-  /*Try to immediately send the packets off*/
-  writeableDgrams=1;
-
-}
-
-/***********************************************************************
- * AssembleDatagram()
- *
- * This function does the actual assembly of datagrams into a
- * message. node->asm_msg holds the current message being
- * assembled. Once the message assemble is complete (known by checking
- * if the total number of datagrams is equal to the number of datagrams
- * constituting the assembled message), the message is pushed into the
- * Producer-Consumer queue
- ***********************************************************************/
-void AssembleDatagram(OtherNode node, ExplicitDgram dg)
-{
-  int i;
-  unsigned int size; char *msg;
-  OtherNode myNode = nodes+CmiMyNode();
-  
-  MACHSTATE3(2,"  AssembleDatagram [seq %d from 'pe' %d, packet len %d]",
-       dg->seqno,node->nodestart,dg->len)
-  LOG(Cmi_clock, Cmi_nodestart, 'X', dg->srcpe, dg->seqno);
-  msg = node->asm_msg;
-  if (msg == 0) {
-    size = CmiMsgHeaderGetLength(dg->data);
-    MACHSTATE3(4,"  Assemble new datagram seq %d from 'pe' %d, len %d",
-       dg->seqno,node->nodestart,size)
-    msg = (char *)CmiAlloc(size);
-    if (!msg)
-      fprintf(stderr, "%d: Out of mem\n", _Cmi_mynode);
-    if (size < dg->len) KillEveryoneCode(4559312);
-#if CMK_CHARMDEBUG
-    setMemoryTypeMessage(msg);
-#endif
-    memcpy(msg, (char*)(dg->data), dg->len);
-    node->asm_rank = dg->rank;
-    node->asm_total = size;
-    node->asm_fill = dg->len;
-    node->asm_msg = msg;
-  } else {
-    size = dg->len - DGRAM_HEADER_SIZE;
-    memcpy(msg + node->asm_fill, ((char*)(dg->data))+DGRAM_HEADER_SIZE, size);
-    node->asm_fill += size;
-  }
-  MACHSTATE3(2,"  AssembleDatagram: now have %d of %d bytes from %d",
-       node->asm_fill, node->asm_total, node->nodestart)
-  if (node->asm_fill > node->asm_total) {
-      fprintf(stderr, "\n\n\t\tLength mismatch!!\n\n");
-      fflush(stderr);
-      MACHSTATE4(5,"Length mismatch seq %d, from 'pe' %d, fill %d, total %d\n", dg->seqno,node->nodestart,node->asm_fill,node->asm_total)
-      KillEveryoneCode(4559313);
-  }
-  if (node->asm_fill == node->asm_total) {
-    /* spanning tree broadcast - send first to avoid invalid msg ptr */
-#if CMK_BROADCAST_SPANNING_TREE
-    if (node->asm_rank == DGRAM_BROADCAST
-#if CMK_NODE_QUEUE_AVAILABLE 
-          || node->asm_rank == DGRAM_NODEBROADCAST
-#endif
-      )
-        SendSpanningChildren(NULL, 0, node->asm_total, msg, dg->broot, dg->rank);
-#elif CMK_BROADCAST_HYPERCUBE
-    if (node->asm_rank == DGRAM_BROADCAST
-#if CMK_NODE_QUEUE_AVAILABLE
-          || node->asm_rank == DGRAM_NODEBROADCAST
-#endif
-      )
-        SendHypercube(NULL, 0, node->asm_total, msg, dg->broot, dg->rank);
-#endif
-    if (node->asm_rank == DGRAM_BROADCAST) {
-      int len = node->asm_total;
-      for (i=1; i<_Cmi_mynodesize; i++)
-         CmiPushPE(i, CopyMsg(msg, len));
-      CmiPushPE(0, msg);
-    } else {
-#if CMK_NODE_QUEUE_AVAILABLE
-         if (node->asm_rank==DGRAM_NODEMESSAGE ||
-            node->asm_rank==DGRAM_NODEBROADCAST) 
-        {
-          CmiPushNode(msg);
-         }
-        else
-#endif
-          CmiPushPE(node->asm_rank, msg);
-    }
-    node->asm_msg = 0;
-    myNode->recd_msgs++;
-    myNode->recd_bytes += node->asm_total;
-  }
-  FreeExplicitDgram(dg);
-}
-
-
-/***********************************************************************
- * AssembleReceivedDatagrams()
- *
- * This function assembles the datagrams received so far, into a
- * single message. This also results in part of the Receive Window being 
- * freed.
- ***********************************************************************/
-void AssembleReceivedDatagrams(OtherNode node)
-{
-  unsigned int next, slot; ExplicitDgram dg;
-  next = node->recv_next;
-  while (1) {
-    slot = (next % Cmi_window_size);
-    dg = node->recv_window[slot];
-    if (dg == 0) break;
-    AssembleDatagram(node, dg);
-    node->recv_window[slot] = 0;
-    node->recv_winsz--;
-    next = ((next + 1) & DGRAM_SEQNO_MASK);
-  }
-  node->recv_next = next;
-}
-
-
-
-
-/************************************************************************
- * IntegrateMessageDatagram()
- *
- * This function integrates the received datagrams. It first
- * increments the count of un-acked datagrams. (This is to aid the
- * heuristic that an ack should be sent when the Receive window is half
- * full). If the current datagram is the first missing packet, then this 
- * means that the datagram that was missing in the incomplete sequence
- * of datagrams so far, has arrived, and hence the datagrams can be
- * assembled. 
- ************************************************************************/
-
-void IntegrateMessageDatagram(ExplicitDgram dg)
-{
-  int seqno;
-  unsigned int slot; OtherNode node;
-
-  LOG(Cmi_clock, Cmi_nodestart, 'M', dg->srcpe, dg->seqno);
-  MACHSTATE2(2,"  IntegrateMessageDatagram [seq %d from pe %d]", dg->seqno,dg->srcpe)
-
-  node = nodes_by_pe[dg->srcpe];
-  node->stat_recv_pkt++;
-  seqno = dg->seqno;
-  writeableAcks=1;
-  node->recv_ack_cnt++;
-  if (node->recv_ack_time == 0.0)
-    node->recv_ack_time = Cmi_clock + Cmi_ack_delay;
-  if (((seqno - node->recv_next) & DGRAM_SEQNO_MASK) < Cmi_window_size) {
-    slot = (seqno % Cmi_window_size);
-    if (node->recv_window[slot] == 0) {
-      node->recv_window[slot] = dg;
-      node->recv_winsz++;
-      if (seqno == node->recv_next)
-       AssembleReceivedDatagrams(node);
-      if (seqno > node->recv_expect)
-       node->recv_ack_time = 0.0;
-      if (seqno >= node->recv_expect)
-       node->recv_expect = ((seqno+1)&DGRAM_SEQNO_MASK);
-      LOG(Cmi_clock, Cmi_nodestart, 'Y', node->recv_next, dg->seqno);
-      return;
-    }
-  }
-  LOG(Cmi_clock, Cmi_nodestart, 'y', node->recv_next, dg->seqno);
-  FreeExplicitDgram(dg);
-}
-
-
-
-/***********************************************************************
- * IntegrateAckDatagram()
- * 
- * This function is called on the message sending side, on receipt of
- * an ack for a message that it sent. Since messages and acks could be 
- * lost, our protocol works in such a way that acks for higher sequence
- * numbered packets act as implict acks for lower sequence numbered
- * packets, in case the acks for the lower sequence numbered packets
- * were lost.
-
- * Recall that the Send and Receive windows are circular queues, and the
- * sequence numbers of the packets (datagrams) are monotically
- * increasing. Hence it is important to know for which sequence number
- * the ack is for, and to correspodinly relate that to tha actual packet 
- * sitting in the Send window. Since every 20th packet occupies the same
- * slot in the windows, a number of sanity checks are required for our
- * protocol to work. 
- * 1. If the ack number (first missing packet sequence number) is less
- * than the last ack number received then this ack can be ignored. 
-
- * 2. The last ack number received must be set to the current ack
- * sequence number (This is done only if 1. is not true).
-
- * 3. Now the whole Send window is examined, in a kind of reverse
- * order. The check starts from a sequence number = 20 + the first
- * missing packet's sequence number. For each of these sequence numbers, 
- * the slot in the Send window is checked for existence of a datagram
- * that should have been sent. If there is no datagram, then the search
- * advances. If there is a datagram, then the sequence number of that is 
- * checked with the expected sequence number for the current iteration
- * (This is decremented in each iteration of the loop).
-
- * If the sequence numbers do not match, then checks are made (for
- * the unlikely scenarios where the current slot sequence number is 
- * equal to the first missing packet's sequence number, and where
- * somehow, packets which have greater sequence numbers than allowed for 
- * the current window)
-
- * If the sequence numbers DO match, then the flag 'rxing' is
- * checked. The semantics for this flag is that : If any packet with a
- * greater sequence number than the current packet (and hence in the
- * previous iteration of the for loop) has been acked, then the 'rxing'
- * flag is set to 1, to imply that all the packets of lower sequence
- * number, for which the ack->window[] element does not indicate that the 
- * packet has been received, must be retransmitted.
- * 
- ***********************************************************************/
-
-void IntegrateAckDatagram(ExplicitDgram dg)
-{
-  OtherNode node; DgramAck *ack; ImplicitDgram idg;
-  int i; unsigned int slot, rxing, dgseqno, seqno, ackseqno;
-  int diff;
-  unsigned int tmp;
-
-  node = nodes_by_pe[dg->srcpe];
-  ack = ((DgramAck*)(dg->data));
-  memcpy(&ackseqno, &(ack->window[Cmi_window_size]), sizeof(unsigned int));
-  dgseqno = dg->seqno;
-  seqno = (dgseqno + Cmi_window_size) & DGRAM_SEQNO_MASK;
-  slot = seqno % Cmi_window_size;
-  rxing = 0;
-  node->stat_recv_ack++;
-  LOG(Cmi_clock, Cmi_nodestart, 'R', node->nodestart, dg->seqno);
-
-  tmp = node->recv_ack_seqno;
-  /* check that the ack being received is actually appropriate */
-  if ( !((node->recv_ack_seqno >= 
-         ((DGRAM_SEQNO_MASK >> 1) + (DGRAM_SEQNO_MASK >> 2))) &&
-        (ackseqno < (DGRAM_SEQNO_MASK >> 1))) &&
-       (ackseqno <= node->recv_ack_seqno))
-    {
-      FreeExplicitDgram(dg);
-      return;
-    } 
-  /* higher ack so adjust */
-  node->recv_ack_seqno = ackseqno;
-  writeableDgrams=1; /* May have freed up some send slots */
-  
-  for (i=Cmi_window_size-1; i>=0; i--) {
-    slot--; if (slot== ((unsigned int)-1)) slot+=Cmi_window_size;
-    seqno = (seqno-1) & DGRAM_SEQNO_MASK;
-    idg = node->send_window[slot];
-    if (idg) {
-      if (idg->seqno == seqno) {
-       if (ack->window[i]) {
-         /* remove those that have been received and are within a window
-            of the first missing packet */
-         node->stat_ack_pkts++;
-         LOG(Cmi_clock, Cmi_nodestart, 'r', node->nodestart, seqno);
-         node->send_window[slot] = 0;
-         DiscardImplicitDgram(idg);
-         rxing = 1;
-       } else if (rxing) {
-         node->send_window[slot] = 0;
-         idg->next = node->send_queue_h;
-         if (node->send_queue_h == 0) {
-           node->send_queue_t = idg;
-         }
-         node->send_queue_h = idg;
-       }
-      } else {
-        diff = dgseqno >= idg->seqno ? 
-         ((dgseqno - idg->seqno) & DGRAM_SEQNO_MASK) :
-         ((dgseqno + (DGRAM_SEQNO_MASK - idg->seqno) + 1) & DGRAM_SEQNO_MASK);
-         
-       if ((diff <= 0) || (diff > Cmi_window_size))
-       {
-         continue;
-       }
-
-        /* if ack is really less than our packet seq (consider wrap around) */
-        if (dgseqno < idg->seqno && (idg->seqno - dgseqno <= Cmi_window_size))
-        {
-          continue;
-        }
-        if (dgseqno == idg->seqno)
-        {
-         continue;
-        }
-       node->stat_ack_pkts++;
-       LOG(Cmi_clock, Cmi_nodestart, 'o', node->nodestart, idg->seqno);
-       node->send_window[slot] = 0;
-       DiscardImplicitDgram(idg);
-      }
-    }
-  }
-  FreeExplicitDgram(dg);  
-}
-
-void ReceiveDatagram(void)
-{
-  ExplicitDgram dg; int ok, magic;
-  MACHLOCK_ASSERT(comm_flag,"ReceiveDatagram")
-  MallocExplicitDgram(dg);
-  ok = recv(dataskt,(char*)(dg->data),Cmi_max_dgram_size,0);
-  /*ok = recvfrom(dataskt,(char*)(dg->data),Cmi_max_dgram_size,0, 0, 0);*/
-  /* if (ok<0) { perror("recv"); KillEveryoneCode(37489437); } */
-  if (ok < 0) {
-    MACHSTATE1(4,"  recv dgram failed (errno=%d)",errno)
-    FreeExplicitDgram(dg);
-    if (errno == EINTR) return;  /* A SIGIO interrupted the receive */
-    if (errno == EAGAIN) return; /* Just try again later */
-#if !defined(_WIN32)
-    if (errno == EWOULDBLOCK) return; /* No more messages on that socket. */
-    if (errno == ECONNREFUSED) return;  /* A "Host unreachable" ICMP packet came in */
-#endif
-    CmiPrintf("ReceiveDatagram: recv: %s(%d)\n", strerror(errno), errno) ;
-    KillEveryoneCode(37489437);
-  }
-  dg->len = ok;
-#ifdef CMK_RANDOMLY_CORRUPT_MESSAGES
-  /* randomly corrupt data and ack datagrams */
-  randomCorrupt((char*)dg->data, dg->len);
-#endif
-
-  if (ok >= DGRAM_HEADER_SIZE) {
-    DgramHeaderBreak(dg->data, dg->rank, dg->srcpe, magic, dg->seqno, dg->broot);
-    MACHSTATE3(2,"  recv dgram [seq %d, for rank %d, from pe %d]",
-              dg->seqno,dg->rank,dg->srcpe)
-#ifdef CMK_USE_CHECKSUM
-    if (computeCheckSum((unsigned char*)dg->data, dg->len) == 0)
-#else
-    if (magic == (Cmi_net_magic&DGRAM_MAGIC_MASK))
-#endif
-    {
-      if (dg->rank == DGRAM_ACKNOWLEDGE)
-       IntegrateAckDatagram(dg);
-      else IntegrateMessageDatagram(dg);
-    } else FreeExplicitDgram(dg);
-  } else {
-    MACHSTATE1(4,"  recv dgram failed (len=%d)",ok)
-    FreeExplicitDgram(dg);
-  }
-}
-
-
-/***********************************************************************
- * CommunicationServer()
- * 
- * This function does the scheduling of the tasks related to the
- * message sends and receives. It is called from the CmiGeneralSend()
- * function, and periodically from the CommunicationInterrupt() (in case 
- * of the single processor version), and from the comm_thread (for the
- * SMP version). Based on which of the data/control read/write sockets
- * are ready, the corresponding tasks are called
- *
- ***********************************************************************/
-void CmiHandleImmediate(void);
-
-static void CommunicationServer(int sleepTime, int where)
-{
-  unsigned int nTimes=0; /* Loop counter */
-  LOG(GetClock(), Cmi_nodestart, 'I', 0, 0);
-  MACHSTATE2(1,"CommunicationsServer(%d,%d)",
-            sleepTime,writeableAcks||writeableDgrams)  
-#if !CMK_SHARED_VARS_UNAVAILABLE /*SMP mode: comm. lock is precious*/
-  if (sleepTime!=0) {/*Sleep *without* holding the comm. lock*/
-    MACHSTATE(1,"CommServer going to sleep (NO LOCK)");
-    if (CheckSocketsReady(sleepTime)<=0) {
-      MACHSTATE(1,"CommServer finished without anything happening.");
-    }
-  }
-  sleepTime=0;
-#endif
-  CmiCommLock();
-  inProgress[CmiMyRank()] += 1;
-  /* in netpoll mode, only perform service to stdout */
-  if (Cmi_netpoll && where == COMM_SERVER_FROM_INTERRUPT) {
-    if (CmiStdoutNeedsService()) {CmiStdoutService();}
-    CmiCommUnlock();
-    inProgress[CmiMyRank()] -= 1;
-    return;
-  }
-  CommunicationsClock();
-  /*Don't sleep if a signal has stored messages for us*/
-  if (sleepTime&&CmiGetState()->idle.hasMessages) sleepTime=0;
-  while (CheckSocketsReady(sleepTime)>0) {
-    int again=0;
-      MACHSTATE(2,"CheckSocketsReady returned true");
-    sleepTime=0;
-    if (ctrlskt_ready_read) {again=1;ctrl_getone();}
-    if (dataskt_ready_read) {again=1;ReceiveDatagram();}
-    if (dataskt_ready_write) {
-      if (writeableAcks) 
-        if (0!=(writeableAcks=TransmitAcknowledgement())) again=1;
-      if (writeableDgrams)
-        if (0!=(writeableDgrams=TransmitDatagram())) again=1; 
-    }
-    if (CmiStdoutNeedsService()) {CmiStdoutService();}
-    if (!again) break; /* Nothing more to do */
-    if ((nTimes++ &16)==15) {
-      /*We just grabbed a whole pile of packets-- try to retire a few*/
-      CommunicationsClock();
-    }
-  }
-  CmiCommUnlock();
-  inProgress[CmiMyRank()] -= 1;
-
-  /* when called by communication thread or in interrupt */
-  if (where == COMM_SERVER_FROM_SMP || where == COMM_SERVER_FROM_INTERRUPT) {
-#if CMK_IMMEDIATE_MSG
-  CmiHandleImmediate();
-#endif
-#if CMK_PERSISTENT_COMM
-  PumpPersistent();
-#endif
-  }
-
-  MACHSTATE(1,"} CommunicationServer") 
-}
-
-void CmiMachineInit(char **argv)
-{
-}
-
-void CmiCommunicationInit(char **argv)
-{
-}
-
-void CmiMachineExit(void)
-{
-}
-
-
diff --git a/src/arch/net/machine-gm.c b/src/arch/net/machine-gm.c
deleted file mode 100644 (file)
index 07d8766..0000000
+++ /dev/null
@@ -1,1333 +0,0 @@
-/** @file
- * Myrinet API GM implementation of Converse NET version
- * @ingroup NET
- * contains only GM API specific code for:
- * - CmiMachineInit()
- * - CmiCommunicationInit()
- * - CmiNotifyIdle()
- * - DeliverViaNetwork()
- * - CommunicationServer()
- * - CmiMachineExit()
-
-  written by 
-  Gengbin Zheng, gzheng@uiuc.edu  4/22/2001
-  
-  ChangeLog:
-  * 3/7/2004,  Gengbin Zheng
-    implemented fault tolerant gm layer. When GM detects a catastrophic error,
-    it temporarily disables the delivery of all messages with the same sender 
-    port, target port, and priority as the message that experienced the error. 
-    This layer needs to properly handle the error message of GM and resume 
-    the port.
-
-  TODO:
-  1. DMAable buffer reuse;
-*/
-
-/**
- * @addtogroup NET
- * @{
- */
-
-#ifdef GM_API_VERSION_2_0
-#if GM_API_VERSION >= GM_API_VERSION_2_0
-#define CMK_USE_GM2    1
-#endif
-#endif
-
-/* default as in busywaiting mode */
-#undef CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT
-#undef CMK_WHEN_PROCESSOR_IDLE_USLEEP
-#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT 1
-#define CMK_WHEN_PROCESSOR_IDLE_USLEEP 0
-
-#ifdef __ONESIDED_IMPL
-#ifdef __ONESIDED_GM_HARDWARE
-#include "conv-onesided.h"
-int getSrcHandler;
-int getDestHandler;
-void handleGetSrc(void *msg);
-void handleGetDest(void *msg);
-#endif
-#endif
-
-static gm_alarm_t gmalarm;
-
-/*#define CMK_USE_CHECKSUM*/
-
-/******************************************************************************
- *
- *  GM layer network statistics collection
- *
- *****************************************************************************/
-
-#define GM_STATS               0
-
-#if GM_STATS
-static FILE *gmf;                      /* one file per processor */
-static int  *gm_stats;                 /* send count for each size */
-static int   possible_streamed = 0;    /* possible streaming counts */
-static int   defrag = 0;               /* number of defragment */
-static int   maxQueueLength = 0;       /* maximum send queue length */
-#endif
-
-/******************************************************************************
- *
- * Send messages pending queue (used internally)
- *
- *****************************************************************************/
-
-
-/* max length of pending messages */
-#define MAXPENDINGSEND  500
-
-typedef struct PendingMsgStruct
-{
-  void *msg;
-  int length;          /* length of message */
-  int size;            /* size of message, usually around log2(length)  */
-  int mach_id;         /* receiver machine id */
-  int dataport;                /* receiver data port */
-  int node_idx;                /* receiver pe id */
-  struct PendingMsgStruct *next;
-}
-*PendingMsg;
-
-static int pendinglen = 0;
-
-/* reuse PendingMsg memory */
-static PendingMsg pend_freelist=NULL;
-
-#define FreePendingMsg(d)      \
-  d->next = pend_freelist;\
-  pend_freelist = d;\
-
-#define MallocPendingMsg(d) \
-  d = pend_freelist;\
-  if (d==0) {d = ((PendingMsg)malloc(sizeof(struct PendingMsgStruct)));\
-             _MEMCHECK(d);\
-  } else pend_freelist = d->next;
-
-void enqueue_sending(char *msg, int length, OtherNode node, int size)
-{
-  PendingMsg pm;
-  MallocPendingMsg(pm);
-  pm->msg = msg;
-  pm->length = length;
-  pm->mach_id = node->mach_id;
-  pm->dataport = node->dataport;
-  pm->node_idx = node-nodes;
-  pm->size = size;
-  pm->next = NULL;
-  if (node->sendhead == NULL) {
-    node->sendhead = node->sendtail = pm;
-  }
-  else {
-    node->sendtail->next = pm;
-    node->sendtail = pm;
-  }
-  pendinglen ++;
-#if GM_STATS
-  if (pendinglen > maxQueueLength) maxQueueLength = pendinglen;
-#endif
-}
-
-#define peek_sending(node) (node->sendhead)
-
-#define dequeue_sending(node)  \
-  if (node->sendhead != NULL) {        \
-    node->sendhead = node->sendhead->next;     \
-    pendinglen --;     \
-  }
-
-static void alarmcallback (void *context) {
-  MACHSTATE(4,"GM Alarm callback executed")
-}
-static int processEvent(gm_recv_event_t *e);
-static void send_progress(void);
-static void alarmInterrupt(int arg);
-static int gmExit(int code,const char *msg);
-static char *getErrorMsg(gm_status_t status);
-
-/******************************************************************************
- *
- * DMA message pool
- *
- *****************************************************************************/
-
-#define CMK_MSGPOOL  1
-
-#define MAXMSGLEN  200
-
-static char* msgpool[MAXMSGLEN];
-static int msgNums = 0;
-
-static int maxMsgSize = 0;
-
-#define putPool(msg)   {       \
-  if (msgNums == MAXMSGLEN) gm_dma_free(gmport, msg);  \
-  else msgpool[msgNums++] = msg; }     
-
-#define getPool(msg, len)      {       \
-  if (msgNums == 0) msg  = gm_dma_malloc(gmport, maxMsgSize);  \
-  else msg = msgpool[--msgNums];       \
-}
-
-
-/******************************************************************************
- *
- * CmiNotifyIdle()-- wait until a packet comes in
- *
- *****************************************************************************/
-typedef struct {
-char none;  
-} CmiIdleState;
-
-static CmiIdleState *CmiNotifyGetState(void) { return NULL; }
-
-static void CmiNotifyStillIdle(CmiIdleState *s);
-
-static void CmiNotifyBeginIdle(CmiIdleState *s)
-{
-  CmiNotifyStillIdle(s);
-}
-
-
-
-static void CmiNotifyStillIdle(CmiIdleState *s)
-{
-#if CMK_SHARED_VARS_UNAVAILABLE
-  /*No comm. thread-- listen on sockets for incoming messages*/
-  int nreadable;
-  gm_recv_event_t *e;
-  int pollMs = 4;
-
-#define SLEEP_USING_ALARM 0
-#if SLEEP_USING_ALARM /*Enable the alarm, so we don't sleep forever*/
-  gm_set_alarm (gmport, &gmalarm, (gm_u64_t) pollMs*1000, alarmcallback,
-                    (void *)NULL );
-#endif
-
-#if SLEEP_USING_ALARM
-  MACHSTATE(3,"Blocking on receive {")
-  e = gm_blocking_receive_no_spin(gmport);
-  MACHSTATE(3,"} receive returned");
-#else
-  MACHSTATE(3,"CmiNotifyStillIdle NonBlocking on receive {")
-  e = gm_receive(gmport);
-  MACHSTATE(3,"} CmiNotifyStillIdle nonblocking receive returned");
-#endif
-
-#if SLEEP_USING_ALARM /*Cancel the alarm*/
-  gm_cancel_alarm (&gmalarm);
-#endif
-  
-  /* have to handle this event now */
-  CmiCommLock();
-  inProgress[CmiMyRank()] += 1;
-  nreadable = processEvent(e);
-  CmiCommUnlock();
-  inProgress[CmiMyRank()] -= 1;
-  if (nreadable) {
-    return;
-  }
-#else
-  /*Comm. thread will listen on sockets-- just sleep*/
-  CmiIdleLock_sleep(&CmiGetState()->idle,5);
-#endif
-}
-
-void CmiNotifyIdle(void) {
-  CmiNotifyStillIdle(NULL);
-}
-
-/****************************************************************************
- *                                                                          
- * CheckSocketsReady
- *
- * Checks both sockets to see which are readable and which are writeable.
- * We check all these things at the same time since this can be done for
- * free with ``select.'' The result is stored in global variables, since
- * this is essentially global state information and several routines need it.
- *
- ***************************************************************************/
-
-int CheckSocketsReady(int withDelayMs)
-{   
-  int nreadable;
-  CMK_PIPE_DECL(withDelayMs);
-
-  CmiStdoutAdd(CMK_PIPE_SUB);
-  if (Cmi_charmrun_fd!=-1) CMK_PIPE_ADDREAD(Cmi_charmrun_fd);
-
-  nreadable=CMK_PIPE_CALL();
-  ctrlskt_ready_read = 0;
-  dataskt_ready_read = 0;
-  dataskt_ready_write = 0;
-  
-  if (nreadable == 0) {
-    MACHSTATE(1,"} CheckSocketsReady (nothing readable)")
-    return nreadable;
-  }
-  if (nreadable==-1) {
-    CMK_PIPE_CHECKERR();
-    MACHSTATE(2,"} CheckSocketsReady (INTERRUPTED!)")
-    return CheckSocketsReady(0);
-  }
-  CmiStdoutCheck(CMK_PIPE_SUB);
-  if (Cmi_charmrun_fd!=-1) 
-          ctrlskt_ready_read = CMK_PIPE_CHECKREAD(Cmi_charmrun_fd);
-  MACHSTATE(1,"} CheckSocketsReady")
-  return nreadable;
-}
-
-/***********************************************************************
- * CommunicationServer()
- * 
- * This function does the scheduling of the tasks related to the
- * message sends and receives. 
- * It first check the charmrun port for message, and poll the gm event
- * for send complete and outcoming messages.
- *
- ***********************************************************************/
-
-/* always called from interrupt */
-static void ServiceCharmrun_nolock(void)
-{
-  int again = 1;
-  MACHSTATE(2,"ServiceCharmrun_nolock begin {")
-  while (again)
-  {
-  again = 0;
-  CheckSocketsReady(0);
-  if (ctrlskt_ready_read) { ctrl_getone(); again=1; }
-  if (CmiStdoutNeedsService()) { CmiStdoutService(); }
-  }
-  MACHSTATE(2,"} ServiceCharmrun_nolock end")
-}
-
-static void CommunicationServer_nolock(int withDelayMs) {
-  gm_recv_event_t *e;
-
-  MACHSTATE(2,"CommunicationServer_nolock start {")
-  while (1) {
-    MACHSTATE(3,"Non-blocking receive {")
-    e = gm_receive(gmport);
-    MACHSTATE(3,"} Non-blocking receive")
-    if (!processEvent(e)) break;
-  }
-  MACHSTATE(2,"}CommunicationServer_nolock end")
-}
-
-/*
-0: from smp thread
-1: from interrupt
-2: from worker thread
-   Note in netpoll mode, charmrun service is only performed in interrupt, 
- pingCharmrun is from sig alarm, so it is lock free 
-*/
-static void CommunicationServer(int withDelayMs, int where)
-{
-  /* standalone mode */
-  if (Cmi_charmrun_pid == 0 && gmport == NULL) return;
-
-  MACHSTATE2(2,"CommunicationServer(%d) from %d {",withDelayMs, where)
-
-  if (where == 1) {
-    /* don't service charmrun if converse exits, this fixed a hang bug */
-    if (!machine_initiated_shutdown) ServiceCharmrun_nolock();
-    return;
-  }
-
-  LOG(GetClock(), Cmi_nodestart, 'I', 0, 0);
-
-  CmiCommLock();
-  inProgress[CmiMyRank()] += 1;
-  CommunicationServer_nolock(withDelayMs);
-  CmiCommUnlock();
-  inProgress[CmiMyRank()] -= 1;
-
-#if CMK_IMMEDIATE_MSG
-  if (where == 0)
-  CmiHandleImmediate();
-#endif
-
-  MACHSTATE(2,"} CommunicationServer")
-}
-
-static void processMessage(char *msg, int len)
-{
-  char *newmsg;
-  int rank, srcpe, seqno, magic, i;
-  unsigned int broot;
-  int size;
-  unsigned char checksum;
-  
-  if (len >= DGRAM_HEADER_SIZE) {
-    DgramHeaderBreak(msg, rank, srcpe, magic, seqno, broot);
-#ifdef CMK_USE_CHECKSUM
-    checksum = computeCheckSum(msg, len);
-    if (checksum == 0)
-#else
-    if (magic == (Cmi_charmrun_pid&DGRAM_MAGIC_MASK))
-#endif
-    {
-      OtherNode node = nodes_by_pe[srcpe];
-      /* check seqno */
-      if (seqno == node->recv_expect) {
-       node->recv_expect = ((seqno+1)&DGRAM_SEQNO_MASK);
-      }
-      else if (seqno < node->recv_expect) {
-        CmiPrintf("[%d] Warning: Past packet received from PE %d, something wrong with GM hardware? (expecting: %d seqno: %d)\n", CmiMyPe(), srcpe, node->recv_expect, seqno);
-       CmiPrintf("\n\n\t\t[%d] packet ignored!\n\n");
-       return;
-      }
-      else {
-         CmiPrintf("[%d] Error detected - Packet out of order from PE %d, something wrong with GM hardware? (expecting: %d got: %d)\n", CmiMyPe(), srcpe, node->recv_expect, seqno);
-         CmiAbort("\n\n\t\tPacket out of order!!\n\n");
-      }
-      newmsg = node->asm_msg;
-      if (newmsg == 0) {
-        size = CmiMsgHeaderGetLength(msg);
-        if (len > size) {
-         CmiPrintf("size: %d, len:%d.\n", size, len);
-         CmiAbort("\n\n\t\tLength mismatch!!\n\n");
-        }
-        newmsg = (char *)CmiAlloc(size);
-        _MEMCHECK(newmsg);
-        memcpy(newmsg, msg, len);
-        node->asm_rank = rank;
-        node->asm_total = size;
-        node->asm_fill = len;
-        node->asm_msg = newmsg;
-      } else {
-        size = len - DGRAM_HEADER_SIZE;
-        if (node->asm_fill+size > node->asm_total) {
-         CmiPrintf("asm_total: %d, asm_fill: %d, len:%d.\n", node->asm_total, node->asm_fill, len);
-         CmiAbort("\n\n\t\tLength mismatch!!\n\n");
-        }
-        memcpy(newmsg + node->asm_fill, msg+DGRAM_HEADER_SIZE, size);
-        node->asm_fill += size;
-      }
-      /* get a full packet */
-      if (node->asm_fill == node->asm_total) {
-        int total_size = node->asm_total;
-        node->asm_msg = 0;
-
-      /* do it after integration - the following function may re-entrant */
-#if CMK_BROADCAST_SPANNING_TREE
-        if (rank == DGRAM_BROADCAST
-#if CMK_NODE_QUEUE_AVAILABLE
-          || rank == DGRAM_NODEBROADCAST
-#endif
-           )
-          SendSpanningChildren(NULL, 0, total_size, newmsg, broot, rank);
-#elif CMK_BROADCAST_HYPERCUBE
-        if (rank == DGRAM_BROADCAST
-#if CMK_NODE_QUEUE_AVAILABLE
-          || rank == DGRAM_NODEBROADCAST
-#endif
-           )
-          SendHypercube(NULL, 0, total_size, newmsg, broot, rank);
-#endif
-
-        switch (rank) {
-        case DGRAM_BROADCAST: {
-          for (i=1; i<_Cmi_mynodesize; i++)
-            CmiPushPE(i, CopyMsg(newmsg, total_size));
-          CmiPushPE(0, newmsg);
-          break;
-        }
-#if CMK_NODE_QUEUE_AVAILABLE
-        case DGRAM_NODEBROADCAST: 
-        case DGRAM_NODEMESSAGE: {
-          CmiPushNode(newmsg);
-          break;
-        }
-#endif
-        default:
-          CmiPushPE(rank, newmsg);
-        }    /* end of switch */
-      }
-    } 
-    else {
-#ifdef CMK_USE_CHECKSUM
-      CmiPrintf("[%d] message ignored: checksum (%d) not 0!\n", CmiMyPe(), checksum);
-#else
-      CmiPrintf("[%d] message ignored: magic not agree:%d != %d!\n", 
-                 CmiMyPe(), magic, Cmi_charmrun_pid&DGRAM_MAGIC_MASK);
-#endif
-      CmiPrintf("recved: rank:%d src:%d mag:%d seqno:%d len:%d\n", rank, srcpe, magic, seqno, len);
-    }
-  } 
-  else {
-      CmiPrintf("[%d] message ignored: size is too small: %d!\n", CmiMyPe(), len);
-      CmiPrintf("[%d] possible size: %d\n", CmiMsgHeaderGetLength(msg));
-  }
-}
-
-/* return 1 - recv'ed  0 - no msg */
-static int processEvent(gm_recv_event_t *e)
-{
-  int size, len;
-  char *msg, *buf;
-  int status = 1;
-  switch (gm_ntohc(e->recv.type))
-  {
-      /* avoid copy from message to buffer */
-    case GM_FAST_PEER_RECV_EVENT:
-    case GM_FAST_RECV_EVENT:
-      MACHSTATE(4,"Incoming message")
-      msg = gm_ntohp(e->recv.message);
-      len = gm_ntohl(e->recv.length);
-      processMessage(msg, len);
-      break;
-/*
-    case GM_FAST_HIGH_PEER_RECV_EVENT:
-    case GM_FAST_HIGH_RECV_EVENT:
-*/
-    case GM_HIGH_RECV_EVENT:
-    case GM_RECV_EVENT:
-      MACHSTATE(4,"Incoming message")
-      size = gm_ntohc(e->recv.size);
-      msg = gm_ntohp(e->recv.buffer);
-      len = gm_ntohl(e->recv.length);
-      processMessage(msg, len);
-      gm_provide_receive_buffer(gmport, msg, size, GM_HIGH_PRIORITY);
-      break;
-    case GM_NO_RECV_EVENT:
-      return 0;
-    case GM_ALARM_EVENT:
-      status = 0;
-    default:
-      MACHSTATE1(3,"Unrecognized GM event %d", gm_ntohc(e->recv.type))
-      gm_unknown(gmport, e);
-  }
-  return status;
-}
-
-
-#ifdef __FAULT__ 
-void drop_send_callback(struct gm_port *p, void *context, gm_status_t status)
-{
-  PendingMsg out = (PendingMsg)context;
-  void *msg = out->msg;
-
-  printf("[%d] drop_send_callback dropped msg: %p\n", CmiMyPe(), msg);
-#if !CMK_MSGPOOL
-  gm_dma_free(gmport, msg);
-#else
-  putPool(msg);
-#endif
-
-  FreePendingMsg(out);
-}
-#else
-void send_callback(struct gm_port *p, void *context, gm_status_t status);
-void drop_send_callback(struct gm_port *p, void *context, gm_status_t status)
-{
-  PendingMsg out = (PendingMsg)context;
-  void *msg = out->msg;
-  gm_send_with_callback(gmport, msg, out->size, out->length,
-                        GM_HIGH_PRIORITY, out->mach_id, out->dataport, 
-                        send_callback, out);
-}
-#endif
-
-void send_callback(struct gm_port *p, void *context, gm_status_t status)
-{
-  PendingMsg out = (PendingMsg)context;
-  void *msg = out->msg;
-  unsigned char cksum;
-  OtherNode  node = nodes+out->node_idx;
-
-  if (status != GM_SUCCESS) { 
-    int srcpe, seqno, magic, broot;
-    char rank;
-    char *errmsg;
-    DgramHeaderBreak(msg, rank, srcpe, magic, seqno, broot);
-    errmsg = getErrorMsg(status);
-    CmiPrintf("GM Error> PE:%d send to msg %p node %d rank %d mach_id %d port %d len %d size %d failed to complete (error %d): %s\n", srcpe, msg, out->node_idx, rank, out->mach_id, out->dataport, out->length, out->size, status, errmsg); 
-    switch (status) {
-#ifdef __FAULT__ 
-      case GM_SEND_DROPPED: {
-        OtherNode node = nodes + out->node_idx;
-        if (out->mach_id == node->mach_id && out->dataport == node->dataport) {
-          /* it not crashed, resent */
-          gm_send_with_callback(gmport, msg, out->size, out->length, 
-                            GM_HIGH_PRIORITY, out->mach_id, out->dataport, 
-                            send_callback, out);
-          return;
-        }
-      }
-      default: {
-        gm_drop_sends (gmport, GM_HIGH_PRIORITY, out->mach_id, out->dataport,
-                                            drop_send_callback, out);
-        return;
-      }
-#else
-      case GM_SEND_TIMED_OUT: {
-       CmiPrintf("gm send_callback timeout, drop sends and re-enable send. \n");
-        gm_drop_sends (gmport, GM_HIGH_PRIORITY, out->mach_id, out->dataport,
-                         drop_send_callback, out);
-       node->disable=1;        /* disable normal send */
-        return;
-      }
-      case GM_SEND_DROPPED:
-        CmiPrintf("Got DROPPED_SEND notification, resend\n");
-        gm_send_with_callback(gmport, msg, out->size, out->length,
-                            GM_HIGH_PRIORITY, out->mach_id, out->dataport, 
-                            send_callback, out);
-        return;
-      default:
-        CmiAbort("gm send_callback failed");
-#endif
-    }
-  }
-
-#ifdef CMK_USE_CHECKSUM
-/*
-  {
-  cksum = computeCheckSum((unsigned char*)msg, out->length);
-  if (cksum != 0) {
-    CmiPrintf("[%d] Message altered during send, checksum (%d) does not agree!\n", CmiMyPe(), cksum);
-    CmiAbort("Myrinet error was detected!\n");
-  }
-  }
-*/
-#endif
-
-#if !CMK_MSGPOOL
-  gm_dma_free(gmport, msg);
-#else
-  putPool(msg);
-#endif
-
-  gm_free_send_token (gmport, GM_HIGH_PRIORITY);
-  FreePendingMsg(out);
-
-  /* send message pending in gm firmware */
-  node->gm_pending --;
-  if (node->disable && node->gm_pending == 0) node->disable = 0;
-
-  /* since we have one free send token, start next send */
-  send_progress();
-}
-
-static void send_progress(void)
-{
-  static int nextnode = 0;
-  int skip;
-  OtherNode node;
-  PendingMsg  out;
-
-  while (1)
-  {
-    int sent = 0;
-    for (skip=0; skip<_Cmi_numnodes; skip++) {
-      node = nodes+nextnode;
-      nextnode = (nextnode + 1) % _Cmi_numnodes;
-      if (node->disable) continue;
-      out = peek_sending(node);
-      if (!out) continue;
-      if (gm_alloc_send_token(gmport, GM_HIGH_PRIORITY)) {
-        if (dataport == out->dataport) {
-          gm_send_to_peer_with_callback(gmport, out->msg, out->size, 
-                            out->length, GM_HIGH_PRIORITY, out->mach_id,
-                            send_callback, out);
-        }
-        else {
-          gm_send_with_callback(gmport, out->msg, out->size, out->length, 
-                            GM_HIGH_PRIORITY, out->mach_id, out->dataport, 
-                            send_callback, out);
-        }
-       node->gm_pending ++;
-        sent=1;
-        /* dequeue out, but not free it, used at callback */
-        dequeue_sending(node);
-#if GM_STATS
-        gm_stats[out->size] ++;
-        /* if we streaming, count how many message we possibly can combine */
-        {
-        PendingMsg  curout;
-        curout = peek_sending(node);
-        if (curout) {
-          out = curout->next;
-          while (out) {
-            possible_streamed ++;
-            out = out->next;
-          }
-        }
-        }
-#endif
-      }
-      else return;             /* no send token */
-    }          /* end for */
-    if (sent==0) return;
-  }
-}
-
-
-/***********************************************************************
- * DeliverViaNetwork()
- *
- * This function is responsible for all non-local transmission. It
- * first allocate a send token, if fails, put the send message to
- * penging message queue, otherwise invoke the GM send.
- ***********************************************************************/
-
-void EnqueueOutgoingDgram
-     (OutgoingMsg ogm, char *ptr, int dlen, OtherNode node, int rank, int broot)
-{
-  char *buf;
-  int size, len, seqno;
-  int alloclen, allocSize;
-
-/* CmiPrintf("DeliverViaNetwork: size:%d\n", size); */
-
-  /* don't have to worry about ref count because we do copy, and
-     ogm can be free'ed right away */
-  /* ogm->refcount++; */
-
-  len = dlen + DGRAM_HEADER_SIZE;
-
-  /* allocate DMAable memory to prepare sending */
-  /* FIXME: another memory copy here from user buffer to DMAable buffer */
-  /* which however means the user buffer is untouched and can be reused */
-#if !CMK_MSGPOOL
-  buf = (char *)gm_dma_malloc(gmport, len);
-#else
-  getPool(buf, len);
-#endif
-  _MEMCHECK(buf);
-
-  seqno = node->send_next;
-  DgramHeaderMake(buf, rank, ogm->src, Cmi_charmrun_pid, seqno, broot);
-  node->send_next = ((seqno+1)&DGRAM_SEQNO_MASK);
-  memcpy(buf+DGRAM_HEADER_SIZE, ptr, dlen);
-#ifdef CMK_USE_CHECKSUM
-  {
-  DgramHeader *head = (DgramHeader *)buf;
-  head->magic ^= computeCheckSum(buf, len);
-  }
-#endif
-  size = gm_min_size_for_length(len);
-
-  /* if queue is not empty, enqueue msg. this is to guarantee the order */
-  if (pendinglen != 0) {
-    /* this potential screw up broadcast, because bcast packets can not be
-       interrupted by other sends in CommunicationServer_nolock */
-    enqueue_sending(buf, len, node, size);
-    return;
-  }
-  enqueue_sending(buf, len, node, size);
-  send_progress();
-}
-
-/* copy is ignored, since we always copy */
-void DeliverViaNetwork(OutgoingMsg ogm, OtherNode node, int rank, unsigned int broot, int copy)
-{
-  int size; char *data;
-  size = ogm->size - DGRAM_HEADER_SIZE;
-  data = ogm->data + DGRAM_HEADER_SIZE;
-#if GM_STATS
-  if (size > Cmi_dgram_max_data) defrag ++;
-#endif
-  while (size > Cmi_dgram_max_data) {
-    EnqueueOutgoingDgram(ogm, data, Cmi_dgram_max_data, node, rank, broot);
-    data += Cmi_dgram_max_data;
-    size -= Cmi_dgram_max_data;
-  }
-  if (size>0) EnqueueOutgoingDgram(ogm, data, size, node, rank, broot);
-
-  /* a simple flow control */
-  while (pendinglen >= MAXPENDINGSEND) {
-      /* pending max len exceeded, busy wait until get a token 
-         Doing this surprisingly improve the performance by 2s for 200MB msg */
-      MACHSTATE(4,"Polling until token available")
-      CommunicationServer_nolock(0);
-  }
-}
-
-/* simple barrier at machine layer */
-/* assuming no other flying messages */
-static void send_callback_nothing(struct gm_port *p, void *context, gm_status_t status)
-{
-  gm_dma_free(gmport, context);
-}
-
-static void sendBarrierMessage(int pe)
-{
-  int len = 32;
-  char *buf = (char *)gm_dma_malloc(gmport, len);
-  int size = gm_min_size_for_length(len);
-  OtherNode  node = nodes + pe;
-  CmiAssert(buf);
-  gm_send_with_callback(gmport, buf, size, len,
-              GM_HIGH_PRIORITY, node->mach_id, node->dataport,
-              send_callback_nothing, buf);
-}
-
-static void recvBarrierMessage(void)
-{
-  gm_recv_event_t *e;
-  int size, len;
-  char *msg;
-  while (1) {
-    e = gm_receive(gmport);
-    switch (gm_ntohc(e->recv.type))
-    {
-      case GM_HIGH_RECV_EVENT:
-      case GM_RECV_EVENT:
-        MACHSTATE(4,"Incoming message")
-        size = gm_ntohc(e->recv.size);
-        msg = gm_ntohp(e->recv.buffer);
-        len = gm_ntohl(e->recv.length);
-        gm_provide_receive_buffer(gmport, msg, size, GM_HIGH_PRIORITY);
-        return;
-      case GM_NO_RECV_EVENT:
-        continue ;
-      default:
-        MACHSTATE1(3,"Unrecognized GM event %d", gm_ntohc(e->recv.type))
-        gm_unknown(gmport, e);
-    }
-  }
-}
-
-/* happen at node level */
-int CmiBarrier(void)
-{
-  int len, size, i;
-  int status;
-  int count = 0;
-  OtherNode node;
-  int numnodes = CmiNumNodes();
-  if (CmiMyRank() == 0) {
-    /* every one send to pe 0 */
-    if (CmiMyNode() != 0) {
-      sendBarrierMessage(0);
-    }
-    /* printf("[%d] HERE\n", CmiMyPe()); */
-    if (CmiMyNode() == 0) 
-    {
-      for (count = 1; count < numnodes; count ++) 
-      {
-        recvBarrierMessage();
-      }
-      /* pe 0 broadcast */
-      for (i=1; i<=BROADCAST_SPANNING_FACTOR; i++) {
-        int p = i;
-        if (p > numnodes - 1) break;
-        /* printf("[%d] BD => %d \n", CmiMyPe(), p); */
-        sendBarrierMessage(p);
-      }
-    }
-    /* non 0 node waiting */
-    if (CmiMyNode() != 0) 
-    {
-      recvBarrierMessage();
-      for (i=1; i<=BROADCAST_SPANNING_FACTOR; i++) {
-        int p = CmiMyNode();
-        p = BROADCAST_SPANNING_FACTOR*p + i;
-        if (p > numnodes - 1) break;
-        p = p%numnodes;
-        /* printf("[%d] RELAY => %d \n", CmiMyPe(), p); */
-        sendBarrierMessage(p);
-      }
-    }
-  }
-  CmiNodeAllBarrier();
-  /* printf("[%d] OUT of barrier \n", CmiMyPe()); */
-  return 0;
-}
-
-/* everyone sends a message to pe 0 and go on */
-int CmiBarrierZero(void)
-{
-  int i;
-
-  if (CmiMyRank() == 0) {
-    if (CmiMyNode()) {
-      sendBarrierMessage(0);
-    }
-    else {
-      for (i=0; i<CmiNumNodes()-1; i++)
-      {
-        recvBarrierMessage();
-      }
-    }
-  }
-  CmiNodeAllBarrier();
-  return 0;
-}
-
-/***********************************************************************
- * CmiMachineInit()
- *
- * This function intialize the GM board. Set receive buffer
- *
- ***********************************************************************/
-
-static int maxsize;
-
-void CmiMachineInit(char **argv)
-{
-  int dataport_max=16; /*number of largest GM port to check*/
-  gm_status_t status;
-  int device, i, j;
-  int retry = 10;
-  char *buf;
-  int mlen;
-
-  MACHSTATE(3,"CmiMachineInit {");
-
-  gmport = NULL;
-  /* standalone mode */
-  if (dataport == -1) return; 
-
-  /* try a few times init gm */
-  for (i=0; i<retry; i++) {
-    status = gm_init();
-    if (status == GM_SUCCESS) break;
-    sleep(1);
-  }
-  if (status != GM_SUCCESS) { 
-    printf("Cannot open GM library (does the machine have a GM card?)\n");
-    gm_perror("gm_init", status); 
-    return; 
-  }
-  
-  device = 0;
-  for (dataport=2;dataport<dataport_max;dataport++) {
-    char portname[200];
-    sprintf(portname, "converse_port%d_%d", Cmi_charmrun_pid, _Cmi_mynode);
-#if CMK_USE_GM2
-    status = gm_open(&gmport, device, dataport, portname, GM_API_VERSION_2_0);
-#else
-    status = gm_open(&gmport, device, dataport, portname, GM_API_VERSION_1_1);
-#endif
-    if (status == GM_SUCCESS) { break; }
-  }
-  if (dataport==dataport_max) 
-  { /* Couldn't open any GM port... */
-    dataport=0;
-    return;
-  }
-  
-  /* get our node id */
-  status = gm_get_node_id(gmport, (unsigned int *)&Cmi_mach_id);
-  if (status != GM_SUCCESS) { gm_perror("gm_get_node_id", status); return; }
-#if CMK_USE_GM2
-  gm_node_id_to_global_id(gmport, (unsigned int)Cmi_mach_id, (unsigned int*)&Cmi_mach_id);
-#endif
-  
-  /* default abort will take care of gm clean up */
-  skt_set_abort(gmExit);
-
-  /* set up recv buffer */
-/*
-  maxsize = gm_min_size_for_length(4096);
-  Cmi_dgram_max_data = 4096 - DGRAM_HEADER_SIZE;
-*/
-  maxsize = 16;
-  CmiGetArgIntDesc(argv,"+gm_maxsize",&maxsize,"maximum packet size in rank (2^maxsize)");
-
-#if GM_STATS
-  gm_stats = (int*)malloc((maxsize+1) * sizeof(int));
-  for (i=0; i<=maxsize; i++) gm_stats[i] = 0;
-#endif
-
-  for (i=1; i<=maxsize; i++) {
-    int len = gm_max_length_for_size(i);
-    int num = 2;
-
-    maxMsgSize = len;
-
-    if (i<5) num = 0;
-    else if (i<7)  num = 4;
-    else if (i<13)  num = 20;
-    else if (i<17)  num = 10;
-    else if (i>22) num = 1;
-    for (j=0; j<num; j++) {
-      buf = gm_dma_malloc(gmport, len);
-      _MEMCHECK(buf);
-      gm_provide_receive_buffer(gmport, buf, i, GM_HIGH_PRIORITY);
-    }
-  }
-  Cmi_dgram_max_data = maxMsgSize - DGRAM_HEADER_SIZE;
-
-  status = gm_set_acceptable_sizes (gmport, GM_HIGH_PRIORITY, (1<<(maxsize+1))-1);
-
-  gm_free_send_tokens (gmport, GM_HIGH_PRIORITY,
-                       gm_num_send_tokens (gmport));
-
-#if CMK_MSGPOOL
-  msgpool[msgNums++]  = gm_dma_malloc(gmport, maxMsgSize);
-#endif
-
-  /* alarm will ping charmrun */
-  gm_initialize_alarm(&gmalarm);
-
-  MACHSTATE(3,"} CmiMachineInit");
-}
-
-void CmiCommunicationInit(char **argv)
-{
-}
-
-void CmiMachineExit(void)
-{
-#if GM_STATS
-  int i;
-  int mype;
-  char fname[128];
-  sprintf(fname, "gm-stats.%d", CmiMyPe());
-  gmf = fopen(fname, "w");
-  mype = CmiMyPe();
-  for (i=5; i<=maxsize; i++)  {
-    fprintf(gmf, "[%d] size:%d count:%d\n", mype, i, gm_stats[i]);
-  }
-  fprintf(gmf, "[%d] max quelen: %d possible streaming: %d  defrag: %d \n", mype, maxQueueLength, possible_streamed, defrag);
-  fclose(gmf);
-#endif
-}
-
-void CmiGmConvertMachineID(unsigned int *mach_id)
-{
-#if CMK_USE_GM2 
-    gm_status_t status;
-    int newid;
-    /* skip if running without charmrun */
-    if (Cmi_charmrun_pid == 0 && gmport == NULL) return;
-    status = gm_global_id_to_node_id(gmport, *mach_id, &newid);
-    if (status == GM_SUCCESS) *mach_id = newid;
-#endif
-}
-
-/* make sure other gm nodes are accessible in routing table */
-void CmiCheckGmStatus(void)
-{
-  int i;
-  int doabort = 0;
-  if (Cmi_charmrun_pid == 0 && gmport == NULL) return;
-  if (gmport == NULL) machine_exit(1);
-  for (i=0; i<_Cmi_numnodes; i++) {
-    gm_status_t status;
-    char uid[6], str[100];
-    unsigned int mach_id=nodes[i].mach_id;
-    status = gm_node_id_to_unique_id(gmport, mach_id, uid);
-    if (status != GM_SUCCESS || ( uid[0]==0 && uid[1]== 0 
-         && uid[2]==0 && uid[3]==0 && uid[4]==0 && uid[5]==0)) { 
-      CmiPrintf("Error> gm node %d can't contact node %d. \n", CmiMyPe(), i);
-      doabort = 1;
-    }
-    /*CmiPrintf("[%d]: %d mach:%d ip:%d %d %d %d\n", CmiMyPe(), i, mach_id, nodes[i].IP,uid[0], uid[3], uid[5]);*/
-  }
-  if (doabort) CmiAbort("");
-}
-
-static int gmExit(int code,const char *msg)
-{
-  fprintf(stderr,"Fatal socket error: code %d-- %s\n",code,msg);
-  machine_exit(code);
-}
-
-
-static char *getErrorMsg(gm_status_t status)
-{
-  char *errmsg;
-  switch (status) {
-  case GM_SEND_TIMED_OUT:
-    errmsg = "send time out"; break;
-  case GM_SEND_REJECTED:
-    errmsg = "send rejected"; break;
-  case GM_SEND_TARGET_NODE_UNREACHABLE:
-    errmsg = "target node unreachable"; break;
-  case GM_SEND_TARGET_PORT_CLOSED:
-    errmsg = "target port closed"; break;
-  case GM_SEND_DROPPED:
-    errmsg = "send dropped"; break;
-  default:
-    errmsg = "unknown error"; break;
-  }
-  return errmsg;
-}
-
-#ifdef __ONESIDED_IMPL
-#ifdef __ONESIDED_GM_HARDWARE
-
-struct CmiCb {
-  CmiRdmaCallbackFn fn;
-  void *param;
-};
-typedef struct CmiCb CmiCb;
-
-struct CmiRMA {
-  int type;
-  union {
-    int completed;
-    CmiCb *cb;
-  } ready;
-};
-typedef struct CmiRMA CmiRMA;
-
-struct CmiRMAMsg {
-  char core[CmiMsgHeaderSizeBytes];
-  CmiRMA* stat;
-};
-typedef struct CmiRMAMsg CmiRMAMsg;
-
-struct RMAPutMsg {
-  char core[CmiMsgHeaderSizeBytes];
-  void *Saddr;
-  void *Taddr;
-  unsigned int size;
-  unsigned int targetId;
-  unsigned int sourceId;
-  unsigned int dataport;
-  CmiRMA *stat;
-};
-typedef struct RMAPutMsg RMAPutMsg;
-
-void *CmiDMAAlloc(int size) {
-  void *addr = gm_dma_calloc(gmport, 1, gm_max_length_for_size(size));
-  //gm_allow_remote_memory_access(gmport);
-  return addr;
-}
-
-int CmiRegisterMemory(void *addr, unsigned int size) {
-  gm_status_t status;
-  status = gm_register_memory(gmport, addr, size);
-  if (status != GM_SUCCESS) { 
-    CmiPrintf("Cannot register memory at %p of size %d\n",addr,size);
-    gm_perror("registerMemory", status); 
-    return 0;
-  }
-  //gm_allow_remote_memory_access(gmport);
-  return 1;
-}
-
-int CmiUnRegisterMemory(void *addr, unsigned int size) {
-  gm_status_t status;
-  status = gm_deregister_memory(gmport, addr, size);
-  if (status != GM_SUCCESS) { 
-    CmiPrintf("Cannot unregister memory at %p of size %d\n",addr,size);
-    gm_perror("UnregisterMemory", status); 
-    return 0;
-  }
-  return 1;
-}
-
-void put_callback(struct gm_port *p, void *context, gm_status_t status)
-{
-  RMAPutMsg *out = (RMAPutMsg*)context;
-  unsigned int destMachId = (nodes_by_pe[out->targetId])->mach_id;
-  if (status != GM_SUCCESS) { 
-    switch (status) {
-      case GM_SEND_DROPPED:
-        CmiPrintf("Got DROPPED_PUT notification, resend\n");
-       gm_directed_send_with_callback(gmport,out->Saddr,(gm_remote_ptr_t)(out->Taddr),
-                                      out->size,GM_HIGH_PRIORITY, 
-                                      destMachId, out->dataport, 
-                                      put_callback, out);
-        return;
-      default:
-        CmiAbort("gm send_callback failed");
-    }
-  }
-  if(out->stat->type==1) {
-    out->stat->ready.completed = 1;
-    //the handle is active, and the user will clean it
-  }
-  else {
-    (*(out->stat->ready.cb->fn))(out->stat->ready.cb->param);
-    //I do not undestand, on turing the following two frees become double frees??
-    CmiFree(out->stat->ready.cb);
-    CmiFree(out->stat); //clean up the internal handle
-  }
-  gm_free_send_token (gmport, GM_HIGH_PRIORITY);
-  return;
-}
-
-void *CmiPut(unsigned int sourceId, unsigned int targetId, void *Saddr, void *Taddr, unsigned int size) {
-  unsigned int dataport = (nodes_by_pe[targetId])->dataport;
-  unsigned int destMachId = (nodes_by_pe[targetId])->mach_id;
-  RMAPutMsg *context = (RMAPutMsg*)CmiAlloc(sizeof(RMAPutMsg));
-  context->Saddr = Saddr;
-  context->Taddr = Taddr;
-  context->size = size;
-  context->targetId = targetId;
-  context->sourceId = sourceId;
-  context->dataport = dataport;
-  context->stat = (CmiRMA*)CmiAlloc(sizeof(CmiRMA));
-  context->stat->type = 1;
-  context->stat->ready.completed = 0;
-  void *stat = (void*)(context->stat); 
-  //get a token before the put
-  if (gm_alloc_send_token(gmport, GM_HIGH_PRIORITY)) {
-    //perform the put
-    gm_directed_send_with_callback(gmport,Saddr,(gm_remote_ptr_t)Taddr,size,
-                                  GM_HIGH_PRIORITY,destMachId,dataport,
-                                  put_callback,(void*)context);
-  }
-  return stat;
-}
-
-void CmiPutCb(unsigned&