Merge branch 'charm' of charmgit:charm into charm
authorNikhil Jain <nikhil@illinois.edu>
Wed, 5 Sep 2012 15:13:53 +0000 (08:13 -0700)
committerNikhil Jain <nikhil@illinois.edu>
Wed, 5 Sep 2012 15:13:53 +0000 (08:13 -0700)
66 files changed:
build
doc/charm++/order.tex
examples/charm++/nodeHelper/fft-trans/Makefile [deleted file]
examples/charm++/nodeHelper/fft-trans/README.nodelib [deleted file]
examples/charm++/nodeHelper/fft-trans/fft1d.C [deleted file]
examples/charm++/nodeHelper/fft-trans/fft1d.ci [deleted file]
examples/charm++/nodeHelper/fft-trans/fft_bench.cpp [deleted file]
examples/charm++/nodeHelper/fft-trans/fft_ref.cpp [deleted file]
examples/charm++/nodeHelper/fft-trans/fftmacro.h [deleted file]
examples/charm++/nodeHelper/fft-trans/fileio.h [deleted file]
examples/charm++/nodeHelper/fft-trans/test [deleted file]
examples/charm++/nodeHelper/fft-trans/test.sh [deleted file]
examples/charm++/nodeHelper/simpleLoopBench/Makefile [deleted file]
examples/charm++/nodeHelper/simpleLoopBench/hello.C [deleted file]
examples/charm++/nodeHelper/simpleLoopBench/hello.ci [deleted file]
examples/charm++/nodeHelper/simpleLoopBench/hello.h [deleted file]
examples/charm++/prio/Makefile [new file with mode: 0644]
examples/charm++/prio/pgm.C [new file with mode: 0644]
examples/charm++/prio/pgm.ci [new file with mode: 0644]
examples/charm++/sync_square/Makefile [new file with mode: 0644]
examples/charm++/sync_square/sync_square.cc [new file with mode: 0644]
examples/charm++/sync_square/sync_square.ci [new file with mode: 0644]
examples/charm++/sync_square/sync_square.h [new file with mode: 0644]
src/QuickThreads/md/setjmp64.c
src/QuickThreads/md/setjmp64_.c
src/arch/bluegenep/machine.c
src/arch/gemini_gni/machine.c
src/arch/lapi/machine.c
src/arch/mpi/conv-mach-causalft.h
src/arch/mpi/conv-mach-mlogft.h
src/arch/mpi/machine.c
src/arch/net/conv-mach-mlogft.h
src/arch/util/machine-broadcast.c
src/arch/util/machine-common-core.c
src/arch/util/machine-lrts.h
src/arch/util/machine-smp.c
src/ck-core/ck.C
src/ck-core/ckarray.C
src/ck-core/cklocation.C
src/ck-core/ckmemcheckpoint.C
src/ck-core/ckmessagelogging.C
src/ck-core/ckmessagelogging.h
src/ck-core/ckreduction.C
src/ck-core/ckreduction.h
src/ck-core/envelope.h
src/ck-core/init.C
src/ck-ldb/CentralLB.C
src/conv-core/threads.c
src/libs/ck-libs/NDMeshStreamer/NDMeshStreamer.h
src/libs/ck-libs/ckloop/CkLoop.C
src/libs/ck-libs/ckloop/CkLoopAPI.h
src/libs/ck-libs/collide/collide_serial.C
src/libs/ck-libs/nodeHelper/Make.depends [deleted file]
src/libs/ck-libs/nodeHelper/Makefile [deleted file]
src/libs/ck-libs/nodeHelper/NodeHelper.C [deleted file]
src/libs/ck-libs/nodeHelper/NodeHelper.ci [deleted file]
src/libs/ck-libs/nodeHelper/NodeHelper.h [deleted file]
src/libs/ck-libs/nodeHelper/NodeHelperAPI.h [deleted file]
src/scripts/Makefile
src/scripts/configure
src/scripts/configure.in
src/util/ckhashtable.h
src/util/uJcontext.c
tests/Makefile
tests/charm++/queue/msgqtest.C
tests/charm++/queue/pgm.C

diff --git a/build b/build
index 176be31d9e0f77d72f2a53472109bc8aea032365..97e692ddab14927dec5367658e7ce2778fdfc289 100755 (executable)
--- a/build
+++ b/build
@@ -27,7 +27,7 @@ syntax() {
   echo ''
   fi
   echo '<versions>: ' 
-  ( cd $src ; ls -1 | egrep -v '(^CVS)|(^shmem$)|(^mpi$)|(^sim$)|(^net$)|(^multicore$)|(^elan$)|(^util$)|(^common$)|(^uth$)|(^conv-mach-fix.sh$)|(^win32$)|(^win64$)|(^paragon$)|(^vmi$)|(^lapi$)|(^cell$)|(^gemini_gni$)|(^template$)|(^cuda$)' | pr -3 -t )
+  ( cd $src ; ls -1 | egrep -v '(^CVS)|(^shmem$)|(^mpi$)|(^sim$)|(^net$)|(^multicore$)|(^elan$)|(^util$)|(^common$)|(^uth$)|(^conv-mach-fix.sh$)|(^win32$)|(^win64$)|(^paragon$)|(^vmi$)|(^lapi$)|(^cell$)|(^gemini_gni$)|(^pami$)|(^template$)|(^cuda$)' | pr -3 -t )
   echo ''
   echo '<options>: compiler and platform specific options'
   echo 'cc cc64 cxx kcc pgcc acc icc ecc gcc3 gcc4 mpcc pathscale clang'
@@ -639,6 +639,7 @@ Echo "Performing '$MAKE $MAKEOPTS basics OPTS="$OPTS" QUIET="$QUIET" CONFIG_OPTS
 cd $DESTINATION/tmp 
 echo "CONFIG_OPTS=\"$CONFIG_OPTS\"" > config_opts.sh
 chmod +x config_opts.sh
+echo "OPTSATBUILDTIME += $OPTS" > buildopts.mk
 $MAKE $MAKEOPTS basics OPTS="$OPTS $BUILD_SHARED" QUIET="$QUIET"
 MAKEEXIT=$?
 [ $MAKEEXIT -ne 0 ] && printError
index 836c3197589834167f5cb4446f5ee65194d1f69f..985f1241c91aa92e9971691806bab1df69cb3ddb 100644 (file)
@@ -9,6 +9,9 @@ delivery order on a per-message basis.
 
 % Example general use cases here
 
+An example program demonstrating how to modify delivery order for messages and
+parameter marshaling can be found in \examplerefdir{prio}.
+
 \subsubsection{Queueing Strategies}
 \label{queueing strategies}
 
@@ -68,25 +71,35 @@ degree of control, \charmpp\ also allows explicit prioritization of
 messages.
 
 The other six queueing strategies involve the use of
-priorities\index{priorities}.  There are two kinds of priorities which
-can be attached to a message: {\sl integer priorities}\index{integer
-  priorities} and {\sl bitvector priorities}\index{bitvector
-  priorities}. These correspond to the {\em I} and {\em B} queueing
-strategies, respectively. In both cases, numerically lower priorities
-will be dequeued and delivered before numerically greater
-priorities. The FIFO and LIFO queueing strategies then control the
-relative order in which messages of the same priority will be
-delivered.
-
-To attach a priority field to a
-message, one needs to set aside space in the message's buffer while
-allocating the message\index{message priority}.  To achieve this, the
-size of the priority field\index{priority field} in bits should be
-specified as a placement argument to the \kw{new} operator, as
-described in Section \ref{memory allocation}.  Although the size of
-the priority field is specified in bits, it is always padded to an
-integral number of {\tt int}s.  A pointer to the priority part of the
-message buffer can be obtained with this call:
+priorities\index{priorities}.  There are two kinds of priorities which can be
+attached to a message: \emph{integer priorities}\index{integer priorities} and
+\emph{bitvector priorities}\index{bitvector priorities}. These correspond to
+the \emph{I} and \emph{B} queueing strategies, respectively. In both cases,
+numerically lower priorities will be dequeued and delivered before numerically
+greater priorities. The FIFO and LIFO queueing strategies then control the
+relative order in which messages of the same priority will be delivered.
+
+\subsubsection{Parameter Marshaling}
+
+Integer proiorities for parameter-marshalled messages can be achieved
+through {\tt CkEntryOptions::setPriority()}:
+
+\begin{alltt}
+  CkEntryOptions opts;
+  opts.setPriority(7);
+  chare.entry_name(arg1, arg2, &opts);
+\end{alltt}
+
+\subsubsection{Messages}
+
+To attach a priority field to a message, space must be explicitly allocated in
+the message's buffer while allocating the message\index{message priority}.  To
+achieve this, the size of the priority field\index{priority field} in bits
+should be specified as a placement argument to the \kw{new} operator, as
+described in section~\ref{memory allocation}.  Although the size of the
+priority field is specified in bits, it is always padded to an integral number
+of {\tt int}s.  A pointer to the priority part of the message buffer can be
+obtained with this call:\\
 
 \function{void *CkPriorityPtr(MsgType msg)}
 \index{CkPriorityPtr}
@@ -105,102 +118,85 @@ priority using \kw{CkSetQueueing}:
   CkSetQueueing(msg, CK_QUEUEING_IFIFO);
 \end{alltt}
 
-Integer proiorities for parameter-marshalled messages can be achieved
-through {\tt CkEntryOptions::setPriority()}:
-
-\begin{alltt}
-  CkEntryOptions opts;
-  opts.setPriority(7);
-  chare.entry_name(arg1, arg2, &opts);
-\end{alltt}
-
-Bitvector priorities are somewhat more complicated.  Bitvector
-priorities are arbitrary-length bit-strings representing fixed-point
-numbers in the range 0 to 1.  For example, the bit-string ``001001''
-represents the number .001001\raisebox{-.5ex}{\scriptsize binary}.  As
-with the simpler kind of priority, higher numbers represent lower
-priorities.  Unlike the simpler kind of priority, bitvectors can be of
-arbitrary length, therefore, the priority numbers they represent can
-be of arbitrary precision.
-
-Arbitrary-precision priorities\index{arbitrary-precision priorities}
-are often useful in AI search-tree applications.  Suppose we have a
-heuristic suggesting that tree node $N_1$ should be searched before
-tree node $N_2$.  We therefore designate that node $N_1$ and its
-descendants will use high priorities, and that node $N_2$ and its
-descendants will use lower priorities.  We have effectively split the
-range of possible priorities in two.  If several such heuristics fire
-in sequence, we can easily split the priority range \index{priority
-range splitting} in two enough times that no significant bits remain,
-and the search begins to fail for lack of meaningful priorities to
-assign.  The solution is to use arbitrary-precision priorities,
-i.e. bitvector priorities.
-
-To assign a bitvector priority, two methods are available.  The
-first is to obtain a pointer to the priority field using  \kw{CkPriorityPtr},
-and then manually set the bits using the bit-setting operations
-inherent to C.  To achieve this, one must know the format
-\index{bitvector format} of the
-bitvector, which is as follows: the bitvector is represented as an
-array of unsigned integers.  The most significant bit of the first
-integer contains the first bit of the bitvector.  The remaining bits
-of the first integer contain the next 31 bits of the bitvector.
-Subsequent integers contain 32 bits each.  If the size of the
-bitvector is not a multiple of 32, then the last integer contains 0
-bits for padding in the least-significant bits of the integer.
-
-The second way to assign priorities is only useful for those who are
-using the priority range-splitting\index{priority range splitting}
-described above.  The root of the tree is assigned the null
-priority-string.  Each child is assigned its parent's priority with
-some number of bits concatenated.  The net effect is that the entire
-priority of a branch is within a small epsilon of the priority of its
-root.
-
-It is possible to utilize unprioritized messages, integer priorities,
-and bitvector priorities in the same program.  The messages will be
-processed in roughly the following order\index{multiple priority types}:
+\subsubsection{Bitvector Prioritization}
+
+Bitvector priorities are arbitrary-length bit-strings representing fixed-point
+numbers in the range 0 to 1.  For example, the bit-string ``001001'' represents
+the number .001001\raisebox{-.5ex}{\scriptsize binary}.  As with integer
+priorities, higher numbers represent lower priorities.  However, bitvectors can
+be of arbitrary length, and hence the priority numbers they represent can be
+of arbitrary precision.
+
+Arbitrary-precision priorities\index{arbitrary-precision priorities} are often
+useful in AI search-tree applications.  Suppose we have a heuristic suggesting
+that tree node $N_1$ should be searched before tree node $N_2$.  We therefore
+designate that node $N_1$ and its descendants will use high priorities, and
+that node $N_2$ and its descendants will use lower priorities.  We have
+effectively split the range of possible priorities in two.  If several such
+heuristics fire in sequence, we can easily split the priority range
+\index{priority range splitting} in two enough times that no significant bits
+remain, and the search begins to fail for lack of meaningful priorities to
+assign.  The solution is to use arbitrary-precision priorities, i.e. bitvector
+priorities.
+
+To assign a bitvector priority, two methods are available.  The first is to
+obtain a pointer to the priority field using \kw{CkPriorityPtr}, and then
+manually set the bits using the bit-setting operations inherent to C.  To
+achieve this, one must know the format \index{bitvector format} of the
+bitvector, which is as follows: the bitvector is represented as an array of
+unsigned integers.  The most significant bit of the first integer contains the
+first bit of the bitvector.  The remaining bits of the first integer contain
+the next 31 bits of the bitvector.  Subsequent integers contain 32 bits each.
+If the size of the bitvector is not a multiple of 32, then the last integer
+contains 0 bits for padding in the least-significant bits of the integer.
+
+The second way to assign priorities is only useful for those who are using the
+priority range-splitting\index{priority range splitting} described above.  The
+root of the tree is assigned the null priority-string.  Each child is assigned
+its parent's priority with some number of bits concatenated.  The net effect is
+that the entire priority of a branch is within a small epsilon of the priority
+of its root.
+
+It is possible to utilize unprioritized messages, integer priorities, and
+bitvector priorities in the same program.  The messages will be processed in
+roughly the following order\index{multiple priority types}:
 
 \begin{itemize}
 
-\item Among messages enqueued with bitvector priorities, the
-messages are dequeued according to their priority.  The
-priority ``0000...'' is dequeued first, and ``1111...'' is
-dequeued last.
+\item Among messages enqueued with bitvector priorities, the messages are
+  dequeued according to their priority.  The priority ``0000...'' is dequeued
+  first, and ``1111...'' is dequeued last.
 
-\item Unprioritized messages are treated as if they had the
-priority ``1000...'' (which is the ``middle'' priority, it
-lies exactly halfway between ``0000...'' and ``1111...'').
+\item Unprioritized messages are treated as if they had the priority
+  ``1000...'' (which is the ``middle'' priority, it lies exactly halfway
+  between ``0000...'' and ``1111...'').
 
-\item Integer priorities are converted to bitvector priorities.  They
-are normalized so that the integer priority of zero is converted to
-``1000...'' (the ``middle'' priority).  To be more specific, the
-conversion is performed by adding 0x80000000 to the integer, and then
-treating the resulting 32-bit quantity as a 32-bit bitvector priority.
+\item Integer priorities are converted to bitvector priorities.  They are
+  normalized so that the integer priority of zero is converted to ``1000...''
+  (the ``middle'' priority).  To be more specific, the conversion is performed
+  by adding 0x80000000 to the integer, and then treating the resulting 32-bit
+  quantity as a 32-bit bitvector priority.
 
-\item Among messages with the same priority, messages are
-dequeued in FIFO order or LIFO order, depending upon which
-queuing strategy was used.
+\item Among messages with the same priority, messages are dequeued in FIFO
+  order or LIFO order, depending upon which queuing strategy was used.
 
 \end{itemize}
 
-Additionally, {\sl long integer priorities} can be specified by the {\em L} strategy. 
+Additionally, {\sl long integer priorities} can be specified by the {\em L}
+strategy.
 
-A final reminder about prioritized execution: \charmpp\ processes
-messages in {\it roughly} the order you specify; it never guarantees
-that it will deliver the messages in {\it precisely} the
-order\index{message delivery order} you specify. Thus, the correctness
-of your program should never depend on the order in which the runtime
-delivers messages. However, it makes a serious attempt to be
-``close'', so priorities can strongly affect the efficiency of your
-program.
+A final reminder about prioritized execution: \charmpp\ processes messages in
+{\it roughly} the order you specify; it never guarantees that it will deliver
+the messages in {\it precisely} the order\index{message delivery order} you
+specify. Thus, the correctness of your program should never depend on the order
+in which the runtime delivers messages. However, it makes a serious attempt to
+be ``close'', so priorities can strongly affect the efficiency of your program.
 
 \subsubsection{Skipping the Queue}
 
 Some operations that one might want to perform are sufficiently
 latency-sensitive that they should never wait in line behind other
-messages. The \charmpp\ runtime offers two attributes for entry
-methods, {\kw expedited} and {\kw immediate}, to serve these
-needs. For more information on these attributes, see
-Section~\ref{attributes} and the example in {\tt
+messages. The \charmpp\ runtime offers two attributes for entry methods, {\kw
+  expedited} and {\kw immediate}, to serve these needs. For more information on
+these attributes, see Section~\ref{attributes} and the example in {\tt
   charm/pgms/charm++/megatest/immediatering.C}.
diff --git a/examples/charm++/nodeHelper/fft-trans/Makefile b/examples/charm++/nodeHelper/fft-trans/Makefile
deleted file mode 100644 (file)
index bd38a6b..0000000
+++ /dev/null
@@ -1,57 +0,0 @@
-USEROPTS       = -O3
-CHARMDIR = ../../..
-
-CHARMINC = $(CHARMDIR)/include
-CHARMLIB = $(CHARMDIR)/lib
-OPTS = -I$(CHARMINC) $(USEROPTS)
-CHARMC = $(CHARMDIR)bin/charmc $(OPTS)
-
-CC=mpicxx
-LIBS = -lfftw3 -lm
-SLIBS = -lfftw3f -lm
-
-OBJS = fft1d.o
-
-all: fft1d fft_ref fft1d.prj fft_bench
-
-#for single precision add -DSINGLE_PRECISION to OPTS on the command line
-singlePrecision: fft1d_s
-
-
-fft_bench: fft_bench.o
-       ${CC} fft_bench.o -o fft_bench $(LIBS)
-
-fft_bench.o: fft_bench.cpp
-       ${CC} -c fft_bench.cpp $(INC)
-
-fft1d: $(OBJS)
-       $(CHARMC) -language charm++ -o fft1d $(OBJS) $(LIBS) -module NodeHelper
-
-fft1d_s: $(OBJS)
-       $(CHARMC) -language charm++ -o fft1d_s $(OBJS) $(SLIBS)  -module NodeHelper
-
-projections: fft1d.prj
-fft1d.prj: $(OBJS)
-       $(CHARMC) -language charm++ -tracemode projections $(LIBS) -lz -o fft1d.prj $(OBJS)  -module NodeHelper
-
-summary: $(OBJS)
-       $(CHARMC) -language charm++ -tracemode summary $(LIBS) -o fft1d.sum $(OBJS)
-
-fft1d.decl.h: fft1d.ci
-       $(CHARMC)  fft1d.ci
-
-fft_ref: fft_ref.o
-       ${CC} fft_ref.o -o fft_ref -L/expand/home/arya/fftw-3.3/lib -lfftw3_mpi -lfftw3 -lm
-
-fft_ref.o: fft_ref.cpp
-       ${CC} -c fft_ref.cpp -I/expand/home/arya/fftw-3.3/include
-
-cleanproj:
-       rm -f *.log *.sts *.projrc
-
-clean:
-       rm -f *.decl.h *.def.h conv-host *.o fft1d fft1d.prj fft_bench charmrun fft_ref *~
-
-fft1d.o: fft1d.C fft1d.decl.h
-       $(CHARMC) -c fft1d.C
-
diff --git a/examples/charm++/nodeHelper/fft-trans/README.nodelib b/examples/charm++/nodeHelper/fft-trans/README.nodelib
deleted file mode 100644 (file)
index 8ccca55..0000000
+++ /dev/null
@@ -1,21 +0,0 @@
-Simple test application to try node level parallelization of FFT using
-threads in the NodeHelper library compared to a standard charm level
-decomposition.
-
-The main distinction here is node level shared memory awareness.  The
-NodeHelper library will exploit that whereas the charm decomposition
-will not.  
-
-We are also interested in getting a sense of at what level of
-granularity it is worth trying to do node level parallelism.  It is
-expected that a lower bound on practically parallelizable grainsize
-will exist as determined by cache locality and function call overhead.
-Below that threshold it won't be worth trying to explicitely exploit
-the available data parallelism.  Hopefully this number is larger than
-a few fft lines with tens to hundreds of points.
-
-For simplicity we do a single FFT for complex to complex using single
-precision as this is typical for NAMD.  A more advanced version would
-do real to complex, but that is more pain than necessary to
-demonstrate utility.
-
diff --git a/examples/charm++/nodeHelper/fft-trans/fft1d.C b/examples/charm++/nodeHelper/fft-trans/fft1d.C
deleted file mode 100644 (file)
index c7c9d25..0000000
+++ /dev/null
@@ -1,289 +0,0 @@
-#include "fft1d.decl.h"
-#include <fftw3.h>
-#include <limits>
-#include "fileio.h"
-#include "NodeHelperAPI.h"
-
-#define TWOPI 6.283185307179586
-
-/*readonly*/ CProxy_Main mainProxy;
-/*readonly*/ int numChunks;
-/*readonly*/ int numTasks;
-/*readonly*/ uint64_t N;
-static CmiNodeLock fft_plan_lock;
-#include "fftmacro.h"
-
-CProxy_FuncNodeHelper nodeHelperProxy;
-/** called by initnode once per node to support node level locking for
-    fftw plan create/destroy operations */
-#define NODEHELPER_MODE NODEHELPER_STATIC 
-
-extern "C" void doCalc(int first,int last, void *result, int paramNum, void * param)
-{
-  //result=first;
-  for(int i=first; i<=last; i++)
-    fft_execute(((fft_plan*)param)[i]);
-}
-
-void initplanlock ()
-
-{
-  fft_plan_lock=CmiCreateLock();
-}
-
-struct fftMsg : public CMessage_fftMsg {
-  int source;
-  fft_complex *data;
-};
-
-struct Main : public CBase_Main {
-  double start;
-  CProxy_fft fftProxy;
-
-  Main(CkArgMsg* m) {
-    numChunks = atoi(m->argv[1]); //#1D partitions
-    N = atol(m->argv[2]); //matrix size
-    if(m->argc>=4)
-      numTasks = atol(m->argv[3]); //the number of tasks that 1D partition is splitted into
-    else
-      numTasks = CmiMyNodeSize();  //default to 1/core
-    delete m;
-    
-    mainProxy = thisProxy;
-
-    /* how to nodify this computation? */
-    /* We make one block alloc per chare and divide the work evenly
-       across the number of threads.
-     * cache locality issues... 
-
-     *       The NodeHelper scheme presents a problem in cache
-     *       ignorance.  We push tasks into the queue as the remote
-     *       message dependencies are met, however the execution of
-     *       dequeued tasks performance will have significant cache
-     *       limitations obfuscated to our scheduler.  Our helper
-     *       threads will block while fetching data into cache local
-     *       to the thread.  If we only have 1 thread per core, we
-     *       have no way to self overlap those operations.  This
-     *       implies that there are probably conditions under which
-     *       more than one nodehelper thread per core will result in
-     *       better performance.  A natural sweet spot for these
-     *       should be explored in the SMT case where one thread per
-     *       SMT will allow for natural overlap of execution based on
-     *       cache availability, as controlled by the OS without
-     *       additional pthread context switching overhead.  A further
-     *       runtime based virtualized overthreading may provide
-     *       further benefits depending on thread overhead.
-     */
-    if (N % numChunks != 0)
-      CkAbort("numChunks not a factor of N\n");
-
-    // Construct an array of fft chares to do the calculation
-    fftProxy = CProxy_fft::ckNew(numChunks);
-
-    // Construct a nodehelper to do the calculation
-    //nodeHelperProxy = NodeHelper_Init(NODEHELPER_MODE, numTasks);
-    nodeHelperProxy = NodeHelper_Init();
-    
-    CkStartQD(CkIndex_Main::initDone((CkQdMsg *)0), &thishandle);
-  }
-
-  void initDone(CkQdMsg *msg){
-    delete msg;
-    startFFT();
-  }
-  
-  void startFFT() {
-    start = CkWallTimer();
-    // Broadcast the 'go' signal to the fft chare array
-    fftProxy.doFFT();
-  }
-
-  void doneFFT() {
-    double time = CkWallTimer() - start;
-    double gflops = 5 * (double)N*N * log2((double)N*N) / (time * 1000000000);
-    CkPrintf("chares: %d\ncores: %d\nTasks: %d\nsize: %ld\ntime: %f sec\nrate: %f GFlop/s\n",
-             numChunks, CkNumPes(), numTasks, N*N, time, gflops);
-
-    fftProxy.initValidation();
-  }
-
-  void printResidual(realType r) {
-    CkPrintf("residual = %g\n", r);
-    CkExit();
-  }
-
-};
-
-struct fft : public CBase_fft {
-  fft_SDAG_CODE
-
-  int iteration, count;
-  uint64_t n;
-  fft_plan *plan;
-  fft_plan p1;
-  fftMsg **msgs;
-  fft_complex *in, *out;
-  bool validating;
-  int nPerThread;
-  fft() {
-    __sdag_init();
-
-    validating = false;
-
-    n = N*N/numChunks;
-
-    in = (fft_complex*) fft_malloc(sizeof(fft_complex) * n);
-    out = (fft_complex*) fft_malloc(sizeof(fft_complex) * n);
-    nPerThread= n/numTasks;
-    
-    int length[] = {N};
-    
-    /** Basically, we want to parallelize the following fftw function call
-     * which is to do fft on each row of a 2D array with #rows=N/numChunks, #cols=N;
-     * 1. create a plan: singlePlan = fft_plan_many_dft(1, len, N/numChunks, out, len,
-     *                                                  1, N, out, len, 1,
-     *                                                  N, FFTW_FORWARD, FFTW_ESTIMATE)
-     * where len is defined as int len[]={N}
-     * 2. execute the plan: fft_execute(singlePlan).
-     * 
-     * It's not a loop, we transformed it into a loop with N/numTasks plans so that 
-     * each task execute one plan. Each plan has N/numChunks/numTasks rows for fftw
-     * processing.
-     */
-    
-    CmiLock(fft_plan_lock);
-    size_t offset=0;
-    plan= new fft_plan[numTasks];
-    for(int i=0; i < numTasks; i++,offset+=nPerThread)
-      {
-       /* ??? should the dist be nPerThread as the fft is performed as 1d of length nPerThread?? */
-       //plan[i] = fft_plan_many_dft(1, length, N/numChunks/numTasks, out+offset, length, 1, N/numTasks,
-    //                        out+offset, length, 1, N/numTasks, FFTW_FORWARD, FFTW_ESTIMATE);
-    
-    plan[i] = fft_plan_many_dft(1, length, N/numChunks/numTasks, out+offset, length, 1, N,
-                            out+offset, length, 1, N, FFTW_FORWARD, FFTW_ESTIMATE);                        
-      }
-    CmiUnlock(fft_plan_lock);
-    
-    srand48(thisIndex);
-    for(int i = 0; i < n; i++) {
-      in[i][0] = drand48();
-      in[i][1] = drand48();
-    }
-
-    msgs = new fftMsg*[numChunks];
-    for(int i = 0; i < numChunks; i++) {
-      msgs[i] = new (n/numChunks) fftMsg;
-      msgs[i]->source = thisIndex;
-    }
-
-    // Reduction to the mainchare to signal that initialization is complete
-    //contribute(CkCallback(CkIndex_Main::startFFT(), mainProxy));
-  }
-
-  void sendTranspose(fft_complex *src_buf) {
-    // All-to-all transpose by constructing and sending
-    // point-to-point messages to each chare in the array.
-    for(int i = thisIndex; i < thisIndex+numChunks; i++) {
-      //  Stagger communication order to avoid hotspots and the
-      //  associated contention.
-      int k = i % numChunks;
-      for(int j = 0, l = 0; j < N/numChunks; j++)
-        memcpy(msgs[k]->data[(l++)*N/numChunks], src_buf[k*N/numChunks+j*N], sizeof(fft_complex)*N/numChunks);
-
-      // Tag each message with the iteration in which it was
-      // generated, to prevent mis-matched messages from chares that
-      // got all of their input quickly and moved to the next step.
-      CkSetRefNum(msgs[k], iteration);
-      thisProxy[k].getTranspose(msgs[k]);
-      // Runtime system takes ownership of messages once they're sent
-      msgs[k] = NULL;
-    }
-  }
-
-  void applyTranspose(fftMsg *m) {
-    int k = m->source;
-    for(int j = 0, l = 0; j < N/numChunks; j++)
-      for(int i = 0; i < N/numChunks; i++) {
-        out[k*N/numChunks+(i*N+j)][0] = m->data[l][0];
-        out[k*N/numChunks+(i*N+j)][1] = m->data[l++][1];
-      }
-
-    // Save just-received messages to reuse for later sends, to
-    // avoid reallocation
-    delete msgs[k];
-    msgs[k] = m;
-    msgs[k]->source = thisIndex;
-  }
-
-  void twiddle(realType sign) {
-    realType a, c, s, re, im;
-
-    int k = thisIndex;
-    for(int i = 0; i < N/numChunks; i++)
-      for(int j = 0; j < N; j++) {
-        a = sign * (TWOPI*(i+k*N/numChunks)*j)/(N*N);
-        c = cos(a);
-        s = sin(a);
-
-        int idx = i*N+j;
-
-        re = c*out[idx][0] - s*out[idx][1];
-        im = s*out[idx][0] + c*out[idx][1];
-        out[idx][0] = re;
-        out[idx][1] = im;
-      }
-  }
-  void fftHelperLaunch()
-  {
-    //kick off thread computation
-    //FuncNodeHelper *nth = nodeHelperProxy[CkMyNode()].ckLocalBranch();
-    //nth->parallelizeFunc(doCalc, numTasks, numTasks, thisIndex, numTasks, 1, 1, plan, 0, NULL);
-    double ffttime = CkWallTimer();
-    NodeHelper_Parallelize(doCalc, 1, plan, numTasks, 0, numTasks-1);    
-    CkPrintf("FFT time: %.3f (ms)\n", (CkWallTimer()-ffttime)*1e3);
-  }
-
-  void initValidation() {
-    memcpy(in, out, sizeof(fft_complex) * n);
-
-    validating = true;
-    int length[] = {N};
-    CmiLock(fft_plan_lock);
-    size_t offset=0;
-    plan= new fft_plan[numTasks];
-    for(int i=0; i < numTasks; i++,offset+=nPerThread)
-      {
-       //      fft_destroy_plan(plan[i]);
-       //plan[i] = fft_plan_many_dft(1, length, N/numChunks/numTasks, out+offset, length, 1, N/numTasks,
-    //                        out+offset, length, 1, N/numTasks, FFTW_BACKWARD, FFTW_ESTIMATE);
-    plan[i] = fft_plan_many_dft(1, length, N/numChunks/numTasks, out+offset, length, 1, N,
-                            out+offset, length, 1, N, FFTW_BACKWARD, FFTW_ESTIMATE);
-      }
-    CmiUnlock(fft_plan_lock);
-    contribute(CkCallback(CkIndex_Main::startFFT(), mainProxy));
-  }
-
-  void calcResidual() {
-    double infNorm = 0.0;
-
-    srand48(thisIndex);
-    for(int i = 0; i < n; i++) {
-      out[i][0] = out[i][0]/(N*N) - drand48();
-      out[i][1] = out[i][1]/(N*N) - drand48();
-
-      double mag = sqrt(pow(out[i][0], 2) + pow(out[i][1], 2));
-      if(mag > infNorm) infNorm = mag;
-    }
-
-    double r = infNorm / (std::numeric_limits<double>::epsilon() * log((double)N * N));
-
-    CkCallback cb(CkReductionTarget(Main, printResidual), mainProxy);
-    contribute(sizeof(double), &r, CkReduction::max_double, cb);
-  }
-
-  fft(CkMigrateMessage* m) {}
-  ~fft() {}
-};
-
-#include "fft1d.def.h"
diff --git a/examples/charm++/nodeHelper/fft-trans/fft1d.ci b/examples/charm++/nodeHelper/fft-trans/fft1d.ci
deleted file mode 100644 (file)
index 197b5e7..0000000
+++ /dev/null
@@ -1,56 +0,0 @@
-mainmodule fft1d {
-  include "fftmacro.h";
-  readonly CProxy_Main mainProxy;
-  readonly CProxy_FuncNodeHelper nodeHelperProxy;
-  readonly int numChunks;
-  readonly int numTasks;
-  readonly uint64_t N;
-  initnode void initplanlock();
-  message fftMsg {
-    fft_complex data[];
-  };
-  mainchare Main {
-    entry Main(CkArgMsg *m);
-    entry void initDone(CkQdMsg *m);
-    entry void startFFT();
-    entry void doneFFT();
-
-    entry [reductiontarget] void printResidual(realType residual);
-  };
-
-  array [1D] fft {
-    entry fft();
-    entry void getTranspose(fftMsg *m);
-    entry void initValidation();
-    entry void doFFT() {
-      for(iteration = 0; iteration < 3; ++iteration) {
-        atomic "transpose" {
-          if(thisIndex == 0)
-            CkPrintf("TRANSPOSING\n");
-          sendTranspose(iteration == 0 ? in : out);
-        }
-
-        for(count = 0; count < numChunks; ++count)
-          when getTranspose[iteration] (fftMsg *m) atomic {
-            applyTranspose(m);
-          }
-
-        if (iteration < 2) atomic "compute" {
-         fftHelperLaunch();
-          if(iteration == 0)
-              twiddle(validating ? 1 : -1);
-        }
-      }        
-      atomic {
-        if(!validating)
-          contribute(CkCallback(CkIndex_Main::doneFFT(), mainProxy));
-        else {
-          char filename[80];
-          sprintf(filename, "%d-%ld.dump%d", numChunks, N, thisIndex);
-          writeCommFile(n, in, filename);
-          calcResidual();
-        }
-      }
-    };
-  };
-};
diff --git a/examples/charm++/nodeHelper/fft-trans/fft_bench.cpp b/examples/charm++/nodeHelper/fft-trans/fft_bench.cpp
deleted file mode 100644 (file)
index 240625b..0000000
+++ /dev/null
@@ -1,67 +0,0 @@
-//#include <stdio.h>
-//#include <iostream>
-
-#include <mpi.h>
-#include <fftw3.h>
-#include <cstdlib>
-#include <limits>
-#include <cmath>
-
-using namespace std;
-
-int main(int argc, char *argv[]) 
-{
-  MPI_Init(&argc, &argv);
-
-  long N = atol(argv[1]);
-  long n = N * N;
-
-  fftw_complex *data;
-  fftw_plan p, q;
-
-  data = (fftw_complex*) fftw_malloc(sizeof(fftw_complex) * n);
-
-  int length[] = {N};
-  p = fftw_plan_many_dft(1, length, N, data, length, 1, N,
-      data, length, 1, N, FFTW_FORWARD, FFTW_ESTIMATE); 
-  q = fftw_plan_many_dft(1, length, N, data, length, 1, N,
-      data, length, 1, N, FFTW_BACKWARD, FFTW_ESTIMATE);
-
-  srand48(0);
-  for(int i = 0; i<n; i++) {
-    data[i][0] = drand48();
-    data[i][1] = drand48();
-  }
-
-  double t1, t2; 
-  t1 = MPI_Wtime(); 
-
-  fftw_execute(p);
-
-  t2 = MPI_Wtime() - t1; 
-  printf( "On %ld elements, Elapsed time is %f with %f Gflop/s\n", n, t2, 5*(double)n*log2((double)n)/(t2*1000000000)); 
-
-  fftw_execute(q);
-
-  double infNorm = 0.0;
-
-  srand48(0);
-  for(int i=0; i<n; i++) {
-    data[i][0] = data[i][0]/N - drand48();
-    data[i][1] = data[i][1]/N - drand48();
-
-    double mag = sqrt(pow(data[i][0],2) + pow(data[i][1], 2));
-    if(mag > infNorm) infNorm = mag;
-  } 
-
-  double r = infNorm / (std::numeric_limits<double>::epsilon() * log((double)n));
-
-  printf("residual = %f\n", r);
-
-  fftw_destroy_plan(p);
-  fftw_destroy_plan(q);
-  fftw_free(data);
-
-  MPI_Finalize();
-  return 0;
-}
diff --git a/examples/charm++/nodeHelper/fft-trans/fft_ref.cpp b/examples/charm++/nodeHelper/fft-trans/fft_ref.cpp
deleted file mode 100644 (file)
index f3603cc..0000000
+++ /dev/null
@@ -1,76 +0,0 @@
-#include <mpi.h> 
-#include <cstdlib>
-#include "fftw3-mpi.h"
-#include <cmath>
-#include <limits>
-#include "fileio.h"
-
-using namespace std;
-
-int main(int argc, char *argv[]) {
-  int rank, size; 
-  MPI_Init(&argc, &argv);
-  MPI_Comm_size(MPI_COMM_WORLD, &size);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank); 
-
-  fftw_plan plan;
-  fftw_complex *data;
-
-  fftw_mpi_init();
-
-  if(rank == 0) {
-    if(argc != 2) {
-      printf("Usage: ./binary <N>\n");
-      MPI_Abort(MPI_COMM_WORLD,-1);
-    }
-  }
-
-  int N = atoi(argv[1]);
-
-  ptrdiff_t local_ni=N*N/size, local_i_start = N*N/size*rank;
-  ptrdiff_t local_no=local_ni, local_o_start = local_i_start;
-
-  int b_or_f = FFTW_BACKWARD;
-
-  ptrdiff_t alloc_local =  fftw_mpi_local_size_1d(N*N, MPI_COMM_WORLD,
-      b_or_f, FFTW_ESTIMATE, &local_ni, &local_i_start,
-      &local_no, &local_o_start);
-
-  data = fftw_alloc_complex(alloc_local);
-
-  plan = fftw_mpi_plan_dft_1d(N*N, data, data, MPI_COMM_WORLD, b_or_f, FFTW_ESTIMATE);
-
-  char filename[80];
-  sprintf(filename, "%d-%d.dump%d", size, N, rank);
-  readCommFile(data, filename);
-
-  fftw_execute(plan);
-
-  double infNorm = 0.0;
-  srand48(rank);
-  for(int i = 0; i < N*N/size; i++) {
-    data[i][0] = data[i][0]/(N*N) - drand48();
-    data[i][1] = data[i][1]/(N*N) - drand48();
-
-    double mag = sqrt(pow(data[i][0], 2) + pow(data[i][1], 2));
-    if(mag > infNorm) infNorm = mag;
-  }
-
-  double my_r = infNorm / (std::numeric_limits<double>::epsilon() * log((double)N * N));
-  double r;
-
-  MPI_Reduce(&my_r, &r, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-
-  if(rank == 0) {
-    if(r < 16)
-      printf("r = %g, PASS!\n",r);
-    else
-      printf("r = %g, FAIL\n",r);
-  }
-
-  fftw_destroy_plan(plan);
-
-  fftw_mpi_cleanup();
-  MPI_Finalize();      
-  return 0; 
-} 
diff --git a/examples/charm++/nodeHelper/fft-trans/fftmacro.h b/examples/charm++/nodeHelper/fft-trans/fftmacro.h
deleted file mode 100644 (file)
index b485067..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-//todo fix this is a more clever macro
-#ifdef SINGLE_PRECISION
-#define fft_complex fftwf_complex
-#define fft_malloc fftwf_malloc
-#define fft_free fftwf_free
-#define fft_plan_many_dft fftwf_plan_many_dft
-#define fft_destroy_plan fftwf_destroy_plan
-#define fft_execute fftwf_execute
-#define fft_plan fftwf_plan
-#define realType float
-#else
-#define fft_complex fftw_complex
-#define fft_malloc fftw_malloc
-#define fft_free fftw_free
-#define fft_plan_many_dft fftw_plan_many_dft
-#define fft_destroy_plan fftw_destroy_plan
-#define fft_execute fftw_execute
-#define fft_plan fftw_plan
-#define realType double
-#endif
diff --git a/examples/charm++/nodeHelper/fft-trans/fileio.h b/examples/charm++/nodeHelper/fft-trans/fileio.h
deleted file mode 100644 (file)
index 35b4c38..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-#include "fftmacro.h"
-void readCommFile(fft_complex *data, char *filename) {
-  FILE *pFile;
-  if(!(pFile = fopen (filename,"r"))) {
-    printf("File open failed\n");
-    return;
-  }
-
-  int l = 0;
-#ifdef SINGLE_PRECISION
-  while(fscanf (pFile, "%f %f", &data[l][0], &data[l][1]) != EOF) {l++;}
-#else
-  while(fscanf (pFile, "%lf %lf", &data[l][0], &data[l][1]) != EOF) {l++;}
-#endif
-
-  fclose(pFile);
-}
-
-void writeCommFile(int n, fft_complex *data, char *filename) {
-  FILE *pFile;
-  if(!(pFile = fopen (filename, "w"))) {
-    printf("File open for write failed\n");
-    return;
-  }
-
-  for(int l = 0; l < n; l++)
-
-    fprintf(pFile, "%.24f %.24f\n", data[l][0], data[l][1]);
-  fclose(pFile);
-}
diff --git a/examples/charm++/nodeHelper/fft-trans/test b/examples/charm++/nodeHelper/fft-trans/test
deleted file mode 100755 (executable)
index 04f5766..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-./charmrun +p4 ++local fft1d $1 $2
-mpirun -np $1 ./fft_ref $2
diff --git a/examples/charm++/nodeHelper/fft-trans/test.sh b/examples/charm++/nodeHelper/fft-trans/test.sh
deleted file mode 100755 (executable)
index 04f5766..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-./charmrun +p4 ++local fft1d $1 $2
-mpirun -np $1 ./fft_ref $2
diff --git a/examples/charm++/nodeHelper/simpleLoopBench/Makefile b/examples/charm++/nodeHelper/simpleLoopBench/Makefile
deleted file mode 100644 (file)
index 1205150..0000000
+++ /dev/null
@@ -1,43 +0,0 @@
-#NOTE: to compile the example, the NodeHelper.decl/def.h should exist
-USEROPTS=-O3 -g -lpthread -fopenmp
-CHARMDIR=../../..
-CHARMINC=$(CHARMDIR)/include
-OPTS=-I$(CHARMINC) $(USEROPTS)
-CHARMC=$(CHARMDIR)/bin/charmc $(OPTS)
-CHARMLIB=$(CHARMDIR)/lib
-
-
-all: hello
-
-hello: hello.o
-       $(CHARMC) -language charm++ -o hello hello.o  -module NodeHelper
-       $(CHARMC) -language charm++ -o hello.prj hello.o  -tracemode projections -module NodeHelper
-
-hello.decl.h: hello.ci
-       $(CHARMC)  hello.ci
-
-hello.o: hello.C hello.decl.h 
-       $(CHARMC) -c hello.C
-
-clean:
-       rm -f *.decl.h *.def.h conv-host *.o hello hello.prj charmrun *.log *.sum *.sts
-
-test: all
-       ./charmrun ./hello +p4 10
-
-bgtest: all
-       ./charmrun ./hello +p4 10 +x2 +y2 +z2 +cth1 +wth1
-
-module: $(CHARMLIB)/libmoduleNodeHelper.a
-
-$(CHARMLIB)/libmoduleNodeHelper.a: NodeHelper.o
-       $(CHARMC)  -o $(CHARMLIB)/libmoduleNodeHelper.a NodeHelper.o
-
-
-NodeHelper.decl.h: NodeHelper.ci
-       $(CHARMC)  NodeHelper.ci
-
-
-NodeHelper.o: NodeHelper.C NodeHelper.decl.h
-       $(CHARMC) -c NodeHelper.C
-
diff --git a/examples/charm++/nodeHelper/simpleLoopBench/hello.C b/examples/charm++/nodeHelper/simpleLoopBench/hello.C
deleted file mode 100644 (file)
index a144b93..0000000
+++ /dev/null
@@ -1,261 +0,0 @@
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include "hello.h"
-
-#include "hello.decl.h"
-
-#include <omp.h>
-
-#define TEST_REPEAT_TIMES 100
-
-CProxy_Main mainProxy;
-CProxy_TestInstance allTestsProxy;
-CProxy_FuncNodeHelper nodeHelperProxy;
-int totalElems; //the number of test instances
-int loopTimes;
-int numChunks;
-int runningMode;
-
-int threadNum; //number of threads to be used in non-SMP
-
-int cmpDFunc(const void *a, const void *b) {
-    double n1 = *(double *)a;
-    double n2 = *(double *)b;
-    if (n1<n2) return -1;
-    if (n1>n2) return 1;
-    return 0;
-}
-
-void work(int start, int end, void *result) {
-    int tmp=0;
-    for (int i=start; i<=end; i++) {
-        tmp+=(int)(sqrt(1+cos(i*1.57)));
-    }
-    *(int *)result = tmp;
-    
-   //CkPrintf("From rank[%d]: start=%d, end=%d, result=%d\n", CkMyRank(), start, end, tmp);
-   //fflush(stdout);
-}
-
-int openMPWork(int start, int end) {
-    int result = 0;
-    
-    #pragma omp parallel for reduction (+:result)
-    for(int i=start; i<=end; i++) {
-        result += (int)(sqrt(1+cos(i*1.57)));
-    }
-    
-    return result;
-}
-
-extern "C" void doCalc(int first, int last, void *result, int paramNum, void * param) {    
-    //double tstart = CkWallTimer();
-    
-       work(first, last, result);
-    
-       //tstart = CkWallTimer() - tstart;
-    //printf("start=%d, end=%d, time: %f,result: %d on proc %d\n",first, last, tstart,result,CkMyPe());
-}
-
-/*mainchare*/
-Main::Main(CkArgMsg* m) {
-    
-       //default values        
-    totalElems = 1;
-       numChunks = CkMyNodeSize();
-       loopTimes = 1000;
-       runningMode = NODEHELPER_USECHARM; 
-       
-    mainStep = 0;
-       numElemFinished = 0;
-       
-    curTestMode = 0;
-    
-    //process command line
-    if (m->argc >1 ){
-        processCommandLine(m->argc,m->argv);
-       }
-    else{              
-               CkPrintf("Usage: -t(#iterations) -c(#chunks) -a(#test instances) -m(running mode, 1 for use Charm threads; 2 for use pthreads )  -p(#threads)\n");
-       }
-    delete m;
-       
-    omp_set_num_threads(numChunks);    
-    
-       mainTimes = new double[TEST_REPEAT_TIMES];
-       memset(mainTimes, 0, sizeof(double)*TEST_REPEAT_TIMES);
-       
-       CkPrintf("Using NodeHelper Lib with mode: %d, nodesize=%d\n", runningMode, CkMyNodeSize());
-       CkPrintf("Testcase info: %d test instances where the loop iterates %d times, each work is partitioned into %d tasks\n", totalElems, loopTimes, numChunks);
-       
-       nodeHelperProxy = NodeHelper_Init(runningMode, threadNum);
-       //nodeHelperProxy = NodeHelper_Init();
-    mainProxy = thishandle;
-    
-       //create test instances
-    CProxy_cyclicMap myMap = CProxy_cyclicMap::ckNew();
-    CkArrayOptions opts(totalElems);
-    opts.setMap(myMap);
-    allTestsProxy = CProxy_TestInstance::ckNew(opts);
-
-    //serial version
-       int result;
-       double starttime, endtime;
-       for(int i=0; i<3; i++){
-               starttime = CkWallTimer();
-               work(0, loopTimes, &result);            
-               endtime = CkWallTimer();
-               CkPrintf("Calibration %d: the loop takes %.3f us with result %d\n", i+1,  (endtime-starttime)*1e6, result);
-       }
-       int results[5];
-       starttime = CkWallTimer();
-       for(int i=0; i<5; i++) work(0, loopTimes, results+i);
-       endtime = CkWallTimer();
-       double avgtime = (endtime-starttime)*1e6/5; //in the unit of us
-       CkPrintf("Calibration: avg time %.3f us of 5 consecutive runs, so a 100us-loop will iterate %d times\n", avgtime, (int)(loopTimes*100.0/avgtime));
-               
-    CmiSetCPUAffinity(0);
-    CkStartQD(CkIndex_Main::doTests((CkQdMsg *)0), &thishandle);
-};
-
-void Main::done(void) {
-    numElemFinished++;
-    if (numElemFinished < totalElems) {
-        return;
-    } else {
-               mainTimes[mainStep] = (CkWallTimer() - timestamp)*1e6;
-        mainStep++;
-        numElemFinished=0;
-        if (mainStep < TEST_REPEAT_TIMES) {
-                       doTests(NULL);
-            return;
-        }
-    }  
-    
-       //do some final output
-       allTestsProxy[0].reportSts();
-};
-
-void Main::exitTest(){
-       //do some final output
-       qsort(mainTimes, TEST_REPEAT_TIMES, sizeof(double), cmpDFunc);
-       double sum = 0.0;
-       for(int i=0; i<TEST_REPEAT_TIMES-3; i++) sum += mainTimes[i];
-       int maxi = TEST_REPEAT_TIMES;
-       CkPrintf("Global timestep info: avg time: %.3f [%.3f, %.3f, %.3f] (us)\n", sum/(maxi-3), mainTimes[0], mainTimes[maxi/2], mainTimes[maxi-1]);
-       
-    if(curTestMode == 0){
-           CkPrintf("Charm++ NodeHelper Test done\n\n");
-        curTestMode++;
-        mainStep = 0;
-        numElemFinished = 0;
-        doTests(NULL);
-    }else if(curTestMode == 1){        
-        CkPrintf("OpenMP Test done\n");
-        CkExit();
-    }
-}
-
-void Main::doTests(CkQdMsg *msg) {
-    delete msg;
-
-    //CkPrintf("===========Starting mainstep %d===========\n", mainStep);    
-
-    if(mainStep == 0){
-        if(curTestMode == 0){
-            CkPrintf("===Start NodeHelper Test===\n");
-        }else if(curTestMode == 1){
-            int numthds = 0;
-            int openmpid;
-            #pragma omp parallel private(openmpid)
-            {
-                openmpid = omp_get_thread_num();
-                if(openmpid == 0) numthds = omp_get_num_threads();
-            }
-            CkPrintf("===Start OpenMP Test with %d threads===\n", numthds);
-        }
-    }
-    
-       timestamp = CkWallTimer(); //record the start time of the whole test
-    for (int i=0; i<totalElems; i++) {
-        allTestsProxy[i].doTest(mainStep, curTestMode);
-        //allTestsProxy[8].doTest(mainStep, curTestMode);
-    }
-};
-
-void Main::processCommandLine(int argc,char ** argv) {
-    for (int i=0; i<argc; i++) {
-        if (argv[i][0]=='-') {
-            switch (argv[i][1]) {
-            case 't':
-                loopTimes = atoi(argv[++i]);
-                break;
-            case 'c':
-                numChunks = atoi(argv[++i]);
-                break;
-            case 'm':
-                runningMode = atoi(argv[++i]);
-                break;
-            case 'a':
-                totalElems = atoi(argv[++i]);
-                break;
-            case 'p':
-                threadNum = atoi(argv[++i]);
-                break;
-            }
-        }
-    }
-}
-
-
-TestInstance::TestInstance() {
-    CkPrintf("test case %d is created on proc %d node %d\n", thisIndex, CkMyPe(),CkMyNode());
-    
-       hasTest = 0; 
-       allTimes = new double[TEST_REPEAT_TIMES];
-       allResults = new int[TEST_REPEAT_TIMES];
-       memset(allTimes, 0, sizeof(double)*TEST_REPEAT_TIMES);
-       memset(allResults, 0, sizeof(int)*TEST_REPEAT_TIMES);
-}
-
-void TestInstance::doTest(int curstep, int curTestMode) {
-    //printf("On proc %d node %d, begin parallel execution for test case %d %dth iteration\n", CkMyPe(), CkMyNode(), thisIndex,flag);
-       hasTest = 1;
-       int result;
-       
-    double timerec = CkWallTimer();
-    
-    if(curTestMode == 0){
-           NodeHelper_Parallelize(doCalc, 0, NULL, numChunks, 0, loopTimes-1, 1, &result, NODEHELPER_INT_SUM);
-    }else if(curTestMode == 1){
-        result = openMPWork(0, loopTimes-1);
-    }
-    
-    allTimes[curstep]=(CkWallTimer()-timerec)*1e6;
-       allResults[curstep] = result;
-       
-    mainProxy.done();
-}
-
-void TestInstance::reportSts(){
-       if(hasTest){
-               //do sts output
-               qsort(allTimes, TEST_REPEAT_TIMES, sizeof(double), cmpDFunc);
-               double sum = 0.0;
-               for(int i=0; i<TEST_REPEAT_TIMES-3; i++) sum += allTimes[i];
-               
-               double avgResult = 0.0;
-               for(int i=0; i<TEST_REPEAT_TIMES; i++) avgResult += allResults[i];
-               avgResult /= TEST_REPEAT_TIMES;
-               
-               int maxi = TEST_REPEAT_TIMES;
-               CkPrintf("Test instance[%d]: result:%.3f, avg time: %.3f [%.3f, %.3f, %.3f] (us)\n",thisIndex, avgResult, sum/(maxi-3), allTimes[0], allTimes[maxi/2], allTimes[maxi-1]);           
-    }
-       
-       if(thisIndex == totalElems-1) mainProxy.exitTest();
-       else thisProxy[thisIndex+1].reportSts();
-}
-
-#include "hello.def.h"
-
diff --git a/examples/charm++/nodeHelper/simpleLoopBench/hello.ci b/examples/charm++/nodeHelper/simpleLoopBench/hello.ci
deleted file mode 100644 (file)
index 4271ef4..0000000
+++ /dev/null
@@ -1,26 +0,0 @@
-mainmodule hello {
-  readonly CProxy_Main mainProxy;
-  readonly CProxy_TestInstance allTestsProxy;
-  readonly CProxy_FuncNodeHelper nodeHelperProxy;
-  readonly int totalElems;
-  readonly int loopTimes;
-  readonly int numChunks;
-  readonly int runningMode;
-  
-  mainchare Main {
-    entry Main(CkArgMsg *m);
-    entry void done(void);
-    entry void doTests(CkQdMsg *msg);
-       entry void exitTest();
-  };
-
-  array [1D] TestInstance{
-    entry TestInstance();
-    entry void doTest(int curstep, int curTestMode);
-       entry void reportSts();
-   };
-
-  group cyclicMap : CkArrayMap{
-    entry cyclicMap();
-  };  
-};
diff --git a/examples/charm++/nodeHelper/simpleLoopBench/hello.h b/examples/charm++/nodeHelper/simpleLoopBench/hello.h
deleted file mode 100644 (file)
index fe625f5..0000000
+++ /dev/null
@@ -1,53 +0,0 @@
-#ifndef _HELLO_H
-#define _HELLO_H
-
-#include "charm++.h"
-#include "NodeHelperAPI.h"
-#include "hello.decl.h"
-#include <assert.h>
-
-class Main : public Chare {
-private:
-       int numElemFinished; //record the number of test instances finished in a timestep
-       double timestamp;
-       int mainStep; //the global counter of timesteps
-       double *mainTimes; //record each timestep from test initiation to test finish (i.e. from the point of main)
-    
-    int curTestMode; //0: nodeHelper; 1: OpenMP
-
-public:
-    Main(CkArgMsg* m) ;
-    void done(void);
-       void exitTest();
-    void doTests(CkQdMsg *msg);
-    void processCommandLine(int argc,char ** argv);
-};
-
-class TestInstance : public CBase_TestInstance {
-       int hasTest; //used for reporting statistics
-       
-    double *allTimes; //record time taken for each timestep
-       int *allResults; //record the result of each timestep
-       
-public:
-    TestInstance();
-    ~TestInstance() {
-               delete [] allTimes;
-               delete [] allResults;
-       }
-    TestInstance(CkMigrateMessage *m) {}
-    void doTest(int curstep, int curTestMode);
-       void reportSts();
-};
-
-class cyclicMap : public CkArrayMap {
-public:
-    int procNum(int, const CkArrayIndex &idx) {
-        int index = *(int *)idx.data();
-        int nid = (index/CkMyNodeSize())%CkNumNodes();
-        int rid = index%CkMyNodeSize();
-        return CkNodeFirst(nid)+rid;
-    }
-};
-
-#endif
diff --git a/examples/charm++/prio/Makefile b/examples/charm++/prio/Makefile
new file mode 100644 (file)
index 0000000..dc1941e
--- /dev/null
@@ -0,0 +1,16 @@
+# modify the path of CHARMC
+CHARMC ?= ../../../bin/charmc $(OPTS)
+
+all: pgm
+
+pgm: pgm.o
+       $(CHARMC) pgm.o  -o pgm -language charm++
+
+pgm.o : pgm.C pgm.def.h
+       $(CHARMC) -c pgm.C
+
+pgm.decl.h pgm.def.h: pgm.ci
+       $(CHARMC)  pgm.ci
+
+clean:
+       rm -f pgm pgm_prof *.o conv-host *.decl.h *.def.h *~ charmrun *.log *.sts core charmrun.exe pgm.exe pgm.pdb pgm.ilk
diff --git a/examples/charm++/prio/pgm.C b/examples/charm++/prio/pgm.C
new file mode 100644 (file)
index 0000000..37cd5c7
--- /dev/null
@@ -0,0 +1,57 @@
+#include "prio.decl.h"
+
+/*readonly*/ CProxy_Main mainProxy;
+
+struct TestMsg : public CMessage_TestMsg { };
+
+struct Main : CBase_Main {
+  int numToSend;
+
+  Main(CkArgMsg* m)
+    : numToSend(30)
+  {
+    delete m;
+
+    mainProxy = thisProxy;
+
+    CProxy_Chare1 c1 = CProxy_Chare1::ckNew();
+
+    for (int i = 0; i < numToSend; i += 3) {
+      CkEntryOptions opts;
+      opts.setPriority(-1);
+      c1.prioMarshalling(10, &opts);
+
+      TestMsg* msg = new (8*sizeof(int)) TestMsg;
+      *(int*)CkPriorityPtr(msg) = -2;
+      CkSetQueueing(msg, CK_QUEUEING_IFIFO);
+      c1.prioMessage(msg);
+
+      c1.normalPrio(20);
+    }
+  }
+
+  void finished() {
+    if (--numToSend == 0) CkExit();
+  }
+};
+
+struct Chare1 : CBase_Chare1 {
+  Chare1() {
+    CkPrintf("Chare: created\n");
+  }
+  void prioMarshalling(int test) {
+    CkPrintf("prioMarshalling arrived\n");
+    mainProxy.finished();
+  }
+  void prioMessage(TestMsg* msg) {
+    CkPrintf("prioMessage arrived\n");
+    mainProxy.finished();
+    delete msg;
+  }
+  void normalPrio(int test2) {
+    CkPrintf("normalPrio arrived\n");
+    mainProxy.finished();
+  }
+};
+
+#include "prio.def.h"
diff --git a/examples/charm++/prio/pgm.ci b/examples/charm++/prio/pgm.ci
new file mode 100644 (file)
index 0000000..f23b502
--- /dev/null
@@ -0,0 +1,17 @@
+mainmodule prio {
+  readonly CProxy_Main mainProxy;
+
+  message TestMsg;
+
+  mainchare Main {
+    entry Main(CkArgMsg*);
+    entry void finished();
+  }
+
+  chare Chare1 {
+    entry Chare1();
+    entry void prioMarshalling(int test);
+    entry void prioMessage(TestMsg* msg);
+    entry void normalPrio(int test2);
+  }
+}
diff --git a/examples/charm++/sync_square/Makefile b/examples/charm++/sync_square/Makefile
new file mode 100644 (file)
index 0000000..1045571
--- /dev/null
@@ -0,0 +1,27 @@
+CHARMC = ../../../bin/charmc
+BINARY = sync_square
+
+CHARMCFLAGS = $(OPTS)
+CHARMCLINKFLAGS = -language charm++
+TESTFLAGS = $(TESTOPTS)
+
+%.o: %.cc
+
+all: $(BINARY)
+$(BINARY): $(patsubst %.cc,%.o,$(wildcard *.cc))
+       $(CHARMC) $(CHARMCLINKFLAGS) -o $@ $+ 
+
+.SECONDARY: $(patsubst %.cc,%.decl.h,$(wildcard *.cc))
+.SECONDARY: $(patsubst %.cc,%.def.h,$(wildcard *.cc))
+
+%.o: %.cc %.decl.h %.def.h
+       $(CHARMC) $(CHARMCFLAGS) $<
+
+%.decl.h %.def.h: %.ci
+       $(CHARMC) $(CHARMCFLAGS) $<
+
+test: $(BINARY)
+       ./charmrun ./$(BINARY) $(TESTFLAGS)
+
+clean:
+       rm -f *.o *.decl.h *.def.h charmrun $(BINARY)
diff --git a/examples/charm++/sync_square/sync_square.cc b/examples/charm++/sync_square/sync_square.cc
new file mode 100644 (file)
index 0000000..176f2bb
--- /dev/null
@@ -0,0 +1,26 @@
+#include "sync_square.h"
+#include <stdlib.h>
+
+Driver::Driver(CkArgMsg* args) {
+    int value = 10;
+    if (args->argc > 1) value = strtol(args->argv[1], NULL, 10);
+    delete args;
+    s = CProxy_Squarer::ckNew();
+    thisProxy.get_square(value);
+}
+
+void Driver::get_square(int value)
+{
+    int_message* square_message = s.square(value);
+    int square = square_message->value;
+    CkPrintf("%d^2 = %d\n", value, square);
+    CkFreeMsg(square_message);
+    CkExit();
+}
+
+int_message* Squarer::square(int x) {
+    return new int_message(x*x);
+}
+
+
+#include "sync_square.def.h"
diff --git a/examples/charm++/sync_square/sync_square.ci b/examples/charm++/sync_square/sync_square.ci
new file mode 100644 (file)
index 0000000..c39ce6b
--- /dev/null
@@ -0,0 +1,14 @@
+mainmodule sync_square {
+
+    message int_message;
+
+    mainchare Driver {
+        entry Driver(CkArgMsg*);
+        entry void get_square(int value);
+    }
+
+    chare Squarer {
+        entry Squarer(void);
+        entry [sync] int_message* square(int x);
+    }
+}
diff --git a/examples/charm++/sync_square/sync_square.h b/examples/charm++/sync_square/sync_square.h
new file mode 100644 (file)
index 0000000..8f15bc9
--- /dev/null
@@ -0,0 +1,22 @@
+#include "sync_square.decl.h"
+
+class int_message : public CMessage_int_message {
+    public:
+        int value;
+        int_message(int val) : value(val) {}
+};
+
+class Driver : public CBase_Driver {
+    public: 
+        Driver(CkArgMsg*);
+        void get_square(int value);
+        CProxy_Squarer s;
+};
+
+class Squarer : public CBase_Squarer {
+    public:
+        Squarer(void) {}
+        Squarer(CkMigrateMessage* m) {}
+        int_message* square(int x);
+};
+
index a83b337201001645083f43be9c78a09451f5f03f..943514e8dd48af149bea7580da4b041e702878c8 100644 (file)
@@ -60,6 +60,8 @@ static void qt_args_1(qt_t *rjb, void *u, void *t,
   write(2,"Never get here 2.\n",18);
 }
 
+qt_t *qt_args(qt_t *sp, void *u, void *t, qt_userf_t *userf, qt_only_t *only)  __attribute__((optimize(0)));
+
 qt_t *qt_args(qt_t *sp, void *u, void *t,
              qt_userf_t *userf, qt_only_t *only)
 {
index f1f035e00a87d9dab694954d5e0a0d9a5d6b3e8f..4c8146f139fd2175c4dd4373294314ff08b43307 100644 (file)
@@ -66,6 +66,8 @@ static void qt_args_1(qt_t *rjb, void *u, void *t,
   write(2,"Never get here 2.\n",18);
 }
 
+qt_t *qt_args(qt_t *sp, void *u, void *t, qt_userf_t *userf, qt_only_t *only)  __attribute__((optimize(0)));
+
 qt_t *qt_args(qt_t *sp, void *u, void *t,
              qt_userf_t *userf, qt_only_t *only)
 {
index 38c8b7a8f515d93dcb41703cc70030c54a92c244..78736467a916390d554d8ed9011403f3482cd3b7 100644 (file)
@@ -390,7 +390,8 @@ static void send_multi_done(void *data)
 }
 
 /* The machine specific send function */
-static CmiCommHandle MachineSpecificSendForDCMF(int destNode, int size, char *msg, int mode) {
+static CmiCommHandle MachineSpecificSendForDCMF(int destPE, int size, char *msg, int mode) {
+    int destNode = CmiNodeOf(destPE);
     SMSG_LIST *msg_tmp = smsg_allocate(); //(SMSG_LIST *) malloc(sizeof(SMSG_LIST));
     //msg_tmp->destpe = destNode;
     //msg_tmp->size = size;
index 2b33f9369075f9119f0a69e410db01253ee7e9df..6d7525285a5b8d94dcc7487d5fbcad7cbea66815 100644 (file)
@@ -1651,8 +1651,9 @@ inline void LrtsPrepareEnvelope(char *msg, int size)
     CMI_SET_CHECKSUM(msg, size);
 }
 
-CmiCommHandle LrtsSendFunc(int destNode, int size, char *msg, int mode)
+CmiCommHandle LrtsSendFunc(int destPE, int size, char *msg, int mode)
 {
+    int destNode = CmiNodeOf(destPE);
     gni_return_t        status  =   GNI_RC_SUCCESS;
     uint8_t tag;
     CONTROL_MSG         *control_msg_tmp;
@@ -1710,6 +1711,8 @@ CmiCommHandle LrtsSendFunc(int destNode, int size, char *msg, int mode)
     return 0;
 }
 
+#if 0
+// this is no different from the common code
 void LrtsSyncListSendFn(int npes, int *pes, int len, char *msg)
 {
   int i;
@@ -1776,6 +1779,7 @@ void LrtsFreeListSendFn(int npes, int *pes, int len, char *msg)
     CmiFree(msg);
 #endif
 }
+#endif
 
 static void    PumpDatagramConnection();
 static      int         event_SetupConnect = 111;
@@ -2280,7 +2284,7 @@ static void PumpNetworkSmsg()
     }   //end while GetEvent
     if(status == GNI_RC_ERROR_RESOURCE)
     {
-        printf("charm> Please use +useRecvQueue 204800 in your command line, if the error comes again, increase this number\n");  
+        printf("charm> Please use +useRecvQueue %d in your command line, if the error comes again, increase this number\n", REMOTE_QUEUE_ENTRIES*2);
         GNI_RC_CHECK("Smsg_rx_cq full", status);
     }
 }
index 999c4dd97c71f338a0e4a631050f1a8d92a3dff7..0164bfea301ffdcc566b07a07e3f014f0cf9fbe0 100755 (executable)
@@ -412,7 +412,8 @@ static INLINE_KEYWORD void lapiSendFn(int destNode, int size, char *msg, scompl_
     MACHSTATE(2,"} lapiSendFn end");
 }
 
-static CmiCommHandle MachineSendFuncForLAPI(int destNode, int size, char *msg, int mode) {
+static CmiCommHandle MachineSendFuncForLAPI(int destPE, int size, char *msg, int mode) {
+    int destNode = CmiNodeOf(destPE);
     scompl_hndlr_t *shdlr = NULL;
     void *sinfo = NULL;
 
index ca28aeb565841d3aa58b1e85374af1867682fbcc..4e6024be28043b2443fcba4782918573a14c0b7f 100644 (file)
@@ -1,3 +1,4 @@
 #define __FAULT__       1
 #define _FAULT_CAUSAL_      1
 #define CMK_CHARE_USE_PTR   1
+#define CMK_MESSAGE_LOGGING 1
index 79abe68e2d4087ddb6b2d68ae705cc59f4b645f8..b0e83053382f0dfcf7bfcb8c3f1db3f43eb202d8 100644 (file)
@@ -1,3 +1,4 @@
 #define __FAULT__              1
 #define _FAULT_MLOG_           1
 #define CMK_CHARE_USE_PTR      1
+#define CMK_MESSAGE_LOGGING 1
index bec29d0e77b34e6b317f69d1a3f6ee491c2000c5..ebfba7f93094bf427ed6a582d45b846435cf086e 100644 (file)
@@ -332,7 +332,7 @@ static CmiNodeLock  sendMsgBufLock = NULL;        /* for sendMsgBuf */
 #endif
 /* =====End of Declarations of Machine Specific Variables===== */
 
-#if CMK_MEM_CHECKPOINT
+#if CMK_MEM_CHECKPOINT || CMK_MESSAGE_LOGGING
 #define FAIL_TAG   1200
 int num_workpes, total_pes;
 int *petorank = NULL;
@@ -357,38 +357,12 @@ static int SendMsgBuf();
 static  void EnqueueMsg(void *m, int size, int node, int mode);
 #endif
 
-/* The machine-specific send function */
-static CmiCommHandle MachineSpecificSendForMPI(int destNode, int size, char *msg, int mode);
-#define LrtsSendFunc MachineSpecificSendForMPI
-
-/* ### Beginning of Machine-startup Related Functions ### */
-static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNodeID);
-#define LrtsInit MachineInitForMPI
-
-static void MachinePreCommonInitForMPI(int everReturn);
-static void MachinePostCommonInitForMPI(int everReturn);
-#define LrtsPreCommonInit MachinePreCommonInitForMPI
-#define LrtsPostCommonInit MachinePostCommonInitForMPI
-/* ### End of Machine-startup Related Functions ### */
-
-/* ### Beginning of Machine-running Related Functions ### */
-static void AdvanceCommunicationForMPI(int whenidle);
-#define LrtsAdvanceCommunication AdvanceCommunicationForMPI
-
-static void DrainResourcesForMPI(); /* used when exit */
-#define LrtsDrainResources DrainResourcesForMPI
-
-static void MachineExitForMPI();
-#define LrtsExit MachineExitForMPI
 /* ### End of Machine-running Related Functions ### */
 
 /* ### Beginning of Idle-state Related Functions ### */
 void CmiNotifyIdleForMPI(void);
 /* ### End of Idle-state Related Functions ### */
 
-static void MachinePostNonLocalForMPI();
-#define LrtsPostNonLocal MachinePostNonLocalForMPI
-
 /* =====End of Declarations of Machine Specific Functions===== */
 
 /**
@@ -463,7 +437,7 @@ static CmiCommHandle MPISendOneMsg(SMSG_LIST *smsg) {
 #else
 /* branch not using MPI_POST_RECV */
 
-#if CMK_MEM_CHECKPOINT
+#if CMK_MEM_CHECKPOINT || CMK_MESSAGE_LOGGING
        dstrank = petorank[node];
 #else
        dstrank=node;
@@ -495,9 +469,10 @@ static CmiCommHandle MPISendOneMsg(SMSG_LIST *smsg) {
     return (CmiCommHandle) &(smsg->req);
 }
 
-static CmiCommHandle MachineSpecificSendForMPI(int destNode, int size, char *msg, int mode) {
+CmiCommHandle LrtsSendFunc(int destPE, int size, char *msg, int mode) {
     /* Ignoring the mode for MPI layer */
 
+    int destNode = CmiNodeOf(destPE);
     CmiState cs = CmiGetState();
     SMSG_LIST *msg_tmp;
     int  rank;
@@ -1012,7 +987,7 @@ static double sendtime = 0.0;
 
 #endif //end of CMK_SMP
 
-static void AdvanceCommunicationForMPI(int whenidle) {
+void LrtsAdvanceCommunication(int whenidle) {
 #if REPORT_COMM_METRICS
     double t1, t2, t3, t4;
     t1 = CmiWallTimer();
@@ -1057,11 +1032,11 @@ static void AdvanceCommunicationForMPI(int whenidle) {
 }
 /* ######End of functions related with communication progress ###### */
 
-static void MachinePostNonLocalForMPI() {
+void LrtsPostNonLocal() {
 #if !CMK_SMP
     if (no_outstanding_sends) {
         while (CpvAccess(MsgQueueLen)>0) {
-            AdvanceCommunicationForMPI(0);
+            LrtsAdvanceCommunication(0);
         }
     }
 
@@ -1113,7 +1088,7 @@ void CmiMachineProgressImpl() {
 #endif
 
 /* ######Beginning of functions related with exiting programs###### */
-void DrainResourcesForMPI() {
+void LrtsDrainResources() {
 #if !CMK_SMP
     while (!CmiAllAsyncMsgsSent()) {
         PumpMsgs();
@@ -1132,18 +1107,18 @@ void DrainResourcesForMPI() {
         }
     }
 #endif
-#if CMK_MEM_CHECKPOINT
+#if CMK_MEM_CHECKPOINT || CMK_MESSAGE_LOGGING
     if (CmiMyPe() == 0) mpi_end_spare();
 #endif
     MACHSTATE(2, "Machine exit barrier begin {");
     START_EVENT();
     if (MPI_SUCCESS != MPI_Barrier(charmComm))
-        CmiAbort("DrainResourcesForMPI: MPI_Barrier failed!\n");
+        CmiAbort("LrtsDrainResources: MPI_Barrier failed!\n");
     END_EVENT(10);
     MACHSTATE(2, "} Machine exit barrier end");
 }
 
-void MachineExitForMPI() {
+void LrtsExit() {
     int i;
 #if (CMK_DEBUG_MODE || CMK_WEB_MODE || NODE_0_IS_CONVHOST)
     int doPrint = 0;
@@ -1272,7 +1247,7 @@ static char *thread_level_tostring(int thread_level) {
  *  Obtain the number of nodes, my node id, and consuming machine layer
  *  specific arguments
  */
-static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNodeID) {
+void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) {
     int n,i;
     int ver, subver;
     int provided;
@@ -1364,7 +1339,7 @@ static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNod
     }
 
 
-#if CMK_MEM_CHECKPOINT
+#if CMK_MEM_CHECKPOINT || CMK_MESSAGE_LOGGING
     if (CmiGetArgInt(largv,"+wp",&num_workpes)) {
        CmiAssert(num_workpes <= *numNodes);
        total_pes = *numNodes;
@@ -1526,7 +1501,7 @@ static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNod
 #endif
 }
 
-static void MachinePreCommonInitForMPI(int everReturn) {
+void LrtsPreCommonInit(int everReturn) {
 
 #if MPI_POST_RECV
     int doInit = 1;
@@ -1622,7 +1597,7 @@ static void MachinePreCommonInitForMPI(int everReturn) {
 #endif
 }
 
-static void MachinePostCommonInitForMPI(int everReturn) {
+void LrtsPostCommonInit(int everReturn) {
 
     CmiIdleState *s=CmiNotifyGetState();
 
@@ -1730,7 +1705,7 @@ void CmiTimerInit(char **argv) {
     if (_absoluteTime && CmiMyPe() == 0)
         printf("Charm++> absolute MPI timer is used\n");
 
-#if ! CMK_MEM_CHECKPOINT
+#if ! CMK_MEM_CHECKPOINT && ! CMK_MESSAGE_LOGGING
     _is_global = CmiTimerIsSynchronized();
 #else
     _is_global = 0;
@@ -1750,7 +1725,7 @@ void CmiTimerInit(char **argv) {
             starttimer = minTimer;
         }
     } else { /* we don't have a synchronous timer, set our own start time */
-#if ! CMK_MEM_CHECKPOINT
+#if ! CMK_MEM_CHECKPOINT && ! CMK_MESSAGE_LOGGING
         CmiBarrier();
         CmiBarrier();
         CmiBarrier();
@@ -1892,7 +1867,7 @@ int CmiBarrierZero() {
 }
 
 
-#if CMK_MEM_CHECKPOINT
+#if CMK_MEM_CHECKPOINT || CMK_MESSAGE_LOGGING
 
 void mpi_restart_crashed(int pe, int rank)
 {
@@ -2192,7 +2167,7 @@ static void recordMsgHistogramInfo(int size)
 }
 #endif /* end of MPI_DYNAMIC_POST_RECV */
 
-static void reportMsgHistogramInfo()
+void reportMsgHistogramInfo()
 {
 #if MPI_DYNAMIC_POST_RECV
     int i, count;
index 79abe68e2d4087ddb6b2d68ae705cc59f4b645f8..b0e83053382f0dfcf7bfcb8c3f1db3f43eb202d8 100644 (file)
@@ -1,3 +1,4 @@
 #define __FAULT__              1
 #define _FAULT_MLOG_           1
 #define CMK_CHARE_USE_PTR      1
+#define CMK_MESSAGE_LOGGING 1
index 0cb6a4d7c7747f19d63965bb90686060dfa0f1bc..72c80b4801ff5504a4ae3132e76197f520411597 100644 (file)
@@ -2,7 +2,7 @@
  *        functions for broadcast
 **/
 
-CmiCommHandle CmiSendNetworkFunc(int destNode, int size, char *msg, int mode);
+CmiCommHandle CmiSendNetworkFunc(int destPE, int size, char *msg, int mode);
 
 static void handleOneBcastMsg(int size, char *msg) {
     CmiAssert(CMI_BROADCAST_ROOT(msg)!=0);
@@ -112,10 +112,10 @@ static void SendSpanningChildren(int size, char *msg, int rankToAssign, int star
         CmiAssert(nd>=0 && nd!=CmiMyNode());
 #if CMK_BROADCAST_USE_CMIREFERENCE
         CmiReference(msg);
-        CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
+        CmiSendNetworkFunc(CmiNodeFirst(nd), size, msg, BCAST_SYNC);
 #else
         newmsg = CopyMsg(msg, size);
-        CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
+        CmiSendNetworkFunc(CmiNodeFirst(nd), size, newmsg, BCAST_SYNC);
 #endif
     }
     CMI_DEST_RANK(msg) = oldRank;
@@ -160,10 +160,10 @@ static void SendHyperCube(int size,  char *msg, int rankToAssign, int startNode)
         CmiAssert(nd>=0 && nd!=CmiMyNode());
 #if CMK_BROADCAST_USE_CMIREFERENCE
         CmiReference(msg);
-        CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
+        CmiSendNetworkFunc(CmiNodeFirst(nd), size, msg, BCAST_SYNC);
 #else
         char *newmsg = CopyMsg(msg, size);
-        CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
+        CmiSendNetworkFunc(CmiNodeFirst(nd), size, newmsg, BCAST_SYNC);
 #endif
     }
     CMI_DEST_RANK(msg) = oldRank;
index a950e5ea1d01abc5dba0e703c928ceeb66fc0fdf..3c3e112256f0b6d93ee652b8f34eac8ce072d254 100644 (file)
@@ -487,9 +487,10 @@ CpvExtern(int, _urgentSend);
 #if CMK_C_INLINE
 inline 
 #endif
-CmiCommHandle CmiSendNetworkFunc(int destNode, int size, char *msg, int mode)
+CmiCommHandle CmiSendNetworkFunc(int destPE, int size, char *msg, int mode)
 {
         int rank;
+        int destNode = CmiNodeOf(destPE); 
 #if CMK_USE_PXSHM
         if (CmiValidPxshm(destNode, size)) {
           CmiSendMessagePxshm(msg, size, destNode, &refcount);
@@ -525,7 +526,7 @@ if (MSG_STATISTIC)
 #if CMK_USE_OOB
     if (CpvAccess(_urgentSend)) mode |= OUT_OF_BAND;
 #endif
-    return LrtsSendFunc(destNode, size, msg, mode);
+    return LrtsSendFunc(destPE, size, msg, mode);
 }
 
 void CmiFreeSendFn(int destPE, int size, char *msg) {
@@ -550,7 +551,7 @@ void CmiFreeSendFn(int destPE, int size, char *msg) {
         }
 #endif
         CMI_DEST_RANK(msg) = destRank;
-        CmiSendNetworkFunc(destNode, size, msg, P2P_SYNC);
+        CmiSendNetworkFunc(destPE, size, msg, P2P_SYNC);
 #if CMK_PERSISTENT_COMM
         if (CpvAccess(phs)) CpvAccess(curphs)++;
 #endif
@@ -613,7 +614,7 @@ if (  MSG_STATISTIC)
     msg_histogram[ret_log]++;
 }
 #endif
-        CmiSendNetworkFunc(destNode, size, msg, P2P_SYNC);
+        CmiSendNetworkFunc(CmiNodeFirst(destNode), size, msg, P2P_SYNC);
     }
 #if CMK_PERSISTENT_COMM
     if (CpvAccess(phs)) CpvAccess(curphs)++;
@@ -635,7 +636,7 @@ if (  MSG_STATISTIC)
         msg_histogram[ret_log]++;
 }
 #endif
-        return CmiSendNetworkFunc(destNode, size, msg, P2P_ASYNC);
+        return CmiSendNetworkFunc(CmiNodeFirst(destNode), size, msg, P2P_ASYNC);
     }
 }
 #endif
index d3427f6787640cebdd51e6b04420587720097bf4..442c1695b43ed81ea18b9837e8a58645b8e513c0 100644 (file)
@@ -4,7 +4,7 @@
 void LrtsPrepareEnvelope(char *msg, int size);
 
 /* The machine-specific send function */
-CmiCommHandle LrtsSendFunc(int destNode, int size, char *msg, int mode);
+CmiCommHandle LrtsSendFunc(int destPE, int size, char *msg, int mode);
 
 void LrtsSyncListSendFn(int npes, int *pes, int len, char *msg);
 CmiCommHandle LrtsAsyncListSendFn(int npes, int *pes, int len, char *msg);
index 1f94e81716ad6176fefa27263fe381f968301921..6e562704aa3c3b02ea98bfc93a6cbf0f50f2ab4c 100644 (file)
@@ -285,11 +285,10 @@ static CmiState     Cmi_state_vector;
 #define CmiGetState() ((CmiState)pthread_getspecific(Cmi_state_key))
 #else
 CmiState CmiGetState() {
-       CmiState ret=(CmiState)pthread_getspecific(Cmi_state_key);
-       if (ret==NULL || Cmi_state_key == (pthread_key_t)(-1)) {
-               return &Cmi_default_state;
-       }
-       return ret;
+       CmiState ret;
+       if (Cmi_state_key == (pthread_key_t)(-1)) return &Cmi_default_state;
+       ret=(CmiState)pthread_getspecific(Cmi_state_key);
+       return (ret==NULL)? &Cmi_default_state : ret;
 }
 
 #endif
index 6cdbf46b53f97866f8a5cb7659a54dcf5110f284..5183909fc7fb840ddd3718fc4538e57ee737dd2b 100644 (file)
@@ -1588,7 +1588,7 @@ void CkSendMsg(int entryIdx, void *msg,const CkChareID *pCid, int opts)
   // VidBlock was not yet filled). The problem is that the creation was never
   // traced later when the VidBlock was filled. One solution is to trace the
   // creation here, the other to trace it in VidBlock->msgDeliver().
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        sendChareMsg(env,destPE,_infoIdx,pCid);
 #else
   _TRACE_CREATION_1(env);
@@ -1673,9 +1673,7 @@ static inline void _sendMsgBranch(int eIdx, void *msg, CkGroupID gID,
 {
   int numPes;
   register envelope *env = _prepareMsgBranch(eIdx,msg,gID,ForBocMsg);
-#if defined(_FAULT_MLOG_) 
-  sendTicketGroupRequest(env,pe,_infoIdx);
-#elif defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        sendGroupMsg(env,pe,_infoIdx);
 #else
   _TRACE_ONLY(numPes = (pe==CLD_BROADCAST_ALL?CkNumPes():1));
@@ -1823,9 +1821,7 @@ static inline void _sendMsgNodeBranch(int eIdx, void *msg, CkGroupID gID,
 {
   int numPes;
   register envelope *env = _prepareMsgBranch(eIdx,msg,gID,ForNodeBocMsg);
-#if defined(_FAULT_MLOG_)
-        sendTicketNodeGroupRequest(env,node,_infoIdx);
-#elif defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        sendNodeGroupMsg(env,node,_infoIdx);
 #else
   numPes = (node==CLD_BROADCAST_ALL?CkNumNodes():1);
@@ -2004,9 +2000,7 @@ extern "C"
 void CkArrayManagerDeliver(int pe,void *msg, int opts) {
   register envelope *env = UsrToEnv(msg);
   _prepareOutgoingArrayMsg(env,ForArrayEltMsg);
-#if defined(_FAULT_MLOG_)
-   sendTicketArrayRequest(env,pe,_infoIdx);
-#elif defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        sendArrayMsg(env,pe,_infoIdx);
 #else
   if (opts & CK_MSG_IMMEDIATE)
index ea3cbb75f4f578d74c8613acd6163945445b6068..f4bded4396c33a0e415c842df85496343d2b2fa4 100644 (file)
@@ -1385,7 +1385,10 @@ void CkArray::broadcastHomeElements(void *data,CkLocRec *rec,CkArrayIndex *index
         CkArrayMessage *copy = (CkArrayMessage *)   CkCopyMsg((void **)&bcast);
         envelope *env = UsrToEnv(copy);
         env->sender.data.group.onPE = CkMyPe();
-        env->TN  = env->SN=0;
+#if defined(_FAULT_CAUSAL_)
+        env->TN = 0;
+#endif
+               env->SN = 0;
         env->piggyBcastIdx = epIdx;
         env->setEpIdx(CkIndex_ArrayElement::recvBroadcast(0));
         env->getsetArrayMgr() = thisgroup;
index ac1aff931e9124da84c37579b95273b76d2abc15..d36d428a72a13cd8a7ab916c5c05db7bd223e6bd 100644 (file)
@@ -2075,11 +2075,11 @@ void CkLocMgr::informHome(const CkArrayIndex &idx,int nowOnPe)
                //Let this element's home Pe know it lives here now
                DEBC((AA"  Telling %s's home %d that it lives on %d.\n"AB,idx2str(idx),home,nowOnPe));
 //#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
-#if defined(_FAULT_MLOG_)
-        informLocationHome(thisgroup,idx,home,CkMyPe());
-#else
+//#if defined(_FAULT_MLOG_)
+//        informLocationHome(thisgroup,idx,home,CkMyPe());
+//#else
                thisProxy[home].updateLocation(idx,nowOnPe);
-#endif
+//#endif
        }
 }
 
@@ -2251,7 +2251,7 @@ int CkLocMgr::deliver(CkMessage *m,CkDeliver_t type,int opts) {
                DEBS((AA"deliver %s rec is null\n"AB,idx2str(idx)));
        }
 //#if (!defined(_FAULT_MLOG_) && !defined(_FAULT_CAUSAL_))
-#if !defined(_FAULT_MLOG_)
+//#if !defined(_FAULT_MLOG_)
 #if CMK_LBDB_ON
        if (type==CkDeliver_queue) {
                if (!(opts & CK_MSG_LB_NOTRACE) && the_lbdb->CollectingCommStats()) {
@@ -2260,7 +2260,7 @@ int CkLocMgr::deliver(CkMessage *m,CkDeliver_t type,int opts) {
                }
        }
 #endif
-#endif
+//#endif
 #if CMK_GRID_QUEUE_AVAILABLE
        int gridSrcPE;
        int gridSrcCluster;
@@ -2671,12 +2671,12 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
        DEBM((AA"Migrated index size %s to %d \n"AB,idx2str(idx),toPe));        
 
 //#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
-#if defined(_FAULT_MLOG_)
-    sendMlogLocation(toPe,UsrToEnv(msg));
-#else
+//#if defined(_FAULT_MLOG_)
+//    sendMlogLocation(toPe,UsrToEnv(msg));
+//#else
        //Send off message and delete old copy
        thisProxy[toPe].immigrate(msg);
-#endif
+//#endif
 
        duringMigration=CmiTrue;
        delete rec; //Removes elements, hashtable entries, local index
@@ -2686,9 +2686,9 @@ void CkLocMgr::emigrate(CkLocRec_local *rec,int toPe)
        //The element now lives on another processor-- tell ourselves and its home
        inform(idx,toPe);
 //#if (!defined(_FAULT_MLOG_) && !defined(_FAULT_CAUSAL_))    
-#if !defined(_FAULT_MLOG_)    
+//#if !defined(_FAULT_MLOG_)    
        informHome(idx,toPe);
-#endif
+//#endif
 
 #if CMK_GLOBAL_LOCATION_UPDATE
         DEBM((AA"Global location update. idx %s " 
@@ -2721,11 +2721,11 @@ void CkLocMgr::immigrate(CkArrayElementMigrateMessage *msg)
 
        //Create a record for this element
 //#if (!defined(_FAULT_MLOG_) && !defined(_FAULT_CAUSAL_))    
-#if !defined(_FAULT_MLOG_)     
+//#if !defined(_FAULT_MLOG_)     
        CkLocRec_local *rec=createLocal(idx,CmiTrue,msg->ignoreArrival,CmiFalse /* home told on departure */ );
-#else
-    CkLocRec_local *rec=createLocal(idx,CmiTrue,CmiTrue,CmiFalse /* home told on departure */ );
-#endif
+//#else
+//    CkLocRec_local *rec=createLocal(idx,CmiTrue,CmiTrue,CmiFalse /* home told on departure */ );
+//#endif
        
        //Create the new elements as we unpack the message
        pupElementsFor(p,rec,CkElementCreation_migrate);
index 734e15623070d5e227db64ecd02e81c7798cd7c9..4f12ca2d2bb115424e8971e73d8f20d860ab51e9 100644 (file)
@@ -63,7 +63,7 @@ void noopck(const char*, ...)
 #if CMK_CONVERSE_MPI
 #define CK_NO_PROC_POOL                                0
 #else
-#define CK_NO_PROC_POOL                                1
+#define CK_NO_PROC_POOL                                0
 #endif
 
 #define STREAMING_INFORMHOME                    1
@@ -588,7 +588,7 @@ void CkMemCheckPT::report()
     objsize += entry->getSize();
   }
   CmiAssert(CpvAccess(procChkptBuf));
-  CkPrintf("[%d] Checkpoint object size: %d len: %d Processor data: %d \n", CkMyPe(), objsize, len, CpvAccess(procChkptBuf)->len);
+//  CkPrintf("[%d] Checkpoint object size: %d len: %d Processor data: %d \n", CkMyPe(), objsize, len, CpvAccess(procChkptBuf)->len);
 }
 
 /*****************************************************************************
@@ -862,7 +862,7 @@ void CkMemCheckPT::recoverArrayElements()
 {
   double curTime = CmiWallTimer();
   int len = ckTable.length();
-  CkPrintf("[%d] CkMemCheckPT ----- %s len: %d in %f seconds \n",CkMyPe(), stage, len, curTime-startTime);
+  //CkPrintf("[%d] CkMemCheckPT ----- %s len: %d in %f seconds \n",CkMyPe(), stage, len, curTime-startTime);
   stage = (char *)"recoverArrayElements";
   if (CkMyPe() == thisFailedPe)
   CkPrintf("[%d] CkMemCheckPT ----- %s starts at %f \n",CkMyPe(), stage, curTime);
@@ -1368,7 +1368,7 @@ void CkRegisterRestartHandler( )
 
 #if ! CMK_CONVERSE_MPI
   // print pid to kill
-  CkPrintf("[%d] PID %d \n", CkMyPe(), getpid());
+//  CkPrintf("[%d] PID %d \n", CkMyPe(), getpid());
 //  sleep(4);
 #endif
 #endif
@@ -1393,8 +1393,12 @@ void killLocal(void *_dummy,double curWallTime){
         printf("[%d] KillLocal called at %.6lf \n",CkMyPe(),CmiWallTimer());          
         if(CmiWallTimer()<killTime-1){
                 CcdCallFnAfter(killLocal,NULL,(killTime-CmiWallTimer())*1000);        
-        }else{  
+        }else{ 
+#if CMK_CONVERSE_MPI
+                               CkDieNow();
+#else 
                 kill(getpid(),SIGKILL);                                               
+#endif
         }              
 } 
 #else
index 9c859099287cf836bf85217489d2e5045e65ef92..efdf53c465b1a19557be486348155580936ba92a 100644 (file)
@@ -1,7 +1,10 @@
 /**
- * Message Logging Fault Tolerance Protocol
- * It includes the main functions for the basic and team-based schemes.
- */
+  * Fast Message Logging Fault Tolerance Protocol.
+  * Features:
+       * Reduces the latency overhead of causal message-logging approach. It does NOT use determinants at all.
+       * Supports multiple concurrent failures.
+       * Assumes the application is iterative and structured in communication.
+  */
 
 #include "charm.h"
 #include "ck.h"
 
 #ifdef _FAULT_MLOG_
 
-//#define DEBUG(x)  if(_restartFlag) {x;}
-#define DEBUG_MEM(x) //x
-#define DEBUG(x)  //x
-#define DEBUGRESTART(x)  //x
-#define DEBUGLB(x) // x
-#define DEBUG_TEAM(x)  // x
+// Collects some statistics about message logging. 
+#define COLLECT_STATS_MSGS 0
+#define COLLECT_STATS_MSGS_TOTAL 0
+#define COLLECT_STATS_MSG_COUNT 0
+#define COLLECT_STATS_MEMORY 0
+#define COLLECT_STATS_TEAM 0
 
-#define BUFFERED_LOCAL
-#define BUFFERED_REMOTE 
+#define RECOVERY_SEND "SEND"
+#define RECOVERY_PROCESS "PROCESS"
+
+#define DEBUG_MEM(x)  //x
+#define DEBUG(x) // x
+#define DEBUG_RESTART(x)  //x
+#define DEBUGLB(x)   // x
+#define DEBUG_TEAM(x)  // x
+#define DEBUG_PERF(x) // x
+#define DEBUG_CHECKPOINT 1
+#define DEBUG_NOW(x) x
+#define DEBUG_PE(x,y) // if(CkMyPe() == x) y
+#define DEBUG_PE_NOW(x,y)  if(CkMyPe() == x) y
+#define DEBUG_RECOVERY(x) //x
 
 extern const char *idx2str(const CkArrayIndex &ind);
 extern const char *idx2str(const ArrayElement *el);
-const char *idx2str(const CkArrayIndex &ind){
-       return idx2str((const CkArrayIndex &)ind);
-};
 
 void getGlobalStep(CkGroupID gID);
 
 bool fault_aware(CkObjID &recver);
-void sendCheckpointData(int mode);
+void sendCheckpointData();
 void createObjIDList(void *data,ChareMlogData *mlogData);
 inline bool isLocal(int destPE);
 inline bool isTeamLocal(int destPE);
+void printLog(CkObjID *log);
 
 int _restartFlag=0;
-//ERASE int restarted=0; // it's not being used anywhere
-
-//TML: variables for measuring savings with teams in message logging
-float MLOGFT_totalLogSize = 0.0;
-float MLOGFT_totalMessages = 0.0;
-float MLOGFT_totalObjects = 0.0;
+int _numRestartResponses=0;
 
-//TODO: remove for perf runs
-int countHashRefs=0; //count the number of gets
-int countHashCollisions=0;
-
-//#define CHECKPOINT_DISK
 char *checkpointDirectory=".";
 int unAckedCheckpoint=0;
-
-int countLocal=0,countBuffered=0;
-int countPiggy=0;
-int countClearBufferedLocalCalls=0;
-
 int countUpdateHomeAcks=0;
 
 extern int teamSize;
 extern int chkptPeriod;
-extern bool parallelRestart;
+extern bool fastRecovery;
+extern int parallelRecovery;
 
 char *killFile;
+char *faultFile;
 int killFlag=0;
+int faultFlag=0;
 int restartingMlogFlag=0;
 void readKillFile();
 double killTime=0.0;
+double faultMean;
 int checkpointCount=0;
 
-
+/***** VARIABLES FOR MESSAGE LOGGING *****/
+// stores the id of current object sending a message
 CpvDeclare(Chare *,_currentObj);
-CpvDeclare(CkQ<LocalMessageLog> *,_localMessageLog);
-CpvDeclare(CkQ<TicketRequest *> *,_delayedTicketRequests);
+// stores checkpoint from buddy
 CpvDeclare(StoredCheckpoint *,_storedCheckpointData);
-CpvDeclare(CkQ<MlogEntry *> *,_delayedLocalTicketRequests);
+// stores the incarnation number from every other processor
+CpvDeclare(char *, _incarnation);
+// stores messages received before object gets created
 CpvDeclare(Queue, _outOfOrderMessageQueue);
-CpvDeclare(CkQ<LocalMessageLog>*,_bufferedLocalMessageLogs);
-//CpvDeclare(CkQ<TicketRequest>**,_bufferedTicketRequests);
-CpvDeclare(char **,_bufferedTicketRequests);
-CpvDeclare(int *,_numBufferedTicketRequests);
-CpvDeclare(char *,_bufferTicketReply);
-
-
-
-static double adjustChkptPeriod=0.0; //in ms
-static double nextCheckpointTime=0.0;//in seconds
-
-double lastBufferedLocalMessageCopyTime;
-
-int _maxBufferedMessages;
-int _maxBufferedTicketRequests;
-int BUFFER_TIME=2; // in ms
-
+/***** *****/
+
+/***** VARIABLES FOR PARALLEL RECOVERY *****/
+CpvDeclare(int, _numEmigrantRecObjs);
+CpvDeclare(int, _numImmigrantRecObjs);
+CpvDeclare(CkVec<CkLocation *> *, _immigrantRecObjs);
+/***** *****/
+
+#if COLLECT_STATS_MSGS
+int *numMsgsTarget;
+int *sizeMsgsTarget;
+int totalMsgsTarget;
+float totalMsgsSize;
+#endif
+#if COLLECT_STATS_MEMORY
+int msgLogSize;
+int bufferedDetsSize;
+int storedDetsSize;
+#endif
 
-int _ticketRequestHandlerIdx;
-int _ticketHandlerIdx;
-int _localMessageCopyHandlerIdx;
-int _localMessageAckHandlerIdx;
+/***** IDS FOR MESSAGE LOGGING HANDLERS *****/
 int _pingHandlerIdx;
-int _bufferedLocalMessageCopyHandlerIdx;
-int _bufferedLocalMessageAckHandlerIdx;
-int _bufferedTicketRequestHandlerIdx;
-int _bufferedTicketHandlerIdx;
-
-
-char objString[100];
 int _checkpointRequestHandlerIdx;
 int _storeCheckpointHandlerIdx;
 int _checkpointAckHandlerIdx;
 int _getCheckpointHandlerIdx;
 int _recvCheckpointHandlerIdx;
-int _removeProcessedLogHandlerIdx;
-
 int _verifyAckRequestHandlerIdx;
 int _verifyAckHandlerIdx;
 int _dummyMigrationHandlerIdx;
-
-
 int    _getGlobalStepHandlerIdx;
 int    _recvGlobalStepHandlerIdx;
-
 int _updateHomeRequestHandlerIdx;
 int _updateHomeAckHandlerIdx;
 int _resendMessagesHandlerIdx;
-int _resendReplyHandlerIdx;
-int _receivedTNDataHandlerIdx;
+int _receivedDetDataHandlerIdx;
 int _distributedLocationHandlerIdx;
+int _sendBackLocationHandlerIdx;
 
-//TML: integer constants for team-based message logging
-int _restartHandlerIdx;
-int _getRestartCheckpointHandlerIdx;
-int _recvRestartCheckpointHandlerIdx;
 void setTeamRecovery(void *data, ChareMlogData *mlogData);
 void unsetTeamRecovery(void *data, ChareMlogData *mlogData);
-
 int verifyAckTotal;
 int verifyAckCount;
-
 int verifyAckedRequests=0;
-
 RestartRequest *storedRequest;
-
-
-
 int _falseRestart =0; /**
                                                                                                        For testing on clusters we might carry out restarts on 
                                                                                                        a porcessor without actually starting it
                                                                                                        1 -> false restart
                                                                                                        0 -> restart after an actual crash
-                                                                                               */                                                                                                                              
-
-//lock for the ticketRequestHandler and ticketLogLocalMessage methods;
-int _lockNewTicket=0;
-
+                                                                                               */
 
 //Load balancing globals
 int onGoingLoadBalancing=0;
@@ -165,7 +142,6 @@ int _receiveMigrationNoticeHandlerIdx;
 int _receiveMigrationNoticeAckHandlerIdx;
 int _checkpointBarrierHandlerIdx;
 int _checkpointBarrierAckHandlerIdx;
-
 CkVec<MigrationRecord> migratedNoticeList;
 CkVec<RetainedMigratedObject *> retainedObjectList;
 int donotCountMigration=0;
@@ -176,122 +152,102 @@ int checkpointBarrierCount=0;
 int globalResumeCount=0;
 CkGroupID globalLBID;
 int restartDecisionNumber=-1;
-
-
 double lastCompletedAlarm=0;
 double lastRestart=0;
 
-
 //update location globals
 int _receiveLocationHandlerIdx;
 
+#if CMK_CONVERSE_MPI
+static int heartBeatHandlerIdx;
+static int heartBeatCheckHandlerIdx;
+static int partnerFailureHandlerIdx;
+static double lastPingTime = -1;
 
+extern "C" void mpi_restart_crashed(int pe, int rank);
+extern "C" int  find_spare_mpirank(int pe);
 
-// initialize message logging datastructures and register handlers
+void heartBeatPartner();
+void heartBeatHandler(void *msg);
+void heartBeatCheckHandler();
+void partnerFailureHandler(char *msg);
+int getReverseCheckPointPE();
+#endif
+
+/***** *****/
+
+/** 
+ * @brief Initialize message logging data structures and register handlers
+ */
 void _messageLoggingInit(){
+       if(CkMyPe() == 0)
+               CkPrintf("[%d] Fast Message Logging Support \n",CkMyPe());
+
        //current object
        CpvInitialize(Chare *,_currentObj);
        
        //registering handlers for message logging
-       _ticketRequestHandlerIdx = CkRegisterHandler((CmiHandler)_ticketRequestHandler);
-       _ticketHandlerIdx = CkRegisterHandler((CmiHandler)_ticketHandler);
-       _localMessageCopyHandlerIdx = CkRegisterHandler((CmiHandler)_localMessageCopyHandler);
-       _localMessageAckHandlerIdx = CkRegisterHandler((CmiHandler)_localMessageAckHandler);
        _pingHandlerIdx = CkRegisterHandler((CmiHandler)_pingHandler);
-       _bufferedLocalMessageCopyHandlerIdx = CkRegisterHandler((CmiHandler)_bufferedLocalMessageCopyHandler);
-       _bufferedLocalMessageAckHandlerIdx = CkRegisterHandler((CmiHandler)_bufferedLocalMessageAckHandler);
-       _bufferedTicketRequestHandlerIdx =  CkRegisterHandler((CmiHandler)_bufferedTicketRequestHandler);
-       _bufferedTicketHandlerIdx = CkRegisterHandler((CmiHandler)_bufferedTicketHandler);
-
                
        //handlers for checkpointing
        _storeCheckpointHandlerIdx = CkRegisterHandler((CmiHandler)_storeCheckpointHandler);
        _checkpointAckHandlerIdx = CkRegisterHandler((CmiHandler) _checkpointAckHandler);
-       _removeProcessedLogHandlerIdx  = CkRegisterHandler((CmiHandler)_removeProcessedLogHandler);
        _checkpointRequestHandlerIdx =  CkRegisterHandler((CmiHandler)_checkpointRequestHandler);
 
-
        //handlers for restart
        _getCheckpointHandlerIdx = CkRegisterHandler((CmiHandler)_getCheckpointHandler);
        _recvCheckpointHandlerIdx = CkRegisterHandler((CmiHandler)_recvCheckpointHandler);
-       _updateHomeRequestHandlerIdx =CkRegisterHandler((CmiHandler)_updateHomeRequestHandler);
-       _updateHomeAckHandlerIdx =  CkRegisterHandler((CmiHandler) _updateHomeAckHandler);
        _resendMessagesHandlerIdx = CkRegisterHandler((CmiHandler)_resendMessagesHandler);
-       _resendReplyHandlerIdx = CkRegisterHandler((CmiHandler)_resendReplyHandler);
-       _receivedTNDataHandlerIdx=CkRegisterHandler((CmiHandler)_receivedTNDataHandler);
        _distributedLocationHandlerIdx=CkRegisterHandler((CmiHandler)_distributedLocationHandler);
+       _sendBackLocationHandlerIdx=CkRegisterHandler((CmiHandler)_sendBackLocationHandler);
        _verifyAckRequestHandlerIdx = CkRegisterHandler((CmiHandler)_verifyAckRequestHandler);
        _verifyAckHandlerIdx = CkRegisterHandler((CmiHandler)_verifyAckHandler);
        _dummyMigrationHandlerIdx = CkRegisterHandler((CmiHandler)_dummyMigrationHandler);
 
-       //TML: handlers for team-based message logging
-       _restartHandlerIdx = CkRegisterHandler((CmiHandler)_restartHandler);
-       _getRestartCheckpointHandlerIdx = CkRegisterHandler((CmiHandler)_getRestartCheckpointHandler);
-       _recvRestartCheckpointHandlerIdx = CkRegisterHandler((CmiHandler)_recvRestartCheckpointHandler);
-
-       
        //handlers for load balancing
        _receiveMlogLocationHandlerIdx=CkRegisterHandler((CmiHandler)_receiveMlogLocationHandler);
        _receiveMigrationNoticeHandlerIdx=CkRegisterHandler((CmiHandler)_receiveMigrationNoticeHandler);
        _receiveMigrationNoticeAckHandlerIdx=CkRegisterHandler((CmiHandler)_receiveMigrationNoticeAckHandler);
        _getGlobalStepHandlerIdx=CkRegisterHandler((CmiHandler)_getGlobalStepHandler);
        _recvGlobalStepHandlerIdx=CkRegisterHandler((CmiHandler)_recvGlobalStepHandler);
-       _receiveMigrationNoticeHandlerIdx=CkRegisterHandler((CmiHandler)_receiveMigrationNoticeHandler);
-       _receiveMigrationNoticeAckHandlerIdx=CkRegisterHandler((CmiHandler)_receiveMigrationNoticeAckHandler);
        _checkpointBarrierHandlerIdx=CkRegisterHandler((CmiHandler)_checkpointBarrierHandler);
        _checkpointBarrierAckHandlerIdx=CkRegisterHandler((CmiHandler)_checkpointBarrierAckHandler);
-
        
        //handlers for updating locations
        _receiveLocationHandlerIdx=CkRegisterHandler((CmiHandler)_receiveLocationHandler);
+
+       // handlers for failure detection in MPI layer
+#if CMK_CONVERSE_MPI
+       heartBeatHandlerIdx = CkRegisterHandler((CmiHandler)heartBeatHandler);
+       heartBeatCheckHandlerIdx = CkRegisterHandler((CmiHandler)heartBeatCheckHandler);
+       partnerFailureHandlerIdx = CkRegisterHandler((CmiHandler)partnerFailureHandler);
+#endif
        
        //Cpv variables for message logging
-       CpvInitialize(CkQ<LocalMessageLog>*,_localMessageLog);
-       CpvAccess(_localMessageLog) = new CkQ<LocalMessageLog>(10000);
-       CpvInitialize(CkQ<TicketRequest *> *,_delayedTicketRequests);
-       CpvAccess(_delayedTicketRequests) = new CkQ<TicketRequest *>;
-       CpvInitialize(CkQ<MlogEntry *>*,_delayedLocalTicketRequests);
-       CpvAccess(_delayedLocalTicketRequests) = new CkQ<MlogEntry *>;
        CpvInitialize(Queue, _outOfOrderMessageQueue);
        CpvAccess(_outOfOrderMessageQueue) = CqsCreate();
-       CpvInitialize(CkQ<LocalMessageLog>*,_bufferedLocalMessageLogs);
-       CpvAccess(_bufferedLocalMessageLogs) = new CkQ<LocalMessageLog>;
        
-       CpvInitialize(char **,_bufferedTicketRequests);
-       CpvAccess(_bufferedTicketRequests) = new char *[CkNumPes()];
-       CpvAccess(_numBufferedTicketRequests) = new int[CkNumPes()];
-       for(int i=0;i<CkNumPes();i++){
-               CpvAccess(_bufferedTicketRequests)[i]=NULL;
-               CpvAccess(_numBufferedTicketRequests)[i]=0;
+       // Cpv variables for causal protocol
+       CpvInitialize(char *, _incarnation);
+       CpvAccess(_incarnation) = (char *) CmiAlloc(CmiNumPes() * sizeof(int));
+       for(int i=0; i<CmiNumPes(); i++){
+               CpvAccess(_incarnation)[i] = 0;
        }
-  CpvInitialize(char *,_bufferTicketReply);
-       CpvAccess(_bufferTicketReply) = (char *)CmiAlloc(sizeof(BufferedTicketRequestHeader)+_maxBufferedTicketRequests*sizeof(TicketReply));
-       
-//     CcdCallOnConditionKeep(CcdPERIODIC_100ms,retryTicketRequest,NULL);
-       CcdCallFnAfter(retryTicketRequest,NULL,100);    
-       
-       
+
+       // Cpv variables for parallel recovery
+       CpvInitialize(int, _numEmigrantRecObjs);
+    CpvAccess(_numEmigrantRecObjs) = 0;
+    CpvInitialize(int, _numImmigrantRecObjs);
+    CpvAccess(_numImmigrantRecObjs) = 0;
+
+    CpvInitialize(CkVec<CkLocation *> *, _immigrantRecObjs);
+    CpvAccess(_immigrantRecObjs) = new CkVec<CkLocation *>;
+
        //Cpv variables for checkpoint
        CpvInitialize(StoredCheckpoint *,_storedCheckpointData);
        CpvAccess(_storedCheckpointData) = new StoredCheckpoint;
-       
-//     CcdCallOnConditionKeep(CcdPERIODIC_10s,startMlogCheckpoint,NULL);
-//     printf("[%d] Checkpoint Period is %d s\n",CkMyPe(),chkptPeriod);
-//     CcdCallFnAfter(startMlogCheckpoint,NULL,chkptPeriod);
-       if(CkMyPe() == 0){
-//             CcdCallFnAfter(checkpointAlarm,NULL,chkptPeriod*1000);
-#ifdef         BUFFERED_LOCAL
-               if(CmiMyPe() == 0){
-                       printf("Local messages being buffered _maxBufferedMessages %d BUFFER_TIME %d ms \n",_maxBufferedMessages,BUFFER_TIME);
-               }
-#endif
-       }
-#ifdef         BUFFERED_REMOTE
-       if(CmiMyPe() == 0){
-               printf("[%d] Remote messages being buffered _maxBufferedTicketRequests %d BUFFER_TIME %d ms %p \n",CkMyPe(),_maxBufferedTicketRequests,BUFFER_TIME,CpvAccess(_bufferTicketReply));
-       }
-#endif
 
+       // registering user events for projections      
        traceRegisterUserEvent("Remove Logs", 20);
        traceRegisterUserEvent("Ticket Request Handler", 21);
        traceRegisterUserEvent("Ticket Handler", 22);
@@ -303,7 +259,6 @@ void _messageLoggingInit(){
        traceRegisterUserEvent("Checkpoint",28);
        traceRegisterUserEvent("Checkpoint Store",29);
        traceRegisterUserEvent("Checkpoint Ack",30);
-       
        traceRegisterUserEvent("Send Ticket Request",31);
        traceRegisterUserEvent("Generalticketrequest1",32);
        traceRegisterUserEvent("TicketLogLocal",33);
@@ -315,9 +270,101 @@ void _messageLoggingInit(){
        
        lastCompletedAlarm=CmiWallTimer();
        lastRestart = CmiWallTimer();
-//     CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,checkBufferedLocalMessageCopy,NULL);
-       CcdCallFnAfter( checkBufferedLocalMessageCopy ,NULL , BUFFER_TIME);
+
+       // fault detection for MPI layer
+#if CMK_CONVERSE_MPI
+  void heartBeatPartner();
+  void heartBeatCheckHandler();
+  CcdCallOnCondition(CcdPERIODIC_100ms,(CcdVoidFn)heartBeatPartner,NULL);
+  CcdCallOnCondition(CcdPERIODIC_5s,(CcdVoidFn)heartBeatCheckHandler,NULL);
+#endif
+
+#if COLLECT_STATS_MSGS
+#if COLLECT_STATS_MSGS_TOTAL
+       totalMsgsTarget = 0;
+       totalMsgsSize = 0.0;
+#else
+       numMsgsTarget = (int *)CmiAlloc(sizeof(int) * CmiNumPes());
+       sizeMsgsTarget = (int *)CmiAlloc(sizeof(int) * CmiNumPes());
+       for(int i=0; i<CmiNumPes(); i++){
+               numMsgsTarget[i] = 0;
+               sizeMsgsTarget[i] = 0;
+       }
+#endif
+#endif
+#if COLLECT_STATS_MEMORY
+       msgLogSize = 0;
+       bufferedDetsSize = 0;
+       storedDetsSize = 0;
+#endif
+
+}
+
+#if CMK_CONVERSE_MPI
+
+/**
+ * Receives the notification of a failure and updates pe-to-rank mapping.
+ */
+void partnerFailureHandler(char *msg)
+{
+   int diepe = *(int *)(msg+CmiMsgHeaderSizeBytes);
+
+   // send message to crash pe to let it restart
+   int newrank = find_spare_mpirank(diepe);
+   int buddy = getCheckPointPE();
+   if (buddy == diepe)  {
+     mpi_restart_crashed(diepe, newrank);
+     CcdCallOnCondition(CcdPERIODIC_5s,(CcdVoidFn)heartBeatCheckHandler,NULL);
+   }
+}
+
+/**
+ * Registers last time it knew about the PE that checkpoints on it.
+ */
+void heartBeatHandler(void *msg)
+{
+       lastPingTime = CmiWallTimer();
+       CmiFree(msg);
+}
+
+/**
+ * Checks whether the PE that checkpoints on it is still alive.
+ */
+void heartBeatCheckHandler()
+{
+       double now = CmiWallTimer();
+       if (lastPingTime > 0 && now - lastPingTime > 4) {
+               int i, pe, buddy;
+               // tell everyone that PE is down
+               buddy = getReverseCheckPointPE();
+               CmiPrintf("[%d] detected buddy processor %d died %f %f. \n", CmiMyPe(), buddy, now, lastPingTime);
+               
+               for (int pe = 0; pe < CmiNumPes(); pe++) {
+                       if (pe == buddy) continue;
+                       char *msg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes+sizeof(int));
+                       *(int *)(msg+CmiMsgHeaderSizeBytes) = buddy;
+                       CmiSetHandler(msg, partnerFailureHandlerIdx);
+                       CmiSyncSendAndFree(pe, CmiMsgHeaderSizeBytes+sizeof(int), (char *)msg);
+               }
+       }
+       else 
+               CcdCallOnCondition(CcdPERIODIC_5s,(CcdVoidFn)heartBeatCheckHandler,NULL);
+}
+
+/**
+ * Pings buddy to let it know this PE is alive. Used for failure detection.
+ */
+void heartBeatPartner()
+{
+       int buddy = getCheckPointPE();
+       //printf("[%d] heartBeatPartner %d\n", CmiMyPe(), buddy);
+       char *msg = (char*)CmiAlloc(CmiMsgHeaderSizeBytes+sizeof(int));
+       *(int *)(msg+CmiMsgHeaderSizeBytes) = CmiMyPe();
+       CmiSetHandler(msg, heartBeatHandlerIdx);
+       CmiSyncSendAndFree(buddy, CmiMsgHeaderSizeBytes+sizeof(int), (char *)msg);
+       CcdCallOnCondition(CcdPERIODIC_100ms,(CcdVoidFn)heartBeatPartner,NULL);
 }
+#endif
 
 void killLocal(void *_dummy,double curWallTime);       
 
@@ -338,21 +385,58 @@ void readKillFile(){
        fclose(fp);
 }
 
+/**
+ * @brief: reads the PE that will be failing throughout the execution and the mean time between failures.
+ * We assume an exponential distribution for the mean-time-between-failures.
+ */
+void readFaultFile(){
+        FILE *fp=fopen(faultFile,"r");
+        if(!fp){
+                return;
+        }
+        int proc;
+        double sec;
+        fscanf(fp,"%d %lf",&proc,&sec);
+       faultMean = sec;
+       if(proc == CkMyPe()){
+               printf("[%d] PE %d to be killed every %.6lf s (MEMCKPT) \n",CkMyPe(),proc,sec);
+               CcdCallFnAfter(killLocal,NULL,sec*1000);
+       }
+        fclose(fp);
+}
+
 void killLocal(void *_dummy,double curWallTime){
        printf("[%d] KillLocal called at %.6lf \n",CkMyPe(),CmiWallTimer());
        if(CmiWallTimer()<killTime-1){
                CcdCallFnAfter(killLocal,NULL,(killTime-CmiWallTimer())*1000);  
-       }else{  
+       }else{
+#if CMK_CONVERSE_MPI
+               CkDieNow();
+#else
                kill(getpid(),SIGKILL);
+#endif
        }
 }
 
+#if ! CMK_CONVERSE_MPI
+void CkDieNow()
+{
+       // kill -9 for non-mpi version
+       CmiPrintf("[%d] die now.\n", CmiMyPe());
+       killTime = CmiWallTimer()+0.001;
+       CcdCallFnAfter(killLocal,NULL,1);
+}
+#endif
+
 
+/*** Auxiliary Functions ***/
 
 /************************ Message logging methods ****************/
 
-// send a ticket request to a group on a processor
-void sendTicketGroupRequest(envelope *env,int destPE,int _infoIdx){
+/**
+ * Sends a group message that might be a broadcast.
+ */
+void sendGroupMsg(envelope *env, int destPE, int _infoIdx){
        if(destPE == CLD_BROADCAST || destPE == CLD_BROADCAST_ALL){
                DEBUG(printf("[%d] Group Broadcast \n",CkMyPe()));
                void *origMsg = EnvToUsr(env);
@@ -361,26 +445,29 @@ void sendTicketGroupRequest(envelope *env,int destPE,int _infoIdx){
                                void *copyMsg = CkCopyMsg(&origMsg);
                                envelope *copyEnv = UsrToEnv(copyMsg);
                                copyEnv->SN=0;
-                               copyEnv->TN=0;
                                copyEnv->sender.type = TypeInvalid;
                                DEBUG(printf("[%d] Sending group broadcast message to proc %d \n",CkMyPe(),i));
-                               sendTicketGroupRequest(copyEnv,i,_infoIdx);
+                               sendGroupMsg(copyEnv,i,_infoIdx);
                        }
                }
                return;
        }
+
+       // initializing values of envelope
+       env->SN=0;
+       env->sender.type = TypeInvalid;
+
        CkObjID recver;
        recver.type = TypeGroup;
        recver.data.group.id = env->getGroupNum();
        recver.data.group.onPE = destPE;
-/*     if(recver.data.group.id.idx == 11 && recver.data.group.onPE == 1){
-               CmiPrintStackTrace(0);
-       }*/
-       generateCommonTicketRequest(recver,env,destPE,_infoIdx);
+       sendCommonMsg(recver,env,destPE,_infoIdx);
 }
 
-//send a ticket request to a nodegroup
-void sendTicketNodeGroupRequest(envelope *env,int destNode,int _infoIdx){
+/**
+ * Sends a nodegroup message that might be a broadcast.
+ */
+void sendNodeGroupMsg(envelope *env, int destNode, int _infoIdx){
        if(destNode == CLD_BROADCAST || destNode == CLD_BROADCAST_ALL){
                DEBUG(printf("[%d] NodeGroup Broadcast \n",CkMyPe()));
                void *origMsg = EnvToUsr(env);
@@ -389,26 +476,52 @@ void sendTicketNodeGroupRequest(envelope *env,int destNode,int _infoIdx){
                                void *copyMsg = CkCopyMsg(&origMsg);
                                envelope *copyEnv = UsrToEnv(copyMsg);
                                copyEnv->SN=0;
-                               copyEnv->TN=0;
                                copyEnv->sender.type = TypeInvalid;
-                               sendTicketNodeGroupRequest(copyEnv,i,_infoIdx);
+                               sendNodeGroupMsg(copyEnv,i,_infoIdx);
                        }
                }
                return;
        }
+
+       // initializing values of envelope
+       env->SN=0;
+       env->sender.type = TypeInvalid;
+
        CkObjID recver;
        recver.type = TypeNodeGroup;
        recver.data.group.id = env->getGroupNum();
        recver.data.group.onPE = destNode;
-       generateCommonTicketRequest(recver,env,destNode,_infoIdx);
+       sendCommonMsg(recver,env,destNode,_infoIdx);
 }
 
-//send a ticket request to an array element
-void sendTicketArrayRequest(envelope *env,int destPE,int _infoIdx){
+/**
+ * Sends a message to an array element.
+ */
+void sendArrayMsg(envelope *env,int destPE,int _infoIdx){
        CkObjID recver;
        recver.type = TypeArray;
        recver.data.array.id = env->getsetArrayMgr();
-       recver.data.array.idx = *(&env->getsetArrayIndex());
+       recver.data.array.idx.asChild() = *(&env->getsetArrayIndex());
+
+       if(CpvAccess(_currentObj)!=NULL &&  CpvAccess(_currentObj)->mlogData->objID.type != TypeArray){
+               char recverString[100],senderString[100];
+               
+               DEBUG(printf("[%d] %s being sent message from non-array %s \n",CkMyPe(),recver.toString(recverString),CpvAccess(_currentObj)->mlogData->objID.toString(senderString)));
+       }
+
+       // initializing values of envelope
+       env->SN = 0;
+
+       sendCommonMsg(recver,env,destPE,_infoIdx);
+};
+
+/**
+ * Sends a message to a singleton chare.
+ */
+void sendChareMsg(envelope *env,int destPE,int _infoIdx, const CkChareID *pCid){
+       CkObjID recver;
+       recver.type = TypeChare;
+       recver.data.chare.id = *pCid;
 
        if(CpvAccess(_currentObj)!=NULL &&  CpvAccess(_currentObj)->mlogData->objID.type != TypeArray){
                char recverString[100],senderString[100];
@@ -416,82 +529,64 @@ void sendTicketArrayRequest(envelope *env,int destPE,int _infoIdx){
                DEBUG(printf("[%d] %s being sent message from non-array %s \n",CkMyPe(),recver.toString(recverString),CpvAccess(_currentObj)->mlogData->objID.toString(senderString)));
        }
 
-       generateCommonTicketRequest(recver,env,destPE,_infoIdx);
+       // initializing values of envelope
+       env->SN = 0;
+
+       sendCommonMsg(recver,env,destPE,_infoIdx);
 };
 
 /**
  * A method to generate the actual ticket requests for groups, nodegroups or arrays.
  */
-void generateCommonTicketRequest(CkObjID &recver,envelope *_env,int destPE,int _infoIdx){
+void sendCommonMsg(CkObjID &recver,envelope *_env,int destPE,int _infoIdx){
        envelope *env = _env;
-       MCount ticketNumber = 0;
        int resend=0; //is it a resend
-       char recverName[100];
-       double _startTime=CkWallTimer();
+       DEBUG(char recverName[100]);
+       DEBUG(char senderString[100]);
        
+       DEBUG_MEM(CmiMemoryCheck());
+
        if(CpvAccess(_currentObj) == NULL){
 //             CkAssert(0);
                DEBUG(printf("[%d] !!!!WARNING: _currentObj is NULL while message is being sent\n",CkMyPe());)
                generalCldEnqueue(destPE,env,_infoIdx);
                return;
        }
-       
-       if(env->sender.type == TypeInvalid){
+
+       // checking if this message should bypass determinants in message-logging
+       if(env->flags & CK_BYPASS_DET_MLOG){
                env->sender = CpvAccess(_currentObj)->mlogData->objID;
-               //Set message logging data in the envelope
-       }else{
-               envelope *copyEnv = copyEnvelope(env);
-               env = copyEnv;
-               env->sender = CpvAccess(_currentObj)->mlogData->objID;
-               env->SN = 0;
+               env->recver = recver;
+               DEBUG(CkPrintf("[%d] Bypassing determinants from %s to %s PE %d\n",CkMyPe(),CpvAccess(_currentObj)->mlogData->objID.toString(senderString),recver.toString(recverName),destPE));
+               generalCldEnqueue(destPE,env,_infoIdx);
+               return;
        }
        
+       // setting message logging data in the envelope
+       env->incarnation = CpvAccess(_incarnation)[CkMyPe()];
+       env->sender = CpvAccess(_currentObj)->mlogData->objID;
+       env->SN = 0;
+       
+       DEBUG_MEM(CmiMemoryCheck());
+
        CkObjID &sender = env->sender;
        env->recver = recver;
 
        Chare *obj = (Chare *)env->sender.getObject();
          
        if(env->SN == 0){
+               DEBUG_MEM(CmiMemoryCheck());
                env->SN = obj->mlogData->nextSN(recver);
        }else{
                resend = 1;
        }
-       
-       char senderString[100];
-//     if(env->SN != 1){
-               DEBUG(printf("[%d] Generate Ticket Request to %s from %s PE %d SN %d \n",CkMyPe(),env->recver.toString(recverName),env->sender.toString(senderString),destPE,env->SN));
-       //      CmiPrintStackTrace(0);
-/*     }else{
-               DEBUGRESTART(printf("[%d] Generate Ticket Request to %s from %s PE %d SN %d \n",CkMyPe(),env->recver.toString(recverName),env->sender.toString(senderString),destPE,env->SN));
-       }*/
-               
-       MlogEntry *mEntry = new MlogEntry(env,destPE,_infoIdx);
-//     CkPackMessage(&(mEntry->env));
-//     traceUserBracketEvent(32,_startTime,CkWallTimer());
-       
-       _startTime = CkWallTimer();
 
-       // uses the proper ticketing mechanism for local, group and general messages
+       // uses the proper sending mechanism for local or remote messages
        if(isLocal(destPE)){
-               ticketLogLocalMessage(mEntry);
+               sendLocalMsg(env, _infoIdx);
        }else{
-               if((teamSize > 1) && isTeamLocal(destPE)){
-
-                       // look to see if this message already has a ticket in the team-table
-                       Chare *senderObj = (Chare *)sender.getObject();
-                       SNToTicket *ticketRow = senderObj->mlogData->teamTable.get(recver);
-                       if(ticketRow != NULL){
-                               Ticket ticket = ticketRow->get(env->SN);
-                               if(ticket.TN != 0){
-                                       ticketNumber = ticket.TN;
-                                       DEBUG(CkPrintf("[%d] Found a team preticketed message\n",CkMyPe()));
-                               }
-                       }
-               }
-               
-               // sending the ticket request
-               sendTicketRequest(sender,recver,destPE,mEntry,env->SN,ticketNumber,resend);
-               
+               MlogEntry *mEntry = new MlogEntry(env,destPE,_infoIdx);
+               sendRemoteMsg(sender,recver,destPE,mEntry,env->SN,resend);
        }
 }
 
@@ -509,7 +604,7 @@ inline bool isLocal(int destPE){
 
 /**
  * Determines if the message is group local or not. A message is group local if:
- * 1) They belong to the same group in the group-based message logging.
+ * 1) They belong to the same team in the team-based message logging.
  */
 inline bool isTeamLocal(int destPE){
 
@@ -520,854 +615,171 @@ inline bool isTeamLocal(int destPE){
        return false;
 }
 
-
-
 /**
  * Method that does the actual send by creating a ticket request filling it up and sending it.
  */
-void sendTicketRequest(CkObjID &sender,CkObjID &recver,int destPE,MlogEntry *entry,MCount SN,MCount TN,int resend){
-       char recverString[100],senderString[100];
+void sendRemoteMsg(CkObjID &sender,CkObjID &recver,int destPE,MlogEntry *entry,MCount SN,int resend){
+       DEBUG_NOW(char recverString[100]);
+       DEBUG_NOW(char senderString[100]);
+
+       int totalSize;
+
        envelope *env = entry->env;
-       DEBUG(printf("[%d] Sending ticket Request to %s from %s PE %d SN %d time %.6lf \n",CkMyPe(),env->recver.toString(recverString),env->sender.toString(senderString),destPE,env->SN,CkWallTimer()));
-/*     envelope *env = entry->env;
-       printf("[%d] Sending ticket Request to %s from %s PE %d SN %d time %.6lf \n",CkMyPe(),env->recver.toString(recverString),env->sender.toString(senderString),destPE,env->SN,CkWallTimer());*/
+       DEBUG_PE(3,printf("[%d] Sending message to %s from %s PE %d SN %d time %.6lf \n",CkMyPe(),env->recver.toString(recverString),env->sender.toString(senderString),destPE,env->SN,CkWallTimer()));
 
+       // setting all the information
        Chare *obj = (Chare *)entry->env->sender.getObject();
+       entry->env->recver = recver;
+       entry->env->SN = SN;
        if(!resend){
-               //TML: only stores message if either it goes to this processor or to a processor in a different group
-               if(!isTeamLocal(entry->destPE)){
-                       obj->mlogData->addLogEntry(entry);
-                       MLOGFT_totalMessages += 1.0;
-                       MLOGFT_totalLogSize += entry->env->getTotalsize();
-               }else{
-                       // the message has to be deleted after it has been sent
-                       entry->env->freeMsg = true;
-               }
-       }
-
-#ifdef BUFFERED_REMOTE
-       //buffer the ticket request 
-       if(CpvAccess(_bufferedTicketRequests)[destPE] == NULL){
-               //first message to this processor, buffer needs to be created
-               int _allocSize = sizeof(TicketRequest)*_maxBufferedTicketRequests + sizeof(BufferedTicketRequestHeader);
-               CpvAccess(_bufferedTicketRequests)[destPE] = (char *)CmiAlloc(_allocSize);
-               DEBUG(CmiPrintf("[%d] _bufferedTicketRequests[%d] allocated as %p\n",CmiMyPe(),destPE,&((CpvAccess(_bufferedTicketRequests))[destPE][0])));
-       }
-       //CpvAccess(_bufferedTicketRequests)[destPE]->enq(ticketRequest);
-       //Buffer the ticketrequests
-       TicketRequest *ticketRequest = (TicketRequest *)&(CpvAccess(_bufferedTicketRequests)[destPE][sizeof(BufferedTicketRequestHeader)+CpvAccess(_numBufferedTicketRequests)[destPE]*sizeof(TicketRequest)]);
-       ticketRequest->sender = sender;
-       ticketRequest->recver = recver;
-       ticketRequest->logEntry = entry;
-       ticketRequest->SN = SN;
-       ticketRequest->TN = TN;
-       ticketRequest->senderPE = CkMyPe();
-
-       CpvAccess(_numBufferedTicketRequests)[destPE]++;
-       
-       
-       if(CpvAccess(_numBufferedTicketRequests)[destPE] >= _maxBufferedTicketRequests){
-               sendBufferedTicketRequests(destPE);
-       }else{
-               if(CpvAccess(_numBufferedTicketRequests)[destPE] == 1){
-                       int *checkPE = new int;
-                       *checkPE = destPE;
-                       CcdCallFnAfter( checkBufferedTicketRequests ,checkPE , BUFFER_TIME);            
-               }
+               obj->mlogData->addLogEntry(entry);
+#if COLLECT_STATS_TEAM
+               MLOGFT_totalMessages += 1.0;
+               MLOGFT_totalLogSize += entry->env->getTotalsize();
+#endif
        }
-#else
 
-       TicketRequest ticketRequest;
-       ticketRequest.sender = sender;
-       ticketRequest.recver = recver;
-       ticketRequest.logEntry = entry;
-       ticketRequest.SN = SN;
-       ticketRequest.TN = TN;
-       ticketRequest.senderPE = CkMyPe();
-       
-       CmiSetHandler((void *)&ticketRequest,_ticketRequestHandlerIdx);
-//     CmiBecomeImmediate(&ticketRequest);
-       CmiSyncSend(destPE,sizeof(TicketRequest),(char *)&ticketRequest);
-#endif
-       DEBUG_MEM(CmiMemoryCheck());
-};
+       // sending the message
+       generalCldEnqueue(destPE, entry->env, entry->_infoIdx);
 
-/**
- * Send the ticket requests buffered for processor PE
- **/
-void sendBufferedTicketRequests(int destPE){
-       DEBUG_MEM(CmiMemoryCheck());
-       int numberRequests = CpvAccess(_numBufferedTicketRequests)[destPE];
-       if(numberRequests == 0){
-               return;
-       }
-       DEBUG(printf("[%d] Send Buffered Ticket Requests to %d number %d\n",CkMyPe(),destPE,numberRequests));
-       int totalSize = sizeof(BufferedTicketRequestHeader )+numberRequests*(sizeof(TicketRequest));
-       void *buf = &(CpvAccess(_bufferedTicketRequests)[destPE][0]);
-       BufferedTicketRequestHeader *header = (BufferedTicketRequestHeader *)buf;
-       header->numberLogs = numberRequests;
-       
-       CmiSetHandler(buf,_bufferedTicketRequestHandlerIdx);
-       CmiSyncSend(destPE,totalSize,(char *)buf);
-       
-       CpvAccess(_numBufferedTicketRequests)[destPE]=0;
        DEBUG_MEM(CmiMemoryCheck());
+#if COLLECT_STATS_MSGS
+#if COLLECT_STATS_MSGS_TOTAL
+       totalMsgsTarget++;
+       totalMsgsSize += (float)env->getTotalsize();
+#else
+       numMsgsTarget[destPE]++;
+       sizeMsgsTarget[destPE] += env->getTotalsize();
+#endif
+#endif
+#if COLLECT_STATS_MEMORY
+       msgLogSize += env->getTotalsize();
+#endif
 };
 
-void checkBufferedTicketRequests(void *_destPE,double curWallTime){
-       int destPE = *(int *)_destPE;
-  if(CpvAccess(_numBufferedTicketRequests)[destPE] > 0){
-               sendBufferedTicketRequests(destPE);
-//             traceUserEvent(35);
-       }
-       delete (int *)_destPE;
-       DEBUG_MEM(CmiMemoryCheck());
-};
 
 /**
- * Gets a ticket for a local message and then sends a copy to the buddy.
- * This method is always in the main thread(not interrupt).. so it should 
- * never find itself locked out of a newTicket.
+ * @brief Function to send a local message. It first gets a ticket and
+ * then enqueues the message. If we are recovering, then the message 
+ * is enqueued in a delay queue.
  */
-void ticketLogLocalMessage(MlogEntry *entry){
-       double _startTime=CkWallTimer();
+void sendLocalMsg(envelope *env, int _infoIdx){
+       DEBUG_PERF(double _startTime=CkWallTimer());
        DEBUG_MEM(CmiMemoryCheck());
+       DEBUG(Chare *senderObj = (Chare *)env->sender.getObject();)
+       DEBUG(char senderString[100]);
+       DEBUG(char recverString[100]);
 
-       Chare *recverObj = (Chare *)entry->env->recver.getObject();
-       DEBUG(Chare *senderObj = (Chare *)entry->env->sender.getObject();)
+       DEBUG(printf("[%d] Local Message being sent for SN %d sender %s recver %s \n",CmiMyPe(),env->SN,env->sender.toString(senderString),env->recver.toString(recverString)));
+
+       // getting the receiver local object
+       Chare *recverObj = (Chare *)env->recver.getObject();
+
+       // if receiver object is not NULL, we will ask it for a ticket
        if(recverObj){
-               //Consider the case, after a restart when this message has already been allotted a ticket number
-               // and should get the same one as the old one.
-               Ticket ticket;
-               if(recverObj->mlogData->mapTable.numObjects() > 0){
-                       ticket.TN = recverObj->mlogData->searchRestoredLocalQ(entry->env->sender,entry->env->recver,entry->env->SN);
-               }else{
-                       ticket.TN = 0;
-               }
-               
-               char senderString[100], recverString[100] ;
-               
-               if(ticket.TN == 0){
-                       ticket = recverObj->mlogData->next_ticket(entry->env->sender,entry->env->SN);
-       
-                       if(ticket.TN == 0){
-                               CpvAccess(_delayedLocalTicketRequests)->enq(entry);
-                               DEBUG(printf("[%d] Local Message request enqueued for SN %d sender %s recver %s \n",CmiMyPe(),entry->env->SN,entry->env->sender.toString(senderString),entry->env->recver.toString(recverString)));
-                               
-       //              _lockNewTicket = 0;
-//                             traceUserBracketEvent(33,_startTime,CkWallTimer());
-                               return;
-                       }
-               }       
-               //TODO: check for the case when an invalid ticket is returned
-               //TODO: check for OLD or RECEIVED TICKETS
-               entry->env->TN = ticket.TN;
-               CkAssert(entry->env->TN > 0);
-               DEBUG(printf("[%d] Local Message gets TN %d for SN %d sender %s recver %s \n",CmiMyPe(),entry->env->TN,entry->env->SN,entry->env->sender.toString(senderString),entry->env->recver.toString(recverString)));
-       
-               // sends a copy of the metadata to the buddy    
-               sendLocalMessageCopy(entry);
-               
-               DEBUG_MEM(CmiMemoryCheck());
 
-               // sets the unackedLocal flag and stores the message in the log
-               entry->unackedLocal = 1;
-               CpvAccess(_currentObj)->mlogData->addLogEntry(entry);
+               // sends the local message
+               _skipCldEnqueue(CmiMyPe(),env,_infoIdx);        
 
                DEBUG_MEM(CmiMemoryCheck());
        }else{
-               CkPrintf("[%d] Local message in team-based message logging %d to %d\n",CkMyPe(),CkMyPe(),entry->destPE);
-               DEBUG(printf("[%d] Local recver object in NULL \n",CmiMyPe()););
+               DEBUG(printf("[%d] Local recver object is NULL \n",CmiMyPe()););
        }
-       _lockNewTicket=0;
-//     traceUserBracketEvent(33,_startTime,CkWallTimer());
 };
 
-/**
- * Sends the metadata of a local message to its buddy.
- */
-void sendLocalMessageCopy(MlogEntry *entry){
-       LocalMessageLog msgLog;
-       msgLog.sender = entry->env->sender;
-       msgLog.recver = entry->env->recver;
-       msgLog.SN = entry->env->SN;
-       msgLog.TN = entry->env->TN;
-       msgLog.entry = entry;
-       msgLog.senderPE = CkMyPe();
-       
-       char recvString[100];
-       char senderString[100];
-       DEBUG(printf("[%d] Sending local message log from %s to %s SN %d TN %d to processor %d handler %d time %.6lf entry %p env %p \n",CkMyPe(),msgLog.sender.toString(senderString),msgLog.recver.toString(recvString),msgLog.SN,    msgLog.TN,getCheckPointPE(),_localMessageCopyHandlerIdx,CkWallTimer(),entry,entry->env));
+/****
+       The handler functions
+*****/
 
-#ifdef BUFFERED_LOCAL
-       countLocal++;
-       CpvAccess(_bufferedLocalMessageLogs)->enq(msgLog);
-       if(CpvAccess(_bufferedLocalMessageLogs)->length() >= _maxBufferedMessages){
-               sendBufferedLocalMessageCopy();
-       }else{
-               if(countClearBufferedLocalCalls < 10 && CpvAccess(_bufferedLocalMessageLogs)->length() == 1){
-                       lastBufferedLocalMessageCopyTime = CkWallTimer();
-                       CcdCallFnAfter( checkBufferedLocalMessageCopy ,NULL , BUFFER_TIME);
-                       countClearBufferedLocalCalls++;
-               }       
+bool fault_aware(CkObjID &recver){
+       switch(recver.type){
+               case TypeChare:
+                       return true;
+               case TypeMainChare:
+                       return false;
+               case TypeGroup:
+               case TypeNodeGroup:
+               case TypeArray:
+                       return true;
+               default:
+                       return false;
        }
-#else  
-       CmiSetHandler((void *)&msgLog,_localMessageCopyHandlerIdx);
-       
-       CmiSyncSend(getCheckPointPE(),sizeof(LocalMessageLog),(char *)&msgLog);
-#endif
-       DEBUG_MEM(CmiMemoryCheck());
 };
 
+/* Preprocesses a received message */
+int preProcessReceivedMessage(envelope *env, Chare **objPointer, MlogEntry **logEntryPointer){
+       DEBUG_NOW(char recverString[100]);
+       DEBUG_NOW(char senderString[100]);
+       DEBUG_MEM(CmiMemoryCheck());
+       int flag;
+       bool ticketSuccess;
 
-void sendBufferedLocalMessageCopy(){
-       int numberLogs = CpvAccess(_bufferedLocalMessageLogs)->length();
-       if(numberLogs == 0){
-               return;
-       }
-       countBuffered++;
-       int totalSize = sizeof(BufferedLocalLogHeader)+numberLogs*(sizeof(LocalMessageLog));
-       void *buf=CmiAlloc(totalSize);
-       BufferedLocalLogHeader *header = (BufferedLocalLogHeader *)buf;
-       header->numberLogs=numberLogs;
+       // getting the receiver object
+       CkObjID recver = env->recver;
 
-       DEBUG_MEM(CmiMemoryCheck());
-       DEBUG(printf("[%d] numberLogs in sendBufferedCopy = %d buf %p\n",CkMyPe(),numberLogs,buf));
-       
-       char *ptr = (char *)buf;
-       ptr = &ptr[sizeof(BufferedLocalLogHeader)];
-       
-       for(int i=0;i<numberLogs;i++){
-               LocalMessageLog log = CpvAccess(_bufferedLocalMessageLogs)->deq();
-               memcpy(ptr,&log,sizeof(LocalMessageLog));
-               ptr = &ptr[sizeof(LocalMessageLog)];
+       // checking for determinants bypass in message logging
+       if(env->flags & CK_BYPASS_DET_MLOG){
+               DEBUG(printf("[%d] Bypassing message sender %s recver %s \n",CkMyPe(),env->sender.toString(senderString), recver.toString(recverString)));
+               return 1;       
        }
 
-       CmiSetHandler(buf,_bufferedLocalMessageCopyHandlerIdx);
+       // checking if receiver is fault aware
+       if(!fault_aware(recver)){
+               CkPrintf("[%d] Receiver NOT fault aware\n",CkMyPe());
+               return 1;
+       }
 
-       CmiSyncSendAndFree(getCheckPointPE(),totalSize,(char *)buf);
-       DEBUG_MEM(CmiMemoryCheck());
-};
+       Chare *obj = (Chare *)recver.getObject();
+       *objPointer = obj;
+       if(obj == NULL){
+               int possiblePE = recver.guessPE();
+               if(possiblePE != CkMyPe()){
+                       int totalSize = env->getTotalsize();
+                       CmiSyncSend(possiblePE,totalSize,(char *)env);
+                       
+                       DEBUG_PE(0,printf("[%d] Forwarding message SN %d sender %s recver %s to %d\n",CkMyPe(),env->SN,env->sender.toString(senderString), recver.toString(recverString), possiblePE));
+               }else{
+                       // this is the case where a message is received and the object has not been initialized
+                       // we delayed the delivery of the message
+                       CqsEnqueue(CpvAccess(_outOfOrderMessageQueue),env);
+                       
+                       DEBUG_PE(0,printf("[%d] Message SN %d TN %d sender %s recver %s, receiver NOT found\n",CkMyPe(),env->SN,env->TN,env->sender.toString(senderString), recver.toString(recverString)));
+               }
+               return 0;
+       }
 
-void checkBufferedLocalMessageCopy(void *_dummy,double curWallTime){
-       countClearBufferedLocalCalls--;
-       if(countClearBufferedLocalCalls > 10){
-               CmiAbort("multiple checkBufferedLocalMessageCopy being called \n");
+       // checking if message comes from an old incarnation, message must be discarded
+       if(env->incarnation < CpvAccess(_incarnation)[env->getSrcPe()]){
+               CmiFree(env);
+               return 0;
        }
+
        DEBUG_MEM(CmiMemoryCheck());
-       DEBUG(printf("[%d] checkBufferedLocalMessageCopy \n",CkMyPe()));
-       if((curWallTime-lastBufferedLocalMessageCopyTime)*1000 > BUFFER_TIME && CpvAccess(_bufferedLocalMessageLogs)->length() > 0){
-               if(CpvAccess(_bufferedLocalMessageLogs)->length() > 0){
-                       sendBufferedLocalMessageCopy();
-//                     traceUserEvent(36);
-               }
+       DEBUG_PE(2,printf("[%d] Message received, sender = %s SN %d TN %d tProcessed %d for recver %s at %.6lf \n",CkMyPe(),env->sender.toString(senderString),env->SN,env->TN,obj->mlogData->tProcessed, recver.toString(recverString),CkWallTimer()));
+
+       // checking if message has already been processed, message must be discarded    
+       if(obj->mlogData->checkAndStoreSsn(env->sender,env->SN)){
+               DEBUG(printf("[%d] Message SN %d sender %s for recver %s being ignored\n",CkMyPe(),env->SN,env->sender.toString(senderString),recver.toString(recverString)));
+               CmiFree(env);
+               return 0;
        }
-       DEBUG_MEM(CmiMemoryCheck());
-}
-
-/****
-       The handler functions
-*****/
-
-
-inline bool _processTicketRequest(TicketRequest *ticketRequest,TicketReply *reply=NULL);
-/**
- *  If there are any delayed requests, process them first before 
- *  processing this request
- * */
-inline void _ticketRequestHandler(TicketRequest *ticketRequest){
-       DEBUG(printf("[%d] Ticket Request handler started \n",CkMyPe()));
-       double  _startTime = CkWallTimer();
-       if(CpvAccess(_delayedTicketRequests)->length() > 0){
-               retryTicketRequest(NULL,_startTime);
-       }
-       _processTicketRequest(ticketRequest);
-       CmiFree(ticketRequest);
-//     traceUserBracketEvent(21,_startTime,CkWallTimer());                     
-}
-/** Handler used for dealing with a bunch of ticket requests
- * from one processor. The replies are also bunched together
- * Does not use _ticketRequestHandler
- * */
-void _bufferedTicketRequestHandler(BufferedTicketRequestHeader *recvdHeader){
-       DEBUG(printf("[%d] Buffered Ticket Request handler started header %p\n",CkMyPe(),recvdHeader));
-       DEBUG_MEM(CmiMemoryCheck());
-       double _startTime = CkWallTimer();
-       if(CpvAccess(_delayedTicketRequests)->length() > 0){
-               retryTicketRequest(NULL,_startTime);
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-  int numRequests = recvdHeader->numberLogs;
-       char *msg = (char *)recvdHeader;
-       msg = &msg[sizeof(BufferedTicketRequestHeader)];
-       int senderPE=((TicketRequest *)msg)->senderPE;
-
-       
-       int totalSize = sizeof(BufferedTicketRequestHeader)+numRequests*sizeof(TicketReply);
-       void *buf = (void *)&(CpvAccess(_bufferTicketReply)[0]);
-       
-       char *ptr = (char *)buf;
-       BufferedTicketRequestHeader *header = (BufferedTicketRequestHeader *)ptr;
-       header->numberLogs = 0;
-
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       ptr = &ptr[sizeof(BufferedTicketRequestHeader)]; //ptr at which the ticket replies will be stored
-       
-       for(int i=0;i<numRequests;i++){
-               TicketRequest *request = (TicketRequest *)msg;
-               msg = &msg[sizeof(TicketRequest)];
-               bool replied = _processTicketRequest(request,(TicketReply *)ptr);
-
-               if(replied){
-                       //the ticket request has been processed and 
-                       //the reply will be stored in the ptr
-                       header->numberLogs++;
-                       ptr = &ptr[sizeof(TicketReply)];
-               }
-       }
-/*     if(header->numberLogs == 0){
-                       printf("[%d] *************** Not sending any replies to previous buffered ticketRequest \n",CkMyPe());
-       }*/
-
-       CmiSetHandler(buf,_bufferedTicketHandlerIdx);
-       CmiSyncSend(senderPE,totalSize,(char *)buf);
-       CmiFree(recvdHeader);
-//     traceUserBracketEvent(21,_startTime,CkWallTimer());                     
-       DEBUG_MEM(CmiMemoryCheck());
-};
-
-/**Process the ticket request. 
- * If it is processed and a reply is being sent 
- * by this processor return true
- * else return false.
- * If a reply buffer is specified put the reply into that
- * else send the reply
- * */
-inline bool _processTicketRequest(TicketRequest *ticketRequest,TicketReply *reply){
-
-/*     if(_lockNewTicket){
-               printf("ddeded %d\n",CkMyPe());
-               if(CmiIsImmediate(ticketRequest)){
-                       CmiSetHandler(ticketRequest, (CmiGetHandler(ticketRequest))^0x8000);
-               }
-               CmiSyncSend(CkMyPe(),sizeof(TicketRequest),(char *)ticketRequest);
-               
-       }else{
-               _lockNewTicket = 1;
-       }*/
-
-       DEBUG_MEM(CmiMemoryCheck());
-
-       // getting information from request
-       CkObjID sender = ticketRequest->sender;
-       CkObjID recver = ticketRequest->recver;
-       MCount SN = ticketRequest->SN;
-       MCount TN = ticketRequest->TN;
-       Chare *recverObj = (Chare *)recver.getObject();
-       
-       DEBUG(char recverName[100]);
-       DEBUG(recver.toString(recverName);)
-
-       if(recverObj == NULL){
-               int estPE = recver.guessPE();
-               if(estPE == CkMyPe() || estPE == -1){           
-                       //try to fulfill the request after some time
-                       char senderString[100];
-                       DEBUG(printf("[%d] Ticket request to %s SN %d from %s delayed estPE %d mesg %p\n",CkMyPe(),recverName, SN,sender.toString(senderString),estPE,ticketRequest));
-                       if(estPE == CkMyPe() && recver.type == TypeArray){
-                               CkArrayID aid(recver.data.array.id);            
-                               CkLocMgr *locMgr = aid.ckLocalBranch()->getLocMgr();
-                               DEBUG(printf("[%d] Object with delayed ticket request has home at %d\n",CkMyPe(),locMgr->homePe(recver.data.array.idx)));
-                       }
-                       TicketRequest *delayed = (TicketRequest*)CmiAlloc(sizeof(TicketRequest));
-                       *delayed = *ticketRequest;
-                       CpvAccess(_delayedTicketRequests)->enq(delayed);
-                       
-               }else{
-                       DEBUGRESTART(printf("[%d] Ticket request to %s SN %d needs to be forwarded estPE %d mesg %p\n",CkMyPe(),recver.toString(recverName), SN,estPE,ticketRequest));
-                       TicketRequest forward = *ticketRequest;
-                       CmiSetHandler(&forward,_ticketRequestHandlerIdx);
-                       CmiSyncSend(estPE,sizeof(TicketRequest),(char *)&forward);
-               }
-       DEBUG_MEM(CmiMemoryCheck());
-               return false; // if the receverObj does not exist the ticket request cannot have been 
-                             // processed successfully
-       }else{
-               char senderString[100];
-               
-               Ticket ticket;
-
-               // checking if the message is team local and if it has a ticket already assigned
-               if(teamSize > 1 && TN != 0){
-                       DEBUG(CkPrintf("[%d] Message has a ticket already assigned\n",CkMyPe()));
-                       ticket.TN = TN;
-                       recverObj->mlogData->verifyTicket(sender,SN,TN);
-               }
-
-               //check if a ticket for this has been already handed out to an object that used to be local but 
-               // is no longer so.. need for parallel restart
-               if(recverObj->mlogData->mapTable.numObjects() > 0){
-                       
-                       ticket.TN = recverObj->mlogData->searchRestoredLocalQ(ticketRequest->sender,ticketRequest->recver,ticketRequest->SN);
-               }
-               
-               if(ticket.TN == 0){
-                       ticket = recverObj->mlogData->next_ticket(sender,SN);
-               }
-               if(ticket.TN > recverObj->mlogData->tProcessed){
-                       ticket.state = NEW_TICKET;
-               }else{
-                       ticket.state = OLD_TICKET;
-               }
-               //TODO: check for the case when an invalid ticket is returned
-               if(ticket.TN == 0){
-                       DEBUG(printf("[%d] Ticket request to %s SN %d from %s delayed mesg %p\n",CkMyPe(),recverName, SN,sender.toString(senderString),ticketRequest));
-                       TicketRequest *delayed = (TicketRequest*)CmiAlloc(sizeof(TicketRequest));
-                       *delayed = *ticketRequest;
-                       CpvAccess(_delayedTicketRequests)->enq(delayed);
-                       return false;
-               }
-/*             if(ticket.TN < SN){ //error state this really should not happen
-                       recver.toString(recverName);
-                 printf("[%d] TN %d handed out to %s SN %d by %s sent to PE %d mesg %p at %.6lf\n",CkMyPe(),ticket.TN,sender.toString(senderString),SN,recverName,ticketRequest->senderPE,ticketRequest,CmiWallTimer());
-               }*/
-//             CkAssert(ticket.TN >= SN);
-               DEBUG(printf("[%d] TN %d handed out to %s SN %d by %s sent to PE %d mesg %p at %.6lf\n",CkMyPe(),ticket.TN,sender.toString(senderString),SN,recverName,ticketRequest->senderPE,ticketRequest,CmiWallTimer()));
-//             TicketReply *ticketReply = (TicketReply *)CmiAlloc(sizeof(TicketReply));
-    if(reply == NULL){ 
-                       //There is no reply buffer and the ticketreply is going to be 
-                       //sent immediately
-                       TicketReply ticketReply;
-                       ticketReply.request = *ticketRequest;
-                       ticketReply.ticket = ticket;
-                       ticketReply.recverPE = CkMyPe();
-                       CmiSetHandler(&ticketReply,_ticketHandlerIdx);
-//             CmiBecomeImmediate(&ticketReply);
-                       CmiSyncSend(ticketRequest->senderPE,sizeof(TicketReply),(char *)&ticketReply);
-        }else{ // Store ticket reply in the buffer provided
-                reply->request = *ticketRequest;
-                reply->ticket = ticket;
-                reply->recverPE = CkMyPe();
-                CmiSetHandler(reply,_ticketHandlerIdx); // not strictly necessary but will do that 
-                                                        // in case the ticket needs to be forwarded or something
-        }
-               DEBUG_MEM(CmiMemoryCheck());
-               return true;
-       }
-//     _lockNewTicket=0;
-};
-
-
-/**
- * @brief This function handles the ticket received after a request.
- */
-inline void _ticketHandler(TicketReply *ticketReply){
-
-       double _startTime = CkWallTimer();
-       DEBUG_MEM(CmiMemoryCheck());    
-       
-       char senderString[100];
-       CkObjID sender = ticketReply->request.sender;
-       CkObjID recver = ticketReply->request.recver;
-       
-       if(sender.guessPE() != CkMyPe()){
-               DEBUG(CkAssert(sender.guessPE()>= 0));
-               DEBUG(printf("[%d] TN %d forwarded to %s on PE %d \n",CkMyPe(),ticketReply->ticket.TN,sender.toString(senderString),sender.guessPE()));
-       //      printf("[%d] TN %d forwarded to %s on PE %d \n",CkMyPe(),ticketReply->ticket.TN,sender.toString(senderString),sender.guessPE());
-               ticketReply->ticket.state = ticketReply->ticket.state | FORWARDED_TICKET;
-               CmiSetHandler(ticketReply,_ticketHandlerIdx);
-#ifdef BUFFERED_REMOTE
-               //will be freed by the buffered ticket handler most of the time
-               //this might lead to a leak just after migration
-               //when the ticketHandler is directly used without going through the buffered handler
-               CmiSyncSend(sender.guessPE(),sizeof(TicketReply),(char *)ticketReply);
-#else
-               CmiSyncSendAndFree(sender.guessPE(),sizeof(TicketReply),(char *)ticketReply);
-#endif 
-       }else{
-               char recverName[100];
-               DEBUG(printf("[%d] TN %d received for %s SN %d from %s  time %.6lf \n",CkMyPe(),ticketReply->ticket.TN,sender.toString(senderString),ticketReply->request.SN,recver.toString(recverName),CmiWallTimer()));
-               MlogEntry *logEntry=NULL;
-               if(ticketReply->ticket.state & FORWARDED_TICKET){
-                       // Handle the case when you receive a forwarded message, We need to search through the message queue since the logEntry pointer is no longer valid
-                       DEBUG(printf("[%d] TN %d received for %s has been forwarded \n",CkMyPe(),ticketReply->ticket.TN,sender.toString(senderString)));
-                       Chare *senderObj = (Chare *)sender.getObject();
-                       if(senderObj){
-                               CkQ<MlogEntry *> *mlog = senderObj->mlogData->getMlog();
-                               for(int i=0;i<mlog->length();i++){
-                                       MlogEntry *tempEntry = (*mlog)[i];
-                                       if(tempEntry->env != NULL && ticketReply->request.sender == tempEntry->env->sender && ticketReply->request.recver == tempEntry->env->recver && ticketReply->request.SN == tempEntry->env->SN){
-                                               logEntry = tempEntry;
-                                               break;
-                                       }
-                               }
-                               if(logEntry == NULL){
-#ifdef BUFFERED_REMOTE
-#else
-                                       CmiFree(ticketReply);
-#endif                                 
-                                       return;
-                               }
-                       }else{
-                               CmiAbort("This processor thinks it should have the sender\n");
-                       }
-                       ticketReply->ticket.state ^= FORWARDED_TICKET;
-               }else{
-                       logEntry = ticketReply->request.logEntry;
-               }
-               if(logEntry->env->TN <= 0){
-                       //This logEntry has not received a TN earlier
-                       char recverString[100];
-                       logEntry->env->TN = ticketReply->ticket.TN;
-                       logEntry->env->setSrcPe(CkMyPe());
-                       if(ticketReply->ticket.state == NEW_TICKET){
-
-                               // if message is group local, we store its metadata in teamTable
-                               if(isTeamLocal(ticketReply->recverPE)){
-                                       //DEBUG_TEAM(CkPrintf("[%d] Storing meta data for intragroup message %u\n",CkMyPe(),ticketReply->request.SN);)
-                                       Chare *senderObj = (Chare *)sender.getObject();
-                                       SNToTicket *ticketRow = senderObj->mlogData->teamTable.get(recver);
-                                       if(ticketRow == NULL){
-                                               ticketRow = new SNToTicket();
-                                               senderObj->mlogData->teamTable.put(recver) = ticketRow; 
-                                       }
-                                       ticketRow->put(ticketReply->request.SN) = ticketReply->ticket;
-                               }
-
-                               DEBUG(printf("[%d] Message sender %s recver %s SN %d TN %d to processor %d env %p size %d \n",CkMyPe(),sender.toString(senderString),recver.toString(recverString), ticketReply->request.SN,ticketReply->ticket.TN,ticketReply->recverPE,logEntry->env,logEntry->env->getTotalsize()));
-                               if(ticketReply->recverPE != CkMyPe()){
-                                       generalCldEnqueue(ticketReply->recverPE,logEntry->env,logEntry->_infoIdx);
-                               }else{
-                                       //It is now a local message use the local message protocol
-                                       sendLocalMessageCopy(logEntry);
-                               }       
-                       }
-               }else{
-                       DEBUG(printf("[%d] Message sender %s recver %s SN %d already had TN %d received TN %d\n",CkMyPe(),sender.toString(senderString),recver.toString(recverName),ticketReply->request.SN,logEntry->env->TN,ticketReply->ticket.TN));
-               }
-               recver.updatePosition(ticketReply->recverPE);
-#ifdef BUFFERED_REMOTE
-#else
-               CmiFree(ticketReply);
-#endif
-       }
-       CmiMemoryCheck();
-
-//     traceUserBracketEvent(22,_startTime,CkWallTimer());     
-};
-
-/**
- * Message to handle the bunch of tickets 
- * that we get from one processor. We send 
- * the tickets to be handled one at a time
- * */
-
-void _bufferedTicketHandler(BufferedTicketRequestHeader *recvdHeader){
-       double _startTime=CmiWallTimer();
-       int numTickets = recvdHeader->numberLogs;
-       char *msg = (char *)recvdHeader;
-       msg = &msg[sizeof(BufferedTicketRequestHeader)];
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       TicketReply *_reply = (TicketReply *)msg;
-
-       
-       for(int i=0;i<numTickets;i++){
-               TicketReply *reply = (TicketReply *)msg;
-               _ticketHandler(reply);
-               
-               msg = &msg[sizeof(TicketReply)];
-       }
-       
-       CmiFree(recvdHeader);
-//     traceUserBracketEvent(22,_startTime,CkWallTimer());
-       DEBUG_MEM(CmiMemoryCheck());
-};
-
-/**
- * Stores the metadata of a local message from its buddy.
- */
-void _localMessageCopyHandler(LocalMessageLog *msgLog){
-       double _startTime = CkWallTimer();
-       
-       char senderString[100],recverString[100];
-       DEBUG(printf("[%d] Local Message log from processor %d sender %s recver %s TN %d handler %d time %.6lf \n",CkMyPe(),msgLog->PE,msgLog->sender.toString(senderString),msgLog->recver.toString(recverString),msgLog->TN,_localMessageAckHandlerIdx,CmiWallTimer()));
-/*     if(!fault_aware(msgLog->recver)){
-               CmiAbort("localMessageCopyHandler with non fault aware local message copy");
-       }*/
-       CpvAccess(_localMessageLog)->enq(*msgLog);
-       
-       LocalMessageLogAck ack;
-       ack.entry = msgLog->entry;
-       DEBUG(printf("[%d] About to send back ack \n",CkMyPe()));
-       CmiSetHandler(&ack,_localMessageAckHandlerIdx);
-       CmiSyncSend(msgLog->senderPE,sizeof(LocalMessageLogAck),(char *)&ack);
-       
-//     traceUserBracketEvent(23,_startTime,CkWallTimer());
-};
-
-void _bufferedLocalMessageCopyHandler(BufferedLocalLogHeader *recvdHeader,int freeHeader){
-       double _startTime = CkWallTimer();
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       int numLogs = recvdHeader->numberLogs;
-       char *msg = (char *)recvdHeader;
-
-       //piggy back the logs already stored on this processor
-       int numPiggyLogs = CpvAccess(_bufferedLocalMessageLogs)->length();
-       numPiggyLogs=0; //uncomment to turn off piggy backing of acks
-/*     if(numPiggyLogs > 0){
-               if((*CpvAccess(_bufferedLocalMessageLogs))[0].PE != getCheckPointPE()){
-                       CmiAssert(0);
-               }
-       }*/
-       DEBUG(printf("[%d] _bufferedLocalMessageCopyHandler numLogs %d numPiggyLogs %d\n",CmiMyPe(),numLogs,numPiggyLogs));
-       
-       int totalSize = sizeof(BufferedLocalLogHeader)+numLogs*sizeof(LocalMessageLogAck)+sizeof(BufferedLocalLogHeader)+numPiggyLogs*sizeof(LocalMessageLog);
-       void *buf = CmiAlloc(totalSize);
-       char *ptr = (char *)buf;
-       memcpy(ptr,msg,sizeof(BufferedLocalLogHeader));
-       
-       msg = &msg[sizeof(BufferedLocalLogHeader)];
-       ptr = &ptr[sizeof(BufferedLocalLogHeader)];
-
-       DEBUG_MEM(CmiMemoryCheck());
-       int PE;
-       for(int i=0;i<numLogs;i++){
-               LocalMessageLog *msgLog = (LocalMessageLog *)msg;
-               CpvAccess(_localMessageLog)->enq(*msgLog);
-               PE = msgLog->senderPE;
-               DEBUG(CmiAssert( PE == getCheckPointPE()));
-
-               LocalMessageLogAck *ack = (LocalMessageLogAck *)ptr;
-               ack->entry = msgLog->entry;
-               
-               msg = &msg[sizeof(LocalMessageLog)];
-               ptr = &ptr[sizeof(LocalMessageLogAck)];
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-
-       BufferedLocalLogHeader *piggyHeader = (BufferedLocalLogHeader *)ptr;
-       piggyHeader->numberLogs = numPiggyLogs;
-       ptr = &ptr[sizeof(BufferedLocalLogHeader)];
-       if(numPiggyLogs > 0){
-               countPiggy++;
-       }
-
-       for(int i=0;i<numPiggyLogs;i++){
-               LocalMessageLog log = CpvAccess(_bufferedLocalMessageLogs)->deq();
-               memcpy(ptr,&log,sizeof(LocalMessageLog));
-               ptr = &ptr[sizeof(LocalMessageLog)];
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       CmiSetHandler(buf,_bufferedLocalMessageAckHandlerIdx);
-       CmiSyncSendAndFree(PE,totalSize,(char *)buf);
-               
-/*     for(int i=0;i<CpvAccess(_localMessageLog)->length();i++){
-                       LocalMessageLog localLogEntry = (*CpvAccess(_localMessageLog))[i];
-                       if(!fault_aware(localLogEntry.recver)){
-                               CmiAbort("Non fault aware logEntry recver found while clearing old local logs");
-                       }
-       }*/
-       if(freeHeader){
-               CmiFree(recvdHeader);
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-//     traceUserBracketEvent(23,_startTime,CkWallTimer());
-}
-
-
-void _localMessageAckHandler(LocalMessageLogAck *ack){
-       
-       double _startTime = CkWallTimer();
-       
-       MlogEntry *entry = ack->entry;
-       if(entry == NULL){
-               CkExit();
-       }
-       entry->unackedLocal = 0;
-       envelope *env = entry->env;
-       char recverName[100];
-       char senderString[100];
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       DEBUG(printf("[%d] at start of local message ack handler for entry %p env %p\n",CkMyPe(),entry,env));
-       if(env == NULL)
-               return;
-       CkAssert(env->SN > 0);
-       CkAssert(env->TN > 0);
-       env->sender.toString(senderString);
-       DEBUG(printf("[%d] local message ack handler verified sender \n",CkMyPe()));
-       env->recver.toString(recverName);
-
-       DEBUG(printf("[%d] Local Message log ack received for message from %s to %s TN %d time %.6lf \n",CkMyPe(),env->sender.toString(senderString),env->recver.toString(recverName),env->TN,CkWallTimer()));
-       
-/*     
-       void *origMsg = EnvToUsr(env);
-       void *copyMsg = CkCopyMsg(&origMsg);
-       envelope *copyEnv = UsrToEnv(copyMsg);
-       entry->env = UsrToEnv(origMsg);*/
-
-//     envelope *copyEnv = copyEnvelope(env);
-
-       envelope *copyEnv = env;
-       copyEnv->localMlogEntry = entry;
-
-       DEBUG(printf("[%d] Local message copied response to ack \n",CkMyPe()));
-       if(CmiMyPe() != entry->destPE){
-               DEBUG(printf("[%d] Formerly remote message to PE %d converted to local\n",CmiMyPe(),entry->destPE));
-       }
-//     generalCldEnqueue(entry->destPE,copyEnv,entry->_infoIdx)
-       _skipCldEnqueue(CmiMyPe(),copyEnv,entry->_infoIdx);     
-       
-       
-#ifdef BUFFERED_LOCAL
-#else
-       CmiFree(ack);
-//     traceUserBracketEvent(24,_startTime,CkWallTimer());
-#endif
-       
-       DEBUG_MEM(CmiMemoryCheck());
-       DEBUG(printf("[%d] Local message log ack handled \n",CkMyPe()));
-}
-
-
-void _bufferedLocalMessageAckHandler(BufferedLocalLogHeader *recvdHeader){
-
-       double _startTime=CkWallTimer();
-       DEBUG_MEM(CmiMemoryCheck());
-
-       int numLogs = recvdHeader->numberLogs;
-       char *msg = (char *)recvdHeader;
-       msg = &msg[sizeof(BufferedLocalLogHeader)];
-
-       DEBUG(printf("[%d] _bufferedLocalMessageAckHandler numLogs %d \n",CmiMyPe(),numLogs));
-       
-       for(int i=0;i<numLogs;i++){
-               LocalMessageLogAck *ack = (LocalMessageLogAck *)msg;
-               _localMessageAckHandler(ack);
-               
-               msg = &msg[sizeof(LocalMessageLogAck)]; 
-       }
-
-       //deal with piggy backed local logs
-       BufferedLocalLogHeader *piggyHeader = (BufferedLocalLogHeader *)msg;
-       //printf("[%d] number of local logs piggied with ack %d \n",CkMyPe(),piggyHeader->numberLogs);
-       if(piggyHeader->numberLogs > 0){
-               _bufferedLocalMessageCopyHandler(piggyHeader,0);
-       }
-       
-       CmiFree(recvdHeader);
-       DEBUG_MEM(CmiMemoryCheck());
-//     traceUserBracketEvent(24,_startTime,CkWallTimer());
-}
-
-bool fault_aware(CkObjID &recver){
-       switch(recver.type){
-               case TypeChare:
-                       return false;
-               case TypeMainChare:
-                       return false;
-               case TypeGroup:
-               case TypeNodeGroup:
-               case TypeArray:
-                       return true;
-               default:
-                       return false;
-       }
-};
-
-int preProcessReceivedMessage(envelope *env,Chare **objPointer,MlogEntry **logEntryPointer){
-       char recverString[100];
-       char senderString[100];
-       
-       DEBUG_MEM(CmiMemoryCheck());
-       CkObjID recver = env->recver;
-       if(!fault_aware(recver))
-               return 1;
-
-
-       Chare *obj = (Chare *)recver.getObject();
-       *objPointer = obj;
-       if(obj == NULL){
-               int possiblePE = recver.guessPE();
-               if(possiblePE != CkMyPe()){
-                       int totalSize = env->getTotalsize();                    
-                       CmiSyncSend(possiblePE,totalSize,(char *)env);
-               }
-               return 0;
-       }
-
-
-       double _startTime = CkWallTimer();
-//env->sender.updatePosition(env->getSrcPe());
-       if(env->TN == obj->mlogData->tProcessed+1){
-               //the message that needs to be processed now
-               DEBUG(printf("[%d] Message SN %d TN %d sender %s recver %s being processed recvPointer %p\n",CkMyPe(),env->SN,env->TN,env->sender.toString(senderString), recver.toString(recverString),obj));
-               // once we find a message that we can process we put back all the messages in the out of order queue
-               // back into the main scheduler queue. 
-               if(env->sender.guessPE() == CkMyPe()){
-                       *logEntryPointer = env->localMlogEntry;
-               }
-       DEBUG_MEM(CmiMemoryCheck());
-               while(!CqsEmpty(CpvAccess(_outOfOrderMessageQueue))){
-                       void *qMsgPtr;
-                       CqsDequeue(CpvAccess(_outOfOrderMessageQueue),&qMsgPtr);
-                       envelope *qEnv = (envelope *)qMsgPtr;
-                       CqsEnqueueGeneral((Queue)CpvAccess(CsdSchedQueue),qEnv,CQS_QUEUEING_FIFO,qEnv->getPriobits(),(unsigned int *)qEnv->getPrioPtr());                       
-       DEBUG_MEM(CmiMemoryCheck());
-               }
-//             traceUserBracketEvent(25,_startTime,CkWallTimer());
-               //TODO: this might be a problem.. change made for leanMD
-//             CpvAccess(_currentObj) = obj;
-       DEBUG_MEM(CmiMemoryCheck());
-               return 1;
-       }
-       if(env->TN <= obj->mlogData->tProcessed){
-               //message already processed
-               DEBUG(printf("[%d] Message SN %d TN %d for recver %s being ignored tProcessed %d \n",CkMyPe(),env->SN,env->TN,recver.toString(recverString),obj->mlogData->tProcessed));
-//             traceUserBracketEvent(26,_startTime,CkWallTimer());
-       DEBUG_MEM(CmiMemoryCheck());
-               return 0;
-       }
-       //message that needs to be processed in the future
-
-//     DEBUG(printf("[%d] Early Message SN %d TN %d tProcessed %d for recver %s stored for future time %.6lf \n",CkMyPe(),env->SN,env->TN,obj->mlogData->tProcessed, recver.toString(recverString),CkWallTimer()));
-       //the message cant be processed now put it back in the out of order message Q, 
-       //It will be transferred to the main queue later
-       CqsEnqueue(CpvAccess(_outOfOrderMessageQueue),env);
-//             traceUserBracketEvent(27,_startTime,CkWallTimer());
-       DEBUG_MEM(CmiMemoryCheck());
-       
-       return 0;
+
+       // message can be processed at this point
+       DEBUG(printf("[%d] Message SN %d sender %s for recver %s being delivered\n",CkMyPe(),env->SN,env->sender.toString(senderString),recver.toString(recverString)));
+       return 1;
 }
 
 /**
  * @brief Updates a few variables once a message has been processed.
  */
-void postProcessReceivedMessage(Chare *obj,CkObjID &sender,MCount SN,MlogEntry *entry){
-       DEBUG(char senderString[100]);
-       if(obj){
-               if(sender.guessPE() == CkMyPe()){
-                       if(entry != NULL){
-                               entry->env = NULL;
-                       }
-               }
-               obj->mlogData->tProcessed++;
-/*             DEBUG(int qLength = CqsLength((Queue )CpvAccess(CsdSchedQueue)));               
-               DEBUG(printf("[%d] Message SN %d %s has been processed  tProcessed %d scheduler queue length %d\n",CkMyPe(),SN,obj->mlogData->objID.toString(senderString),obj->mlogData->tProcessed,qLength));         */
-//             CpvAccess(_currentObj)= NULL;
-       }
-       DEBUG_MEM(CmiMemoryCheck());
+void postProcessReceivedMessage(Chare *obj, CkObjID &sender, MCount SN, MlogEntry *entry){
 }
 
 /***
        Helpers for the handlers and message logging methods
 ***/
 
-void generalCldEnqueue(int destPE,envelope *env,int _infoIdx){
+void generalCldEnqueue(int destPE, envelope *env, int _infoIdx){
 //     double _startTime = CkWallTimer();
        if(env->recver.type != TypeNodeGroup){
        //This repeats a step performed in skipCldEnq for messages sent to
@@ -1382,58 +794,6 @@ void generalCldEnqueue(int destPE,envelope *env,int _infoIdx){
 //     traceUserBracketEvent(22,_startTime,CkWallTimer());
 }
 //extern "C" int CmiGetNonLocalLength();
-/** This method is used to retry the ticket requests
- * that had been queued up earlier
- * */
-
-int calledRetryTicketRequest=0;
-
-void retryTicketRequestTimer(void *_dummy,double _time){
-               calledRetryTicketRequest=0;
-               retryTicketRequest(_dummy,_time);
-}
-
-void retryTicketRequest(void *_dummy,double curWallTime){      
-       double start = CkWallTimer();
-       DEBUG_MEM(CmiMemoryCheck());
-       int length = CpvAccess(_delayedTicketRequests)->length();
-       for(int i=0;i<length;i++){
-               TicketRequest *ticketRequest = CpvAccess(_delayedTicketRequests)->deq();
-               if(ticketRequest){
-                       char senderString[100],recverString[100];
-                       DEBUGRESTART(printf("[%d] RetryTicketRequest for ticket %p sender %s recver %s SN %d at %.6lf \n",CkMyPe(),ticketRequest,ticketRequest->sender.toString(senderString),ticketRequest->recver.toString(recverString), ticketRequest->SN, CmiWallTimer()));
-                       DEBUG_MEM(CmiMemoryCheck());
-                       _processTicketRequest(ticketRequest);
-                 CmiFree(ticketRequest);
-                       DEBUG_MEM(CmiMemoryCheck());
-               }       
-       }       
-       for(int i=0;i<CpvAccess(_delayedLocalTicketRequests)->length();i++){
-               MlogEntry *entry = CpvAccess(_delayedLocalTicketRequests)->deq();
-               ticketLogLocalMessage(entry);
-       }
-       int qLength = CqsLength((Queue )CpvAccess(CsdSchedQueue));
-//     int converse_qLength = CmiGetNonLocalLength();
-       
-//     DEBUG(printf("[%d] Total RetryTicketRequest took %.6lf scheduler queue length %d converse queue length %d \n",CkMyPe(),CkWallTimer()-start,qLength,converse_qLength));
-
-/*     PingMsg pingMsg;
-       pingMsg.PE = CkMyPe();
-       CmiSetHandler(&pingMsg,_pingHandlerIdx);
-       if(CkMyPe() == 0 || CkMyPe() == CkNumPes() -1){
-               for(int i=0;i<CkNumPes();i++){
-                       if(i != CkMyPe()){
-                               CmiSyncSend(i,sizeof(PingMsg),(char *)&pingMsg);
-                       }
-               }
-       }*/     
-       //TODO: change this back to 100
-       if(calledRetryTicketRequest == 0){
-               CcdCallFnAfter(retryTicketRequestTimer,NULL,500);       
-               calledRetryTicketRequest =1;
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-}
 
 void _pingHandler(CkPingMsg *msg){
        printf("[%d] Received Ping from %d\n",CkMyPe(),msg->PE);
@@ -1446,7 +806,6 @@ void _pingHandler(CkPingMsg *msg){
                Pack all the data on a processor and send it to the buddy periodically
                Also used to throw away message logs
 *****************************************************************************/
-CkVec<TProcessedLog> processedTicketLog;
 void buildProcessedTicketLog(void *data,ChareMlogData *mlogData);
 void clearUpMigratedRetainedLists(int PE);
 
@@ -1468,23 +827,31 @@ void checkpointAlarm(void *_dummy,double curWallTime){
 };
 
 void _checkpointRequestHandler(CheckpointRequest *request){
-       startMlogCheckpoint(NULL,CmiWallTimer());       
+       startMlogCheckpoint(NULL,CmiWallTimer());
 }
 
-void startMlogCheckpoint(void *_dummy,double curWallTime){
+/**
+ * @brief Starts the checkpoint phase after migration.
+ */
+void startMlogCheckpoint(void *_dummy, double curWallTime){
        double _startTime = CkWallTimer();
+
+       // increasing the checkpoint counter
        checkpointCount++;
-/*     if(checkpointCount == 3 && CmiMyPe() == 4 && restarted == 0){
-               kill(getpid(),SIGKILL);
-       }*/
-       if(CmiNumPes() < 256 || CmiMyPe() == 0){
+       
+#if DEBUG_CHECKPOINT
+       if(CmiMyPe() == 0){
                printf("[%d] starting checkpoint at %.6lf CmiTimer %.6lf \n",CkMyPe(),CmiWallTimer(),CmiTimer());
        }
-       PUP::sizer psizer;
+#endif
+
        DEBUG_MEM(CmiMemoryCheck());
 
+       PUP::sizer psizer;
        psizer | checkpointCount;
-       
+       for(int i=0; i<CmiNumPes(); i++){
+               psizer | CpvAccess(_incarnation)[i];
+       }
        CkPupROData(psizer);
        DEBUG_MEM(CmiMemoryCheck());
        CkPupGroupData(psizer,CmiTrue);
@@ -1500,13 +867,14 @@ void startMlogCheckpoint(void *_dummy,double curWallTime){
        CheckPointDataMsg *chkMsg = (CheckPointDataMsg *)msg;
        chkMsg->PE = CkMyPe();
        chkMsg->dataSize = dataSize;
-
        
        char *buf = &msg[sizeof(CheckPointDataMsg)];
-       PUP::toMem pBuf(buf);   
+       PUP::toMem pBuf(buf);
 
        pBuf | checkpointCount;
-       
+       for(int i=0; i<CmiNumPes(); i++){
+               pBuf | CpvAccess(_incarnation)[i];
+       }
        CkPupROData(pBuf);
        CkPupGroupData(pBuf,CmiTrue);
        CkPupNodeGroupData(pBuf,CmiTrue);
@@ -1515,15 +883,19 @@ void startMlogCheckpoint(void *_dummy,double curWallTime){
        unAckedCheckpoint=1;
        CmiSetHandler(msg,_storeCheckpointHandlerIdx);
        CmiSyncSendAndFree(getCheckPointPE(),totalSize,msg);
-       
-       /*
-               Store the highest Ticket number processed for each chare on this processor
-       */
-       processedTicketLog.removeAll();
-       forAllCharesDo(buildProcessedTicketLog,(void *)&processedTicketLog);
-       if(CmiNumPes() < 256 || CmiMyPe() == 0){
+
+#if DEBUG_CHECKPOINT
+       if(CmiMyPe() == 0){
                printf("[%d] finishing checkpoint at %.6lf CmiTimer %.6lf with dataSize %d\n",CkMyPe(),CmiWallTimer(),CmiTimer(),dataSize);
        }
+#endif
+
+#if COLLECT_STATS_MEMORY
+       CkPrintf("[%d] CKP=%d BUF_DET=%d STO_DET=%d MSG_LOG=%d\n",CkMyPe(),totalSize,bufferedDetsSize*sizeof(Determinant),storedDetsSize*sizeof(Determinant),msgLogSize);
+       msgLogSize = 0;
+       bufferedDetsSize = 0;
+       storedDetsSize = 0;
+#endif
 
        if(CkMyPe() ==  0 && onGoingLoadBalancing==0 ){
                lastCompletedAlarm = curWallTime;
@@ -1532,29 +904,20 @@ void startMlogCheckpoint(void *_dummy,double curWallTime){
        traceUserBracketEvent(28,_startTime,CkWallTimer());
 };
 
-void buildProcessedTicketLog(void *data,ChareMlogData *mlogData){
-       CkVec<TProcessedLog> *log = (   CkVec<TProcessedLog> *)data;
-       TProcessedLog logEntry;
-       logEntry.recver = mlogData->objID;
-       logEntry.tProcessed = mlogData->tProcessed;
-       log->push_back(logEntry);
-       char objString[100];
-       DEBUG(printf("[%d] Tickets lower than %d to be thrown away for %s \n",CkMyPe(),logEntry.tProcessed,logEntry.recver.toString(objString)));
-}
 
 class ElementPacker : public CkLocIterator {
 private:
        CkLocMgr *locMgr;
        PUP::er &p;
 public:
-               ElementPacker(CkLocMgr* mgr_, PUP::er &p_):locMgr(mgr_),p(p_){};
-               void addLocation(CkLocation &loc) {
-                       CkArrayIndex idx=loc.getIndex();
-                       CkGroupID gID = locMgr->ckGetGroupID();
-                       p|gID;      // store loc mgr's GID as well for easier restore
-                       p|idx;
-                       p|loc;
-    }
+       ElementPacker(CkLocMgr* mgr_, PUP::er &p_):locMgr(mgr_),p(p_){};
+       void addLocation(CkLocation &loc) {
+               CkArrayIndexMax idx=loc.getIndex();
+               CkGroupID gID = locMgr->ckGetGroupID();
+               p|gID;      // store loc mgr's GID as well for easier restore
+               p|idx;
+               p|loc;
+       }
 };
 
 /**
@@ -1584,7 +947,7 @@ void pupArrayElementsSkip(PUP::er &p, CmiBool create, MigrationRecord *listToSki
        
                for (int i=0; i<numElements; i++) {
                        CkGroupID gID;
-                       CkArrayIndex idx;
+                       CkArrayIndexMax idx;
                        p|gID;
                p|idx;
                        int flag=0;
@@ -1629,13 +992,11 @@ void writeCheckpointToDisk(int size,char *chkpt){
        unlink(fName);
 
        rename(fNameTemp,fName);
-       
 }
 
 //handler that receives the checkpoint from a processor
 //it stores it and acks it
 void _storeCheckpointHandler(char *msg){
-       
        double _startTime=CkWallTimer();
                
        CheckPointDataMsg *chkMsg = (CheckPointDataMsg *)msg;
@@ -1656,13 +1017,6 @@ void _storeCheckpointHandler(char *msg){
        CpvAccess(_storedCheckpointData)->bufSize = chkMsg->dataSize;
        CpvAccess(_storedCheckpointData)->PE = sendingPE;
 
-#ifdef CHECKPOINT_DISK
-       //store the checkpoint on disk
-       writeCheckpointToDisk(chkMsg->dataSize,chkpt);
-       CpvAccess(_storedCheckpointData)->buf = NULL;
-       CmiFree(msg);
-#endif
-
        int count=0;
        for(int j=migratedNoticeList.size()-1;j>=0;j--){
                if(migratedNoticeList[j].fromPE == sendingPE){
@@ -1684,143 +1038,22 @@ void _storeCheckpointHandler(char *msg){
        CmiSetHandler(&ackMsg,_checkpointAckHandlerIdx);
        CmiSyncSend(sendingPE,sizeof(CheckPointAck),(char *)&ackMsg);
        
-       
-       
        traceUserBracketEvent(29,_startTime,CkWallTimer());
 };
 
 
-void sendRemoveLogRequests(){
-       double _startTime = CkWallTimer();      
-       //send out the messages asking senders to throw away message logs below a certain ticket number
-       /*
-               The remove log request message looks like
-               |RemoveLogRequest||List of TProcessedLog|
-       */
-       int totalSize = sizeof(RemoveLogRequest)+processedTicketLog.size()*sizeof(TProcessedLog);
-       char *requestMsg = (char *)CmiAlloc(totalSize);
-       RemoveLogRequest *request = (RemoveLogRequest *)requestMsg;
-       request->PE = CkMyPe();
-       request->numberObjects = processedTicketLog.size();
-       char *listProcessedLogs = &requestMsg[sizeof(RemoveLogRequest)];
-       memcpy(listProcessedLogs,(char *)processedTicketLog.getVec(),processedTicketLog.size()*sizeof(TProcessedLog));
-       CmiSetHandler(requestMsg,_removeProcessedLogHandlerIdx);
-       
-       DEBUG_MEM(CmiMemoryCheck());
-       for(int i=0;i<CkNumPes();i++){
-               CmiSyncSend(i,totalSize,requestMsg);
-       }
-       CmiFree(requestMsg);
-
-       clearUpMigratedRetainedLists(CmiMyPe());
-       //TODO: clear ticketTable
-       
-       traceUserBracketEvent(30,_startTime,CkWallTimer());
-       DEBUG_MEM(CmiMemoryCheck());
-}
-
-
 void _checkpointAckHandler(CheckPointAck *ackMsg){
        DEBUG_MEM(CmiMemoryCheck());
        unAckedCheckpoint=0;
-       DEBUG(printf("[%d] CheckPoint Acked from PE %d with size %d onGoingLoadBalancing %d \n",CkMyPe(),ackMsg->PE,ackMsg->dataSize,onGoingLoadBalancing));
+       DEBUGLB(printf("[%d] CheckPoint Acked from PE %d with size %d onGoingLoadBalancing %d \n",CkMyPe(),ackMsg->PE,ackMsg->dataSize,onGoingLoadBalancing));
        DEBUGLB(CkPrintf("[%d] ACK HANDLER with %d\n",CkMyPe(),onGoingLoadBalancing));  
        if(onGoingLoadBalancing){
                onGoingLoadBalancing = 0;
                finishedCheckpointLoadBalancing();
-       }else{
-               sendRemoveLogRequests();
        }
        CmiFree(ackMsg);
-       
-};
-
-void removeProcessedLogs(void *_data,ChareMlogData *mlogData){
-       DEBUG_MEM(CmiMemoryCheck());
-       CmiMemoryCheck();
-       char *data = (char *)_data;
-       RemoveLogRequest *request = (RemoveLogRequest *)data;
-       TProcessedLog *list = (TProcessedLog *)(&data[sizeof(RemoveLogRequest)]);
-       CkQ<MlogEntry *> *mlog = mlogData->getMlog();
-
-       int count=0;
-       for(int i=0;i<mlog->length();i++){
-               MlogEntry *logEntry = mlog->deq();
-               int match=0;
-               for(int j=0;j<request->numberObjects;j++){
-                       if(logEntry->env == NULL || (logEntry->env->recver == list[j].recver && logEntry->env->TN > 0 && logEntry->env->TN < list[j].tProcessed && logEntry->unackedLocal != 1)){
-                               //this log Entry should be removed
-                               match = 1;
-                               break;
-                       }
-               }
-               char senderString[100],recverString[100];
-//             DEBUG(CkPrintf("[%d] Message sender %s recver %s TN %d removed %d PE %d\n",CkMyPe(),logEntry->env->sender.toString(senderString),logEntry->env->recver.toString(recverString),logEntry->env->TN,match,request->PE));
-               if(match){
-                       count++;
-                       delete logEntry;
-               }else{
-                       mlog->enq(logEntry);
-               }
-       }
-       if(count > 0){
-               char nameString[100];
-               DEBUG(printf("[%d] Removed %d processed Logs for %s\n",CkMyPe(),count,mlogData->objID.toString(nameString)));
-       }
-       DEBUG_MEM(CmiMemoryCheck());
-       CmiMemoryCheck();
-}
-
-void _removeProcessedLogHandler(char *requestMsg){
-       double start = CkWallTimer();
-       forAllCharesDo(removeProcessedLogs,requestMsg);
-       // printf("[%d] Removing Processed logs took %.6lf \n",CkMyPe(),CkWallTimer()-start);
-       RemoveLogRequest *request = (RemoveLogRequest *)requestMsg;
-       DEBUG(printf("[%d] Removing Processed logs for proc %d took %.6lf \n",CkMyPe(),request->PE,CkWallTimer()-start));
-       //this assumes the buddy relationship between processors is symmetric. TODO:remove this assumption later
-       if(request->PE == getCheckPointPE()){
-               TProcessedLog *list = (TProcessedLog *)(&requestMsg[sizeof(RemoveLogRequest)]);
-               CkQ<LocalMessageLog> *localQ = CpvAccess(_localMessageLog);
-               CkQ<LocalMessageLog> *tempQ = new CkQ<LocalMessageLog>;
-               int count=0;
-/*             DEBUG(for(int j=0;j<request->numberObjects;j++){)
-               DEBUG(char nameString[100];)
-                       DEBUG(printf("[%d] Remove local message logs for %s with TN less than %d\n",CkMyPe(),list[j].recver.toString(nameString),list[j].tProcessed));
-               DEBUG(})*/
-               for(int i=0;i<localQ->length();i++){
-                       LocalMessageLog localLogEntry = (*localQ)[i];
-                       if(!fault_aware(localLogEntry.recver)){
-                               CmiAbort("Non fault aware logEntry recver found while clearing old local logs");
-                       }
-                       bool keep = true;
-                       for(int j=0;j<request->numberObjects;j++){                              
-                               if(localLogEntry.recver == list[j].recver && localLogEntry.TN > 0 && localLogEntry.TN < list[j].tProcessed){
-                                       keep = false;
-                                       break;
-                               }
-                       }       
-                       if(keep){
-                               tempQ->enq(localLogEntry);
-                       }else{
-                               count++;
-                       }
-               }
-               delete localQ;
-               CpvAccess(_localMessageLog) = tempQ;
-               DEBUG(printf("[%d] %d Local logs for proc %d deleted on buddy \n",CkMyPe(),count,request->PE));
-       }
-
-       /*
-               Clear up the retainedObjectList and the migratedNoticeList that were created during load balancing
-       */
-       CmiMemoryCheck();
-       clearUpMigratedRetainedLists(request->PE);
-       
-       traceUserBracketEvent(20,start,CkWallTimer());
-       CmiFree(requestMsg);    
 };
 
-
 void clearUpMigratedRetainedLists(int PE){
        int count=0;
        CmiMemoryCheck();
@@ -1859,255 +1092,25 @@ void clearUpMigratedRetainedLists(int PE){
  * It sets the restart flag and contacts the buddy
  * processor to get the latest checkpoint.
  */
-void CkMlogRestart(const char * dummy, CkArgMsg * dummyMsg){
-       RestartRequest msg;
-
-       fprintf(stderr,"[%d] Restart started at %.6lf \n",CkMyPe(),CmiWallTimer());
-
-       // setting the restart flag
-       _restartFlag = 1;
-
-       // if we are using team-based message logging, all members of the group have to be restarted
-       if(teamSize > 1){
-               for(int i=(CkMyPe()/teamSize)*teamSize; i<((CkMyPe()/teamSize)+1)*teamSize; i++){
-                       if(i != CkMyPe() && i < CkNumPes()){
-                               // sending a message to the team member
-                               msg.PE = CkMyPe();
-                           CmiSetHandler(&msg,_restartHandlerIdx);
-                           CmiSyncSend(i,sizeof(RestartRequest),(char *)&msg);
-                       }
-               }
-       }
-
-       // requesting the latest checkpoint from its buddy
-       msg.PE = CkMyPe();
-       CmiSetHandler(&msg,_getCheckpointHandlerIdx);
-       CmiSyncSend(getCheckPointPE(),sizeof(RestartRequest),(char *)&msg);
-};
-
-/**
- * Function to restart this processor.
- * The handler is invoked by a member of its same team in message logging.
- */
-void _restartHandler(RestartRequest *restartMsg){
-       int i;
-       int numGroups = CkpvAccess(_groupIDTable)->size();
-       RestartRequest msg;
-       
-       fprintf(stderr,"[%d] Restart-team started at %.6lf \n",CkMyPe(),CmiWallTimer());
-
-    // setting the restart flag
-       _restartFlag = 1;
-
-       // flushing all buffers
-       //TEST END
-/*     CkPrintf("[%d] HERE numGroups = %d\n",CkMyPe(),numGroups);
-       CKLOCMGR_LOOP(mgr->flushAllRecs(););    
-       for(int i=0;i<numGroups;i++){
-       CkGroupID gID = (*CkpvAccess(_groupIDTable))[i];
-               IrrGroup *obj = CkpvAccess(_groupTable)->find(gID).getObj();
-               obj->flushStates();
-               obj->ckJustMigrated();
-       }*/
-
-    // requesting the latest checkpoint from its buddy
-       msg.PE = CkMyPe();
-       CmiSetHandler(&msg,_getRestartCheckpointHandlerIdx);
-       CmiSyncSend(getCheckPointPE(),sizeof(RestartRequest),(char *)&msg);
-}
-
-
-/**
- * Gets the stored checkpoint but calls another function in the sender.
- */
-void _getRestartCheckpointHandler(RestartRequest *restartMsg){
-
-       // retrieving the stored checkpoint
-       StoredCheckpoint *storedChkpt = CpvAccess(_storedCheckpointData);
-
-       // making sure it is its buddy who is requesting the checkpoint
-       CkAssert(restartMsg->PE == storedChkpt->PE);
-
-       storedRequest = restartMsg;
-       verifyAckTotal = 0;
-
-       for(int i=0;i<migratedNoticeList.size();i++){
-               if(migratedNoticeList[i].fromPE == restartMsg->PE){
-//                     if(migratedNoticeList[i].ackFrom == 0 && migratedNoticeList[i].ackTo == 0){
-                       if(migratedNoticeList[i].ackFrom == 0){
-                               //need to verify if the object actually exists .. it might not
-                               //have been acked but it might exist on it
-                               VerifyAckMsg msg;
-                               msg.migRecord = migratedNoticeList[i];
-                               msg.index = i;
-                               msg.fromPE = CmiMyPe();
-                               CmiPrintf("[%d] Verify  gid %d idx %s from proc %d\n",CmiMyPe(),migratedNoticeList[i].gID.idx,idx2str(migratedNoticeList[i].idx),migratedNoticeList[i].toPE);
-                               CmiSetHandler(&msg,_verifyAckRequestHandlerIdx);
-                               CmiSyncSend(migratedNoticeList[i].toPE,sizeof(VerifyAckMsg),(char *)&msg);
-                               verifyAckTotal++;
-                       }
-               }
-       }
-
-       // sending the checkpoint back to its buddy     
-       if(verifyAckTotal == 0){
-               sendCheckpointData(MLOG_RESTARTED);
-       }
-       verifyAckCount = 0;
-}
-
-/**
- * Receives the checkpoint coming from its buddy. This is the case of restart for one team member that did not crash.
- */
-void _recvRestartCheckpointHandler(char *_restartData){
-       RestartProcessorData *restartData = (RestartProcessorData *)_restartData;
-       MigrationRecord *migratedAwayElements;
-
-       globalLBID = restartData->lbGroupID;
-       
-       restartData->restartWallTime *= 1000;
-       adjustChkptPeriod = restartData->restartWallTime/(double) chkptPeriod - floor(restartData->restartWallTime/(double) chkptPeriod);
-       adjustChkptPeriod = (double )chkptPeriod*(adjustChkptPeriod);
-       if(adjustChkptPeriod < 0) adjustChkptPeriod = 0;
-
-       
-       printf("[%d] Team Restart Checkpointdata received from PE %d at %.6lf with checkpointSize %d\n",CkMyPe(),restartData->PE,CmiWallTimer(),restartData->checkPointSize);
-       char *buf = &_restartData[sizeof(RestartProcessorData)];
-       
-       if(restartData->numMigratedAwayElements != 0){
-               migratedAwayElements = new MigrationRecord[restartData->numMigratedAwayElements];
-               memcpy(migratedAwayElements,buf,restartData->numMigratedAwayElements*sizeof(MigrationRecord));
-               printf("[%d] Number of migratedaway elements %d\n",CmiMyPe(),restartData->numMigratedAwayElements);
-               buf = &buf[restartData->numMigratedAwayElements*sizeof(MigrationRecord)];
-       }
-
-       // turning on the team recovery flag
-       forAllCharesDo(setTeamRecovery,NULL);
-       
-       PUP::fromMem pBuf(buf);
-       pBuf | checkpointCount;
-       CkPupROData(pBuf);
-       CkPupGroupData(pBuf,CmiFalse);
-       CkPupNodeGroupData(pBuf,CmiFalse);
-       pupArrayElementsSkip(pBuf,CmiFalse,NULL);
-       CkAssert(pBuf.size() == restartData->checkPointSize);
-       printf("[%d] Restart Objects created from CheckPointData at %.6lf \n",CkMyPe(),CmiWallTimer());
-       
-       // turning off the team recovery flag
-       forAllCharesDo(unsetTeamRecovery,NULL);
-
-       // initializing a few variables for handling local messages
-       forAllCharesDo(initializeRestart,NULL);
-       
-       //store the restored local message log in a vector
-       buf = &buf[restartData->checkPointSize];        
-       for(int i=0;i<restartData->numLocalMessages;i++){
-               LocalMessageLog logEntry;
-               memcpy(&logEntry,buf,sizeof(LocalMessageLog));
-               
-               Chare *recverObj = (Chare *)logEntry.recver.getObject();
-               if(recverObj!=NULL){
-                       recverObj->mlogData->addToRestoredLocalQ(&logEntry);
-                       recverObj->mlogData->receivedTNs->push_back(logEntry.TN);
-                       char senderString[100];
-                       char recverString[100];
-                       DEBUGRESTART(printf("[%d] Received local message log sender %s recver %s SN %d  TN %d\n",CkMyPe(),logEntry.sender.toString(senderString),logEntry.recver.toString(recverString),logEntry.SN,logEntry.TN));
-               }else{
-//                     DEBUGRESTART(printf("Object receiving local message doesnt exist on restarted processor .. ignoring it"));
-               }
-               buf = &buf[sizeof(LocalMessageLog)];
-       }
-
-       forAllCharesDo(sortRestoredLocalMsgLog,NULL);
-       CmiFree(_restartData);  
-
-       /*HERE _initDone();
-
-       getGlobalStep(globalLBID);
-       
-       countUpdateHomeAcks = 0;
-       RestartRequest updateHomeRequest;
-       updateHomeRequest.PE = CmiMyPe();
-       CmiSetHandler (&updateHomeRequest,_updateHomeRequestHandlerIdx);
-       for(int i=0;i<CmiNumPes();i++){
-               if(i != CmiMyPe()){
-                       CmiSyncSend(i,sizeof(RestartRequest),(char *)&updateHomeRequest);
-               }
-       }
-*/
-
-
-       // Send out the request to resend logged messages to all other processors
-       CkVec<TProcessedLog> objectVec;
-       forAllCharesDo(createObjIDList, (void *)&objectVec);
-       int numberObjects = objectVec.size();
-       
-       /*
-               resendMsg layout |ResendRequest|Array of TProcessedLog|
-       */
-       int totalSize = sizeof(ResendRequest)+numberObjects*sizeof(TProcessedLog);
-       char *resendMsg = (char *)CmiAlloc(totalSize);  
-
-       ResendRequest *resendReq = (ResendRequest *)resendMsg;
-       resendReq->PE =CkMyPe(); 
-       resendReq->numberObjects = numberObjects;
-       char *objList = &resendMsg[sizeof(ResendRequest)];
-       memcpy(objList,objectVec.getVec(),numberObjects*sizeof(TProcessedLog));
-       
-
-       /* test for parallel restart migrate away object**/
-//     if(parallelRestart){
-//             distributeRestartedObjects();
-//             printf("[%d] Redistribution of objects done at %.6lf \n",CkMyPe(),CmiWallTimer());
-//     }
-       
-       /*      To make restart work for load balancing.. should only
-       be used when checkpoint happens along with load balancing
-       **/
-//     forAllCharesDo(resumeFromSyncRestart,NULL);
-
-       CentralLB *lb = (CentralLB *)CkpvAccess(_groupTable)->find(globalLBID).getObj();
-       CpvAccess(_currentObj) = lb;
-       lb->ReceiveDummyMigration(restartDecisionNumber);
-
-       sleep(10);
-       
-       CmiSetHandler(resendMsg,_resendMessagesHandlerIdx);
-       for(int i=0;i<CkNumPes();i++){
-               if(i != CkMyPe()){
-                       CmiSyncSend(i,totalSize,resendMsg);
-               }       
-       }
-       _resendMessagesHandler(resendMsg);
-
-}
+void CkMlogRestart(const char * dummy, CkArgMsg * dummyMsg){
+       RestartRequest msg;
 
+       fprintf(stderr,"[%d] Restart started at %.6lf \n",CkMyPe(),CmiWallTimer());
 
-void CkMlogRestartDouble(void *,double){
-       CkMlogRestart(NULL,NULL);
-};
+       // setting the restart flag
+       _restartFlag = 1;
+       _numRestartResponses = 0;
 
-//TML: restarting from local (group) failure
-void CkMlogRestartLocal(){
-    CkMlogRestart(NULL,NULL);
+       // requesting the latest checkpoint from its buddy
+       msg.PE = CkMyPe();
+       CmiSetHandler(&msg,_getCheckpointHandlerIdx);
+       CmiSyncSend(getCheckPointPE(),sizeof(RestartRequest),(char *)&msg);
 };
 
-
-void readCheckpointFromDisk(int size,char *buf){
-       char fName[100];
-       sprintf(fName,"%s/mlogCheckpoint%d",checkpointDirectory,CkMyPe());
-
-       int fd = open(fName,O_RDONLY);
-       int count=0;
-       while(count < size){
-               count += read(fd,&buf[count],size-count);
-       }
-       close(fd);
-       
+void CkMlogRestartDouble(void *,double){
+       CkMlogRestart(NULL,NULL);
 };
 
-
-
 /**
  * Gets the stored checkpoint for its buddy processor.
  */
@@ -2142,7 +1145,7 @@ void _getCheckpointHandler(RestartRequest *restartMsg){
 
        // sending the checkpoint back to its buddy     
        if(verifyAckTotal == 0){
-               sendCheckpointData(MLOG_CRASHED);
+               sendCheckpointData();
        }
        verifyAckCount = 0;
 }
@@ -2182,33 +1185,27 @@ void _verifyAckHandler(VerifyAckMsg *verifyReply){
        verifyAckCount++;
        CmiPrintf("[%d] VerifyReply received %d for  gid %d idx %s from proc %d\n",CmiMyPe(),migratedNoticeList[index].ackTo, migratedNoticeList[index].gID,idx2str(migratedNoticeList[index].idx),migratedNoticeList[index].toPE);
        if(verifyAckCount == verifyAckTotal){
-               sendCheckpointData(MLOG_CRASHED);
+               sendCheckpointData();
        }
 }
 
 
 /**
- * Sends the checkpoint to its buddy. The mode distinguishes between the two cases:
- * MLOG_RESTARTED: sending the checkpoint to a team member that did not crash but is restarting.
- * MLOG_CRASHED: sending the checkpoint to the processor that crashed.
+ * Sends the checkpoint to its buddy. 
  */
-void sendCheckpointData(int mode){     
+void sendCheckpointData(){     
        RestartRequest *restartMsg = storedRequest;
-       StoredCheckpoint *storedChkpt =         CpvAccess(_storedCheckpointData);
+       StoredCheckpoint *storedChkpt = CpvAccess(_storedCheckpointData);
        int numMigratedAwayElements = migratedNoticeList.size();
        if(migratedNoticeList.size() != 0){
                        printf("[%d] size of migratedNoticeList %d\n",CmiMyPe(),migratedNoticeList.size());
 //                     CkAssert(migratedNoticeList.size() == 0);
        }
-       
-       
        int totalSize = sizeof(RestartProcessorData)+storedChkpt->bufSize;
        
-       DEBUGRESTART(CkPrintf("[%d] Sending out checkpoint for processor %d size %d \n",CkMyPe(),restartMsg->PE,totalSize);)
+       DEBUG_RESTART(CkPrintf("[%d] Sending out checkpoint for processor %d size %d \n",CkMyPe(),restartMsg->PE,totalSize);)
        CkPrintf("[%d] Sending out checkpoint for processor %d size %d \n",CkMyPe(),restartMsg->PE,totalSize);
        
-       CkQ<LocalMessageLog > *localMsgQ = CpvAccess(_localMessageLog);
-       totalSize += localMsgQ->length()*sizeof(LocalMessageLog);
        totalSize += numMigratedAwayElements*sizeof(MigrationRecord);
        
        char *msg = (char *)CmiAlloc(totalSize);
@@ -2237,47 +1234,25 @@ void sendCheckpointData(int mode){
        }
        
 
-#ifdef CHECKPOINT_DISK
-       readCheckpointFromDisk(storedChkpt->bufSize,buf);
-#else  
        memcpy(buf,storedChkpt->buf,storedChkpt->bufSize);
-#endif
        buf = &buf[storedChkpt->bufSize];
 
+       CmiSetHandler(msg,_recvCheckpointHandlerIdx);
+       CmiSyncSendAndFree(restartMsg->PE,totalSize,msg);
+       CmiFree(restartMsg);
 
-       //store localmessage Log
-       dataMsg->numLocalMessages = localMsgQ->length();
-       for(int i=0;i<localMsgQ->length();i++){
-               if(!fault_aware(((*localMsgQ)[i]).recver )){
-                       CmiAbort("Non fault aware localMsgQ");
-               }
-               memcpy(buf,&(*localMsgQ)[i],sizeof(LocalMessageLog));
-               buf = &buf[sizeof(LocalMessageLog)];
-       }
-       
-       if(mode == MLOG_RESTARTED){
-               CmiSetHandler(msg,_recvRestartCheckpointHandlerIdx);
-               CmiSyncSendAndFree(restartMsg->PE,totalSize,msg);
-               CmiFree(restartMsg);
-       }else{
-               CmiSetHandler(msg,_recvCheckpointHandlerIdx);
-               CmiSyncSendAndFree(restartMsg->PE,totalSize,msg);
-               CmiFree(restartMsg);
-       }
 };
 
 
 // this list is used to create a vector of the object ids of all
 //the chares on this processor currently and the highest TN processed by them 
 //the first argument is actually a CkVec<TProcessedLog> *
-void createObjIDList(void *data,ChareMlogData *mlogData){
-       CkVec<TProcessedLog> *list = (CkVec<TProcessedLog> *)data;
-       TProcessedLog entry;
-       entry.recver = mlogData->objID;
-       entry.tProcessed = mlogData->tProcessed;
+void createObjIDList(void *data, ChareMlogData *mlogData){
+       CkVec<CkObjID> *list = (CkVec<CkObjID> *)data;
+       CkObjID entry;
+       entry = mlogData->objID;
        list->push_back(entry);
-       DEBUG_TEAM(char objString[100]);
-       DEBUG_TEAM(CkPrintf("[%d] %s restored with tProcessed set to %d \n",CkMyPe(),mlogData->objID.toString(objString),mlogData->tProcessed));
+       DEBUG_RECOVERY(printLog(&entry));
 }
 
 
@@ -2291,12 +1266,6 @@ void _recvCheckpointHandler(char *_restartData){
 
        globalLBID = restartData->lbGroupID;
        
-       restartData->restartWallTime *= 1000;
-       adjustChkptPeriod = restartData->restartWallTime/(double) chkptPeriod - floor(restartData->restartWallTime/(double) chkptPeriod);
-       adjustChkptPeriod = (double )chkptPeriod*(adjustChkptPeriod);
-       if(adjustChkptPeriod < 0) adjustChkptPeriod = 0;
-
-       
        printf("[%d] Restart Checkpointdata received from PE %d at %.6lf with checkpointSize %d\n",CkMyPe(),restartData->PE,CmiWallTimer(),restartData->checkPointSize);
        char *buf = &_restartData[sizeof(RestartProcessorData)];
        
@@ -2310,119 +1279,36 @@ void _recvCheckpointHandler(char *_restartData){
        PUP::fromMem pBuf(buf);
 
        pBuf | checkpointCount;
-
+       for(int i=0; i<CmiNumPes(); i++){
+               pBuf | CpvAccess(_incarnation)[i];
+       }
        CkPupROData(pBuf);
        CkPupGroupData(pBuf,CmiTrue);
        CkPupNodeGroupData(pBuf,CmiTrue);
        pupArrayElementsSkip(pBuf,CmiTrue,NULL);
        CkAssert(pBuf.size() == restartData->checkPointSize);
        printf("[%d] Restart Objects created from CheckPointData at %.6lf \n",CkMyPe(),CmiWallTimer());
+
+       // increases the incarnation number
+       CpvAccess(_incarnation)[CmiMyPe()]++;
        
        forAllCharesDo(initializeRestart,NULL);
        
-       //store the restored local message log in a vector
-       buf = &buf[restartData->checkPointSize];        
-       for(int i=0;i<restartData->numLocalMessages;i++){
-               LocalMessageLog logEntry;
-               memcpy(&logEntry,buf,sizeof(LocalMessageLog));
-               
-               Chare *recverObj = (Chare *)logEntry.recver.getObject();
-               if(recverObj!=NULL){
-                       recverObj->mlogData->addToRestoredLocalQ(&logEntry);
-                       recverObj->mlogData->receivedTNs->push_back(logEntry.TN);
-                       char senderString[100];
-                       char recverString[100];
-                       DEBUGRESTART(printf("[%d] Received local message log sender %s recver %s SN %d  TN %d\n",CkMyPe(),logEntry.sender.toString(senderString),logEntry.recver.toString(recverString),logEntry.SN,logEntry.TN));
-               }else{
-//                     DEBUGRESTART(printf("Object receiving local message doesnt exist on restarted processor .. ignoring it"));
-               }
-               buf = &buf[sizeof(LocalMessageLog)];
-       }
-
-       forAllCharesDo(sortRestoredLocalMsgLog,NULL);
-
        CmiFree(_restartData);
        
-       
        _initDone();
 
        getGlobalStep(globalLBID);
-       
-       countUpdateHomeAcks = 0;
-       RestartRequest updateHomeRequest;
-       updateHomeRequest.PE = CmiMyPe();
-       CmiSetHandler (&updateHomeRequest,_updateHomeRequestHandlerIdx);
-       for(int i=0;i<CmiNumPes();i++){
-               if(i != CmiMyPe()){
-                       CmiSyncSend(i,sizeof(RestartRequest),(char *)&updateHomeRequest);
-               }
-       }
-
-}
-
-/**
- * Receives the updateHome ACKs from all other processors. Once everybody
- * has replied, it sends a request to resend the logged messages.
- */
-void _updateHomeAckHandler(RestartRequest *updateHomeAck){
-       countUpdateHomeAcks++;
-       CmiFree(updateHomeAck);
-       // one is from the recvglobal step handler .. it is a dummy updatehomeackhandler
-       if(countUpdateHomeAcks != CmiNumPes()){
-               return;
-       }
-
-       // Send out the request to resend logged messages to all other processors
-       CkVec<TProcessedLog> objectVec;
-       forAllCharesDo(createObjIDList, (void *)&objectVec);
-       int numberObjects = objectVec.size();
-       
-       //      resendMsg layout: |ResendRequest|Array of TProcessedLog|
-       int totalSize = sizeof(ResendRequest)+numberObjects*sizeof(TProcessedLog);
-       char *resendMsg = (char *)CmiAlloc(totalSize);  
-
-       ResendRequest *resendReq = (ResendRequest *)resendMsg;
-       resendReq->PE =CkMyPe(); 
-       resendReq->numberObjects = numberObjects;
-       char *objList = &resendMsg[sizeof(ResendRequest)];
-       memcpy(objList,objectVec.getVec(),numberObjects*sizeof(TProcessedLog)); 
 
-       /* test for parallel restart migrate away object**/
-       if(parallelRestart){
-               distributeRestartedObjects();
-               printf("[%d] Redistribution of objects done at %.6lf \n",CkMyPe(),CmiWallTimer());
-       }
-       
-       /*      To make restart work for load balancing.. should only
-       be used when checkpoint happens along with load balancing
-       **/
-//     forAllCharesDo(resumeFromSyncRestart,NULL);
-
-       CentralLB *lb = (CentralLB *)CkpvAccess(_groupTable)->find(globalLBID).getObj();
-       CpvAccess(_currentObj) = lb;
-       lb->ReceiveDummyMigration(restartDecisionNumber);
-
-       sleep(10);
        
-       CmiSetHandler(resendMsg,_resendMessagesHandlerIdx);
-       for(int i=0;i<CkNumPes();i++){
-               if(i != CkMyPe()){
-                       CmiSyncSend(i,totalSize,resendMsg);
-               }       
-       }
-       _resendMessagesHandler(resendMsg);
-       CmiFree(resendMsg);
-};
+}
 
 /**
  * @brief Initializes variables and flags for restarting procedure.
  */
 void initializeRestart(void *data, ChareMlogData *mlogData){
        mlogData->resendReplyRecvd = 0;
-       mlogData->receivedTNs = new CkVec<MCount>;
        mlogData->restartFlag = 1;
-       mlogData->restoredLocalMsgLog.removeAll();
-       mlogData->mapTable.empty();
 };
 
 /**
@@ -2436,194 +1322,93 @@ void updateHomePE(void *data,ChareMlogData *mlogData){
        if(mlogData->objID.type == TypeArray){
                //it is an array element
                CkGroupID myGID = mlogData->objID.data.array.id;
-               CkArrayIndex myIdx =  mlogData->objID.data.array.idx;
+               CkArrayIndexMax myIdx =  mlogData->objID.data.array.idx.asChild();
                CkArrayID aid(mlogData->objID.data.array.id);           
                //check if the restarted processor is the home processor for this object
                CkLocMgr *locMgr = aid.ckLocalBranch()->getLocMgr();
                if(locMgr->homePe(myIdx) == PE){
-                       DEBUGRESTART(printf("[%d] Tell %d of current location of array element",CkMyPe(),PE));
-                       DEBUGRESTART(myIdx.print());
+                       DEBUG_RESTART(printf("[%d] Tell %d of current location of array element",CkMyPe(),PE));
+                       DEBUG_RESTART(myIdx.print());
                        informLocationHome(locMgr->getGroupID(),myIdx,PE,CkMyPe());
                }
        }
 };
 
-
-/**
- * Updates the homePe for all chares in this processor.
- */
-void _updateHomeRequestHandler(RestartRequest *updateRequest){
-       int sender = updateRequest->PE;
-       
-       forAllCharesDo(updateHomePE,updateRequest);
-       
-       updateRequest->PE = CmiMyPe();
-       CmiSetHandler(updateRequest,_updateHomeAckHandlerIdx);
-       CmiSyncSendAndFree(sender,sizeof(RestartRequest),(char *)updateRequest);
-       if(sender == getCheckPointPE() && unAckedCheckpoint==1){
-               CmiPrintf("[%d] Crashed processor did not ack so need to checkpoint again\n",CmiMyPe());
-               checkpointCount--;
-               startMlogCheckpoint(NULL,0);
-       }
-       if(sender == getCheckPointPE()){
-               for(int i=0;i<retainedObjectList.size();i++){
-                       if(retainedObjectList[i]->acked == 0){
-                               MigrationNotice migMsg;
-                               migMsg.migRecord = retainedObjectList[i]->migRecord;
-                               migMsg.record = retainedObjectList[i];
-                               CmiSetHandler((void *)&migMsg,_receiveMigrationNoticeHandlerIdx);
-                               CmiSyncSend(getCheckPointPE(),sizeof(migMsg),(char *)&migMsg);
-                       }
-               }
-       }
-}
-
 /**
- * @brief Fills up the ticket vector for each chare.
+ * Prints a processed log.
  */
-void fillTicketForChare(void *data, ChareMlogData *mlogData){
-       ResendData *resendData = (ResendData *)data;
-       int PE = resendData->PE; //restarted PE
-       int count=0;
-       CkHashtableIterator *iterator;
-       void *objp;
-       void *objkey;
-       CkObjID *objID;
-       SNToTicket *snToTicket;
-       Ticket ticket;
-       
-       // traversing the team table looking up for the maximum TN received     
-       iterator = mlogData->teamTable.iterator();
-       while( (objp = iterator->next(&objkey)) != NULL ){
-               objID = (CkObjID *)objkey;
-       
-               // traversing the resendData structure to add ticket numbers
-               for(int j=0;j<resendData->numberObjects;j++){
-                       if((*objID) == (resendData->listObjects)[j].recver){
-char name[100];
-                               snToTicket = *(SNToTicket **)objp;
-//CkPrintf("[%d] ---> Traversing the resendData for %s start=%u finish=%u \n",CkMyPe(),objID->toString(name),snToTicket->getStartSN(),snToTicket->getFinishSN());
-                               for(MCount snIndex=snToTicket->getStartSN(); snIndex<=snToTicket->getFinishSN(); snIndex++){
-                                       ticket = snToTicket->get(snIndex);      
-                                       if(ticket.TN > resendData->maxTickets[j]){
-                                               resendData->maxTickets[j] = ticket.TN;
-                                       }
-                                       if(ticket.TN >= (resendData->listObjects)[j].tProcessed){
-                                               //store the TNs that have been since the recver last checkpointed
-                                               resendData->ticketVecs[j].push_back(ticket.TN);
-                                       }
-                               }
-                       }
-               }
-       }
-
-       //releasing the memory for the iterator
-       delete iterator;
+void printLog(CkObjID &recver){
+       char recverString[100];
+       CkPrintf("[RECOVERY] [%d] OBJECT=\"%s\" \n",CkMyPe(),recver.toString(recverString));
 }
 
-
 /**
- * @brief Turns on the flag for team recovery that selectively restores
- * particular metadata information.
+ * Prints information about a message.
  */
-void setTeamRecovery(void *data, ChareMlogData *mlogData){
-       char name[100];
-       mlogData->teamRecoveryFlag = 1; 
+void printMsg(envelope *env, const char* par){
+       char senderString[100];
+       char recverString[100];
+       CkPrintf("[RECOVERY] [%d] MSG-%s FROM=\"%s\" TO=\"%s\" SN=%d\n",CkMyPe(),par,env->sender.toString(senderString),env->recver.toString(recverString),env->SN);
 }
 
 /**
- * @brief Turns off the flag for team recovery.
+ * @brief Resends all the logged messages to a particular chare list.
+ * @param data is of type ResendData which contains the array of objects on  the restartedProcessor.
+ * @param mlogData a particular chare living in this processor.
  */
-void unsetTeamRecovery(void *data, ChareMlogData *mlogData){
-       mlogData->teamRecoveryFlag = 0;
-}
+void resendMessageForChare(void *data, ChareMlogData *mlogData){
+       DEBUG_RESTART(char nameString[100]);
+       DEBUG_RESTART(char recverString[100]);
+       DEBUG_RESTART(char senderString[100]);
 
-//the data argument is of type ResendData which contains the 
-//array of objects on  the restartedProcessor
-//this method resends the messages stored in this chare's message log 
-//to the restarted processor. It also accumulates the maximum TN
-//for all the objects on the restarted processor
-void resendMessageForChare(void *data,ChareMlogData *mlogData){
-       char nameString[100];
        ResendData *resendData = (ResendData *)data;
        int PE = resendData->PE; //restarted PE
-       DEBUGRESTART(printf("[%d] Resend message from %s to processor %d \n",CkMyPe(),mlogData->objID.toString(nameString),PE);)
        int count=0;
        int ticketRequests=0;
        CkQ<MlogEntry *> *log = mlogData->getMlog();
-       
+
+       DEBUG_RESTART(printf("[%d] Resend message from %s to processor %d \n",CkMyPe(),mlogData->objID.toString(nameString),PE);)
+
+       // traversing the message log to see if we must resend a message        
        for(int i=0;i<log->length();i++){
                MlogEntry *logEntry = (*log)[i];
                
-               // if we sent out the logs of a local message to buddy and he crashed
-               //before acking
+               // if we sent out the logs of a local message to buddy and it crashed
+               //before acknowledging 
                envelope *env = logEntry->env;
                if(env == NULL){
                        continue;
                }
-               if(logEntry->unackedLocal){
-                       char recverString[100];
-                       DEBUGRESTART(printf("[%d] Resend Local unacked message from %s to %s SN %d TN %d \n",CkMyPe(),env->sender.toString(nameString),env->recver.toString(recverString),env->SN,env->TN);)
-                       sendLocalMessageCopy(logEntry);
-               }
-               //looks like near a crash messages between uninvolved processors can also get lost. Resend ticket requests as a result
-               if(env->TN <= 0){
-                       //ticket not yet replied send it out again
-                       sendTicketRequest(env->sender,env->recver,logEntry->destPE,logEntry,env->SN,0,1);
-               }
-               
+       
+               // resend if type is not invalid        
                if(env->recver.type != TypeInvalid){
-                       int flag = 0;//marks if any of the restarted objects matched this log entry
                        for(int j=0;j<resendData->numberObjects;j++){
-                               if(env->recver == (resendData->listObjects)[j].recver){
-                                       flag = 1;
-                                       //message has a valid TN
-                                       if(env->TN > 0){
-                                               //store maxTicket
-                                               if(env->TN > resendData->maxTickets[j]){
-                                                       resendData->maxTickets[j] = env->TN;
+                               if(env->recver == (resendData->listObjects)[j]){
+                                       if(PE != CkMyPe()){
+                                               DEBUG_RECOVERY(printMsg(env,RECOVERY_SEND));
+                                               if(env->recver.type == TypeNodeGroup){
+                                                       CmiSyncNodeSend(PE,env->getTotalsize(),(char *)env);
+                                               }else{
+                                                       CmiSetHandler(env,CmiGetXHandler(env));
+                                                       CmiSyncSend(PE,env->getTotalsize(),(char *)env);
                                                }
-                                               //if the TN for this entry is more than the TN processed, send the message out
-                                               if(env->TN >= (resendData->listObjects)[j].tProcessed){
-                                                       //store the TNs that have been since the recver last checkpointed
-                                                       resendData->ticketVecs[j].push_back(env->TN);
-                                                       
-                                                       if(PE != CkMyPe()){
-                                                               if(env->recver.type == TypeNodeGroup){
-                                                                       CmiSyncNodeSend(PE,env->getTotalsize(),(char *)env);
-                                                               }else{
-                                                                       CmiSetHandler(env,CmiGetXHandler(env));
-                                                                       CmiSyncSend(PE,env->getTotalsize(),(char *)env);
-                                                               }
-                                                       }else{
-                                                               envelope *copyEnv = copyEnvelope(env);
-                                                               CqsEnqueueGeneral((Queue)CpvAccess(CsdSchedQueue),copyEnv, copyEnv->getQueueing(),copyEnv->getPriobits(),(unsigned int *)copyEnv->getPrioPtr());
-                                                       }
-                                                       char senderString[100];
-                                                       DEBUGRESTART(printf("[%d] Resent message sender %s recver %s SN %d TN %d \n",CkMyPe(),env->sender.toString(senderString),env->recver.toString(nameString),env->SN,env->TN));
-                                                       count++;
-                                               }       
                                        }else{
-/*                                     //the message didnt get a ticket the last time and needs to start with a ticket request
-                                               DEBUGRESTART(printf("[%d] Resent ticket request SN %d to %s needs ticket at %d in logQ \n",CkMyPe(),env->SN,env->recver.toString(nameString),i));
-                                               //generateCommonTicketRequest(env->recver,env,PE,logEntry->_infoIdx);                                           
-                                               CkAssert(logEntry->destPE != CkMyPe());
-                                               
-                                               sendTicketRequest(env->sender,env->recver,PE,logEntry,env->SN,1);
-                                               
-                                               ticketRequests++;*/
+                                               envelope *copyEnv = copyEnvelope(env);
+                                               CqsEnqueueGeneral((Queue)CpvAccess(CsdSchedQueue),copyEnv, copyEnv->getQueueing(),copyEnv->getPriobits(),(unsigned int *)copyEnv->getPrioPtr());
                                        }
+                                       DEBUG_RESTART(printf("[%d] Resent message sender %s recver %s SN %d TN %d \n",CkMyPe(),env->sender.toString(senderString),env->recver.toString(nameString),env->SN,env->TN));
+                                       count++;
                                }
                        }//end of for loop of objects
                        
                }       
        }
-       DEBUGRESTART(printf("[%d] Resent  %d/%d (%d) messages  from %s to processor %d \n",CkMyPe(),count,log->length(),ticketRequests,mlogData->objID.toString(nameString),PE);)       
+       DEBUG_RESTART(printf("[%d] Resent  %d/%d (%d) messages  from %s to processor %d \n",CkMyPe(),count,log->length(),ticketRequests,mlogData->objID.toString(nameString),PE);)      
 }
 
 /**
- * Resends the messages since the last checkpoint to the list of objects included in the 
- * request.
+ * Resends messages since last checkpoint to the list of objects included in the 
+ * request. It also sends stored remote determinants to the particular failed PE.
  */
 void _resendMessagesHandler(char *msg){
        ResendData d;
@@ -2633,405 +1418,157 @@ void _resendMessagesHandler(char *msg){
        char *listObjects = &msg[sizeof(ResendRequest)];
        d.numberObjects = resendReq->numberObjects;
        d.PE = resendReq->PE;
-       d.listObjects = (TProcessedLog *)listObjects;
-       d.maxTickets = new MCount[d.numberObjects];
-       d.ticketVecs = new CkVec<MCount>[d.numberObjects];
-       for(int i=0;i<d.numberObjects;i++){
-               d.maxTickets[i] = 0;
-       }
-
-       //Check if any of the retained objects need to be recreated
-       //If they have not been recreated on the restarted processor
-       //they need to be recreated on this processor
-       int count=0;
-       for(int i=0;i<retainedObjectList.size();i++){
-               if(retainedObjectList[i]->migRecord.toPE == d.PE){
-                       count++;
-                       int recreate=1;
-                       for(int j=0;j<d.numberObjects;j++){
-                               if(d.listObjects[j].recver.type != TypeArray ){
-                                       continue;
-                               }
-                               CkArrayID aid(d.listObjects[j].recver.data.array.id);           
-                               CkLocMgr *locMgr = aid.ckLocalBranch()->getLocMgr();
-                               if(retainedObjectList[i]->migRecord.gID == locMgr->getGroupID()){
-                                       if(retainedObjectList[i]->migRecord.idx == d.listObjects[j].recver.data.array.idx){
-                                               recreate = 0;
-                                               break;
-                                       }
-                               }
-                       }
-                       CmiPrintf("[%d] Object migrated away but did not checkpoint recreate %d locmgrid %d idx %s\n",CmiMyPe(),recreate,retainedObjectList[i]->migRecord.gID.idx,idx2str(retainedObjectList[i]->migRecord.idx));
-                       if(recreate){
-                               donotCountMigration=1;
-                               _receiveMlogLocationHandler(retainedObjectList[i]->msg);
-                               donotCountMigration=0;
-                               CkLocMgr *locMgr =  (CkLocMgr*)CkpvAccess(_groupTable)->find(retainedObjectList[i]->migRecord.gID).getObj();
-                               int homePE = locMgr->homePe(retainedObjectList[i]->migRecord.idx);
-                               informLocationHome(retainedObjectList[i]->migRecord.gID,retainedObjectList[i]->migRecord.idx,homePE,CmiMyPe());
-                               sendDummyMigration(d.PE,globalLBID,retainedObjectList[i]->migRecord.gID,retainedObjectList[i]->migRecord.idx,CmiMyPe());
-                               CkLocRec *rec = locMgr->elementRec(retainedObjectList[i]->migRecord.idx);
-                               CmiAssert(rec->type() == CkLocRec::local);
-                               CkVec<CkMigratable *> eltList;
-                               locMgr->migratableList((CkLocRec_local *)rec,eltList);
-                               for(int j=0;j<eltList.size();j++){
-                                       if(eltList[j]->mlogData->toResumeOrNot == 1 && eltList[j]->mlogData->resumeCount < globalResumeCount){
-                                               CpvAccess(_currentObj) = eltList[j];
-                                               eltList[j]->ResumeFromSync();
-                                       }
-                               }
-                               retainedObjectList[i]->msg=NULL;        
-                       }
-               }
-       }
-       
-       if(count > 0){
-//             CmiAbort("retainedObjectList for restarted processor not empty");
-       }
+       d.listObjects = (CkObjID *)listObjects;
        
        DEBUG(printf("[%d] Received request to Resend Messages to processor %d numberObjects %d at %.6lf\n",CkMyPe(),resendReq->PE,resendReq->numberObjects,CmiWallTimer()));
 
+       // resends messages for the list of objects
+       forAllCharesDo(resendMessageForChare,&d);
 
-       //TML: examines the origin processor to determine if it belongs to the same group.
-       // In that case, it only returns the maximum ticket received for each object in the list.
-       if(isTeamLocal(resendReq->PE) && CkMyPe() != resendReq->PE)
-               forAllCharesDo(fillTicketForChare,&d);
-       else
-               forAllCharesDo(resendMessageForChare,&d);
-
-       //send back the maximum ticket number for a message sent to each object on the 
-       //restarted processor
-       //Message: |ResendRequest|List of CkObjIDs|List<#number of objects in vec,TN of tickets seen>|
-       
-       int totalTNStored=0;
-       for(int i=0;i<d.numberObjects;i++){
-               totalTNStored += d.ticketVecs[i].size();
-       }
-       
-       int totalSize = sizeof(ResendRequest)+d.numberObjects*(sizeof(CkObjID)+sizeof(int)) + totalTNStored*sizeof(MCount);
-       char *resendReplyMsg = (char *)CmiAlloc(totalSize);
-       
-       ResendRequest *resendReply = (ResendRequest *)resendReplyMsg;
-       resendReply->PE = CkMyPe();
-       resendReply->numberObjects = d.numberObjects;
-       
-       char *replyListObjects = &resendReplyMsg[sizeof(ResendRequest)];
-       CkObjID *replyObjects = (CkObjID *)replyListObjects;
-       for(int i=0;i<d.numberObjects;i++){
-               replyObjects[i] = d.listObjects[i].recver;
-       }
-       
-       char *ticketList = &replyListObjects[sizeof(CkObjID)*d.numberObjects];
-       for(int i=0;i<d.numberObjects;i++){
-               int vecsize = d.ticketVecs[i].size();
-               memcpy(ticketList,&vecsize,sizeof(int));
-               ticketList = &ticketList[sizeof(int)];
-               memcpy(ticketList,d.ticketVecs[i].getVec(),sizeof(MCount)*vecsize);
-               ticketList = &ticketList[sizeof(MCount)*vecsize];
-       }       
+       DEBUG_MEM(CmiMemoryCheck());
 
-       CmiSetHandler(resendReplyMsg,_resendReplyHandlerIdx);
-       CmiSyncSendAndFree(d.PE,totalSize,(char *)resendReplyMsg);
-       
-/*     
-       if(verifyAckRequestsUnacked){
-               CmiPrintf("[%d] verifyAckRequestsUnacked %d call dummy migrates\n",CmiMyPe(),verifyAckRequestsUnacked);
-               for(int i=0;i<verifyAckRequestsUnacked;i++){
-                       CentralLB *lb = (CentralLB *)CkpvAccess(_groupTable)->find(globalLBID).getObj();
-                       LDObjHandle h;
-                       lb->Migrated(h,1);
-               }
-       }
-       
-       verifyAckRequestsUnacked=0;*/
-       
-       delete [] d.maxTickets;
-       delete [] d.ticketVecs;
        if(resendReq->PE != CkMyPe()){
                CmiFree(msg);
        }       
-//     CmiPrintf("[%d] End of resend Request \n",CmiMyPe());
-       lastRestart = CmiWallTimer();
-}
-
-void sortVec(CkVec<MCount> *TNvec);
-int searchVec(CkVec<MCount> *TNVec,MCount searchTN);
-
-/**
- * @brief Receives the tickets assigned to message to other objects.
- */
-void _resendReplyHandler(char *msg){   
-       /**
-               need to rewrite this method to deal with parallel restart
-       */
-       ResendRequest *resendReply = (ResendRequest *)msg;
-       CkObjID *listObjects = (CkObjID *)( &msg[sizeof(ResendRequest)]);
-
-       char *listTickets = (char *)(&listObjects[resendReply->numberObjects]);
-       
-//     DEBUGRESTART(printf("[%d] _resendReply from %d \n",CmiMyPe(),resendReply->PE));
-       DEBUG_TEAM(printf("[%d] _resendReply from %d \n",CmiMyPe(),resendReply->PE));
-       for(int i =0; i< resendReply->numberObjects;i++){       
-               Chare *obj = (Chare *)listObjects[i].getObject();
-               
-               int vecsize;
-               memcpy(&vecsize,listTickets,sizeof(int));
-               listTickets = &listTickets[sizeof(int)];
-               MCount *listTNs = (MCount *)listTickets;        
-               listTickets = &listTickets[vecsize*sizeof(MCount)];
-               
-               if(obj != NULL){
-                       //the object was restarted on the processor on which it existed
-                       processReceivedTN(obj,vecsize,listTNs);
-               }else{
-               //pack up objID vecsize and listTNs and send it to the correct processor
-                       int totalSize = sizeof(ReceivedTNData)+vecsize*sizeof(MCount);
-                       char *TNMsg = (char *)CmiAlloc(totalSize);
-                       ReceivedTNData *receivedTNData = (ReceivedTNData *)TNMsg;
-                       receivedTNData->recver = listObjects[i];
-                       receivedTNData->numTNs = vecsize;
-                       char *tnList = &TNMsg[sizeof(ReceivedTNData)];
-                       memcpy(tnList,listTNs,sizeof(MCount)*vecsize);
-
-                       CmiSetHandler(TNMsg,_receivedTNDataHandlerIdx);
-                       CmiSyncSendAndFree(listObjects[i].guessPE(),totalSize,TNMsg);
-               }       
-       }
-};
-
-void _receivedTNDataHandler(ReceivedTNData *msg){
-       char objName[100];
-       Chare *obj = (Chare *) msg->recver.getObject();
-       if(obj){                
-               char *_msg = (char *)msg;
-               DEBUGRESTART(printf("[%d] receivedTNDataHandler for %s\n",CmiMyPe(),obj->mlogData->objID.toString(objName)));
-               MCount *listTNs = (MCount *)(&_msg[sizeof(ReceivedTNData)]);
-               processReceivedTN(obj,msg->numTNs,listTNs);
-       }else{
-               int totalSize = sizeof(ReceivedTNData)+sizeof(MCount)*msg->numTNs;
-               CmiSyncSendAndFree(msg->recver.guessPE(),totalSize,(char *)msg);
-       }
-};
-
-/**
- * @brief Processes the received list of tickets from a particular PE.
- */
-void processReceivedTN(Chare *obj, int listSize, MCount *listTNs){
-       // increases the number of resendReply received
-       obj->mlogData->resendReplyRecvd++;
-
-       DEBUG(char objName[100]);
-       DEBUG(CkPrintf("[%d] processReceivedTN obj->mlogData->resendReplyRecvd=%d CkNumPes()=%d\n",CkMyPe(),obj->mlogData->resendReplyRecvd,CkNumPes()));
-       //CkPrintf("[%d] processReceivedTN with %d listSize by %s\n",CkMyPe(),listSize,obj->mlogData->objID.toString(objName));
-       //if(obj->mlogData->receivedTNs == NULL)
-       //      CkPrintf("NULL\n");     
-       //CkPrintf("using %d entries\n",obj->mlogData->receivedTNs->length());  
-
-       // includes the tickets into the receivedTN structure
-       for(int j=0;j<listSize;j++){
-               obj->mlogData->receivedTNs->push_back(listTNs[j]);
-       }
-       
-       //if this object has received all the replies find the ticket numbers
-       //that senders know about. Those less than the ticket number processed 
-       //by the receiver can be thrown away. The rest need not be consecutive
-       // ie there can be holes in the list of ticket numbers seen by senders
-       if(obj->mlogData->resendReplyRecvd == CkNumPes()){
-               obj->mlogData->resendReplyRecvd = 0;
-               //sort the received TNS
-               sortVec(obj->mlogData->receivedTNs);
-       
-               //after all the received tickets are in we need to sort them and then 
-               // calculate the holes  
-               if(obj->mlogData->receivedTNs->size() > 0){
-                       int tProcessedIndex = searchVec(obj->mlogData->receivedTNs,obj->mlogData->tProcessed);
-                       int vecsize = obj->mlogData->receivedTNs->size();
-                       int numberHoles = ((*obj->mlogData->receivedTNs)[vecsize-1] - obj->mlogData->tProcessed)-(vecsize -1 - tProcessedIndex);
-                       
-                       // updating tCount with the highest ticket handed out
-                       if(teamSize > 1){
-                               if(obj->mlogData->tCount < (*obj->mlogData->receivedTNs)[vecsize-1])
-                                       obj->mlogData->tCount = (*obj->mlogData->receivedTNs)[vecsize-1];
-                       }else{
-                               obj->mlogData->tCount = (*obj->mlogData->receivedTNs)[vecsize-1];
-                       }
-                       
-                       if(numberHoles == 0){
-                       }else{
-                               char objName[100];                                      
-                               printf("[%d] Holes detected in the TNs for %s number %d \n",CkMyPe(),obj->mlogData->objID.toString(objName),numberHoles);
-                               obj->mlogData->numberHoles = numberHoles;
-                               obj->mlogData->ticketHoles = new MCount[numberHoles];
-                               int countHoles=0;
-                               for(int k=tProcessedIndex+1;k<vecsize;k++){
-                                       if((*obj->mlogData->receivedTNs)[k] != (*obj->mlogData->receivedTNs)[k-1]+1){
-                                               //the TNs are not consecutive at this point
-                                               for(MCount newTN=(*obj->mlogData->receivedTNs)[k-1]+1;newTN<(*obj->mlogData->receivedTNs)[k];newTN++){
-                                                       DEBUG(CKPrintf("hole no %d at %d next available ticket %d \n",countHoles,newTN,(*obj->mlogData->receivedTNs)[k]));
-                                                       obj->mlogData->ticketHoles[countHoles] = newTN;
-                                                       countHoles++;
-                                               }       
-                                       }
-                               }
-                               //Holes have been given new TN
-                               if(countHoles != numberHoles){
-                                       char str[100];
-                                       printf("[%d] Obj %s countHoles %d numberHoles %d\n",CmiMyPe(),obj->mlogData->objID.toString(str),countHoles,numberHoles);
-                               }
-                               CkAssert(countHoles == numberHoles);                                    
-                               obj->mlogData->currentHoles = numberHoles;
-                       }
-               }
-       
-               // cleaning up structures and getting ready to continue execution       
-               delete obj->mlogData->receivedTNs;
-               DEBUG(CkPrintf("[%d] Resetting receivedTNs\n",CkMyPe()));
-               obj->mlogData->receivedTNs = NULL;
-               obj->mlogData->restartFlag = 0;
-
-               DEBUGRESTART(char objString[100]);
-               DEBUGRESTART(CkPrintf("[%d] Can restart handing out tickets again at %.6lf for %s\n",CkMyPe(),CmiWallTimer(),obj->mlogData->objID.toString(objString)));
-       }
-
 }
 
-
-void sortVec(CkVec<MCount> *TNvec){
-       //sort it ->its bloddy bubble sort
-       //TODO: use quicksort
-       for(int i=0;i<TNvec->size();i++){
-               for(int j=i+1;j<TNvec->size();j++){
-                       if((*TNvec)[j] < (*TNvec)[i]){
-                               MCount temp;
-                               temp = (*TNvec)[i];
-                               (*TNvec)[i] = (*TNvec)[j];
-                               (*TNvec)[j] = temp;
-                       }
-               }
-       }
-       //make it unique .. since its sorted all equal units will be consecutive
-       MCount *tempArray = new MCount[TNvec->size()];
-       int     uniqueCount=-1;
-       for(int i=0;i<TNvec->size();i++){
-               tempArray[i] = 0;
-               if(uniqueCount == -1 || tempArray[uniqueCount] != (*TNvec)[i]){
-                       uniqueCount++;
-                       tempArray[uniqueCount] = (*TNvec)[i];
-               }
-       }
-       uniqueCount++;
-       TNvec->removeAll();
-       for(int i=0;i<uniqueCount;i++){
-               TNvec->push_back(tempArray[i]);
-       }
-       delete [] tempArray;
-}      
-
-int searchVec(CkVec<MCount> *TNVec,MCount searchTN){
-       if(TNVec->size() == 0){
-               return -1; //not found in an empty vec
-       }
-       //binary search to find 
-       int left=0;
-       int right = TNVec->size();
-       int mid = (left +right)/2;
-       while(searchTN != (*TNVec)[mid] && left < right){
-               if((*TNVec)[mid] > searchTN){
-                       right = mid-1;
-               }else{
-                       left = mid+1;
-               }
-               mid = (left + right)/2;
-       }
-       if(left < right){
-               //mid is the element to be returned
-               return mid;
-       }else{
-               if(mid < TNVec->size() && mid >=0){
-                       if((*TNVec)[mid] == searchTN){
-                               return mid;
-                       }else{
-                               return -1;
-                       }
-               }else{
-                       return -1;
-               }
-       }
-};
-
-
 /*
        Method to do parallel restart. Distribute some of the array elements to other processors.
        The problem is that we cant use to charm entry methods to do migration as it will get
        stuck in the protocol that is going to restart
+       Note: in order to avoid interference between the objects being recovered, the current PE
+    will NOT keep any object. It will be devoted to forward the messages to recovering objects.    Otherwise, the current PE has to do both things, recover objects and forward messages and 
+    objects end up stepping into each other's shoes (interference).
 */
 
 class ElementDistributor: public CkLocIterator{
        CkLocMgr *locMgr;
        int *targetPE;
+
        void pupLocation(CkLocation &loc,PUP::er &p){
-               CkArrayIndex idx=loc.getIndex();
+               CkArrayIndexMax idx=loc.getIndex();
                CkGroupID gID = locMgr->ckGetGroupID();
                p|gID;      // store loc mgr's GID as well for easier restore
                p|idx;
                p|loc;
        };
-       public:
-               ElementDistributor(CkLocMgr *mgr_,int *toPE_):locMgr(mgr_),targetPE(toPE_){};
-               void addLocation(CkLocation &loc){
-                       if(*targetPE == CkMyPe()){
-                               *targetPE = (*targetPE +1)%CkNumPes();                          
-                               return;
-                       }
-                       
-                       CkArrayIndex idx=loc.getIndex();
-                       CkLocRec_local *rec = loc.getLocalRecord();
+public:
+       ElementDistributor(CkLocMgr *mgr_,int *toPE_):locMgr(mgr_),targetPE(toPE_){};
+
+       void addLocation(CkLocation &loc){
+
+               // leaving object on this PE
+               if(*targetPE == CkMyPe()){
+                       *targetPE = (*targetPE +1)%CkNumPes();
+                       return;
+               }
                        
-                       CkPrintf("[%d] Distributing objects to Processor %d: ",CkMyPe(),*targetPE);
-                       idx.print();
+               CkArrayIndexMax idx = loc.getIndex();
+               CkLocRec_local *rec = loc.getLocalRecord();
+               CkLocMgr *locMgr = loc.getManager();
+               CkVec<CkMigratable *> eltList;
                        
+               CkPrintf("[%d] Distributing objects to Processor %d: ",CkMyPe(),*targetPE);
+               idx.print();
 
-                       //TODO: an element that is being moved should leave some trace behind so that
-                       // the arraybroadcaster can forward messages to it
-                       
-                       //pack up this location and send it across
-                       PUP::sizer psizer;
-                       pupLocation(loc,psizer);
-                       int totalSize = psizer.size()+CmiMsgHeaderSizeBytes;
-                       char *msg = (char *)CmiAlloc(totalSize);
-                       char *buf = &msg[CmiMsgHeaderSizeBytes];
-                       PUP::toMem pmem(buf);
-                       pmem.becomeDeleting();
-                       pupLocation(loc,pmem);
+               // incrementing number of emigrant objects
+               CpvAccess(_numEmigrantRecObjs)++;
+       locMgr->migratableList((CkLocRec_local *)rec,eltList);
+               CkReductionMgr *reductionMgr = (CkReductionMgr*)CkpvAccess(_groupTable)->find(eltList[0]->mlogData->objID.data.array.id).getObj();
+               
+               // let everybody else know the object is leaving
+               locMgr->callMethod(rec,&CkMigratable::ckAboutToMigrate);
+               reductionMgr->incNumEmigrantRecObjs();
+       
+               //pack up this location and send it across
+               PUP::sizer psizer;
+               pupLocation(loc,psizer);
+               int totalSize = psizer.size() + sizeof(DistributeObjectMsg);
+               char *msg = (char *)CmiAlloc(totalSize);
+               DistributeObjectMsg *distributeMsg = (DistributeObjectMsg *)msg;
+               distributeMsg->PE = CkMyPe();
+               char *buf = &msg[sizeof(DistributeObjectMsg)];
+               PUP::toMem pmem(buf);
+               pmem.becomeDeleting();
+               pupLocation(loc,pmem);
                        
-                       locMgr->setDuringMigration(CmiTrue);                    
-                       delete rec;
-                       locMgr->setDuringMigration(CmiFalse);                   
-                       locMgr->inform(idx,*targetPE);
+               locMgr->setDuringMigration(CmiTrue);
+               delete rec;
+               locMgr->setDuringMigration(CmiFalse);
+               locMgr->inform(idx,*targetPE);
 
-                       CmiSetHandler(msg,_distributedLocationHandlerIdx);
-                       CmiSyncSendAndFree(*targetPE,totalSize,msg);
+               CmiSetHandler(msg,_distributedLocationHandlerIdx);
+               CmiSyncSendAndFree(*targetPE,totalSize,msg);
 
-                       CmiAssert(locMgr->lastKnown(idx) == *targetPE);
-                       //decide on the target processor for the next object
-                       *targetPE = (*targetPE +1)%CkNumPes();
+               CmiAssert(locMgr->lastKnown(idx) == *targetPE);
+
+               //decide on the target processor for the next object
+               *targetPE = *targetPE + 1;
+               if(*targetPE > (CkMyPe() + parallelRecovery)){
+                       *targetPE = CkMyPe() + 1;
                }
-               
+       }
+
+};
+
+/**
+ * Distributes objects to accelerate recovery after a failure.
+ */
+void distributeRestartedObjects(){
+       int numGroups = CkpvAccess(_groupIDTable)->size();      
+       int i;
+       int targetPE=CkMyPe()+1;
+       CKLOCMGR_LOOP(ElementDistributor distributor(mgr,&targetPE);mgr->iterate(distributor););
 };
 
-void distributeRestartedObjects(){
-       int numGroups = CkpvAccess(_groupIDTable)->size();      
-       int i;
-       int targetPE=CkMyPe();
-       CKLOCMGR_LOOP(ElementDistributor distributor(mgr,&targetPE);mgr->iterate(distributor););
-};
+/**
+ * Handler to receive back a location.
+ */
+void _sendBackLocationHandler(char *receivedMsg){
+       printf("Array element received at processor %d after recovery\n",CkMyPe());
+       DistributeObjectMsg *distributeMsg = (DistributeObjectMsg *)receivedMsg;
+       int sourcePE = distributeMsg->PE;
+       char *buf = &receivedMsg[sizeof(DistributeObjectMsg)];
+       PUP::fromMem pmem(buf);
+       CkGroupID gID;
+       CkArrayIndexMax idx;
+       pmem |gID;
+       pmem |idx;
+       CkLocMgr *mgr = (CkLocMgr*)CkpvAccess(_groupTable)->find(gID).getObj();
+       donotCountMigration=1;
+       mgr->resume(idx,pmem,CmiTrue);
+       donotCountMigration=0;
+       informLocationHome(gID,idx,mgr->homePe(idx),CkMyPe());
+       printf("Array element inserted at processor %d after parallel recovery\n",CkMyPe());
+       idx.print();
+
+       // decrementing number of emigrant objects at reduction manager
+       CkVec<CkMigratable *> eltList;
+       CkLocRec *rec = mgr->elementRec(idx);
+       mgr->migratableList((CkLocRec_local *)rec,eltList);
+       CkReductionMgr *reductionMgr = (CkReductionMgr*)CkpvAccess(_groupTable)->find(eltList[0]->mlogData->objID.data.array.id).getObj();
+       reductionMgr->decNumEmigrantRecObjs();
+       reductionMgr->decGCount();
+
+       // checking if it has received all emigrant recovering objects
+       CpvAccess(_numEmigrantRecObjs)--;
+       if(CpvAccess(_numEmigrantRecObjs) == 0){
+               (*resumeLbFnPtr)(centralLb);
+       }
+
+}
 
+/**
+ * Handler to update information about an object just received.
+ */
 void _distributedLocationHandler(char *receivedMsg){
        printf("Array element received at processor %d after distribution at restart\n",CkMyPe());
-       char *buf = &receivedMsg[CmiMsgHeaderSizeBytes];
+       DistributeObjectMsg *distributeMsg = (DistributeObjectMsg *)receivedMsg;
+       int sourcePE = distributeMsg->PE;
+       char *buf = &receivedMsg[sizeof(DistributeObjectMsg)];
        PUP::fromMem pmem(buf);
        CkGroupID gID;
-       CkArrayIndex idx;
+       CkArrayIndexMax idx;
        pmem |gID;
        pmem |idx;
        CkLocMgr *mgr = (CkLocMgr*)CkpvAccess(_groupTable)->find(gID).getObj();
@@ -3044,23 +1581,33 @@ void _distributedLocationHandler(char *receivedMsg){
 
        CkLocRec *rec = mgr->elementRec(idx);
        CmiAssert(rec->type() == CkLocRec::local);
+
+       // adding object to the list of immigrant recovery objects
+       CpvAccess(_immigrantRecObjs)->push_back(new CkLocation(mgr,(CkLocRec_local *)rec));
+       CpvAccess(_numImmigrantRecObjs)++;
        
        CkVec<CkMigratable *> eltList;
        mgr->migratableList((CkLocRec_local *)rec,eltList);
        for(int i=0;i<eltList.size();i++){
                if(eltList[i]->mlogData->toResumeOrNot == 1 && eltList[i]->mlogData->resumeCount < globalResumeCount){
                        CpvAccess(_currentObj) = eltList[i];
+                       eltList[i]->mlogData->immigrantRecFlag = 1;
+                       eltList[i]->mlogData->immigrantSourcePE = sourcePE;
+
+                       // incrementing immigrant counter at reduction manager
+                       CkReductionMgr *reductionMgr = (CkReductionMgr*)CkpvAccess(_groupTable)->find(eltList[i]->mlogData->objID.data.array.id).getObj();
+                       reductionMgr->incNumImmigrantRecObjs();
+                       reductionMgr->decGCount();
+
                        eltList[i]->ResumeFromSync();
                }
        }
-       
-       
 }
 
 
 /** this method is used to send messages to a restarted processor to tell
  * it that a particular expected object is not going to get to it */
-void sendDummyMigration(int restartPE,CkGroupID lbID,CkGroupID locMgrID,CkArrayIndex &idx,int locationPE){
+void sendDummyMigration(int restartPE,CkGroupID lbID,CkGroupID locMgrID,CkArrayIndexMax &idx,int locationPE){
        DummyMigrationMsg buf;
        buf.flag = MLOG_OBJECT;
        buf.lbID = lbID;
@@ -3096,12 +1643,12 @@ void sendDummyMigrationCounts(int *dummyCounts){
 void _dummyMigrationHandler(DummyMigrationMsg *msg){
        CentralLB *lb = (CentralLB *)CkpvAccess(_groupTable)->find(msg->lbID).getObj();
        if(msg->flag == MLOG_OBJECT){
-               DEBUGRESTART(CmiPrintf("[%d] dummy Migration received from pe %d for %d:%s \n",CmiMyPe(),msg->locationPE,msg->mgrID.idx,idx2str(msg->idx)));
+               DEBUG_RESTART(CmiPrintf("[%d] dummy Migration received from pe %d for %d:%s \n",CmiMyPe(),msg->locationPE,msg->mgrID.idx,idx2str(msg->idx)));
                LDObjHandle h;
                lb->Migrated(h,1);
        }
        if(msg->flag == MLOG_COUNT){
-               DEBUGRESTART(CmiPrintf("[%d] dummyMigration count %d received from restarted processor\n",CmiMyPe(),msg->count));
+               DEBUG_RESTART(CmiPrintf("[%d] dummyMigration count %d received from restarted processor\n",CmiMyPe(),msg->count));
                msg->count -= verifyAckedRequests;
                for(int i=0;i<msg->count;i++){
                        LDObjHandle h;
@@ -3144,7 +1691,7 @@ public:
 /**
  * Map function pointed by fnPointer over all the chares living in this processor.
  */
-void forAllCharesDo(MlogFn fnPointer,void *data){
+void forAllCharesDo(MlogFn fnPointer, void *data){
        int numGroups = CkpvAccess(_groupIDTable)->size();
        for(int i=0;i<numGroups;i++){
                Chare *obj = (Chare *)CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
@@ -3164,6 +1711,11 @@ void forAllCharesDo(MlogFn fnPointer,void *data){
  Load Balancing
 ******************************************************************/
 
+/**
+ * This is the first time Converse is called after AtSync method has been called by every local object.
+ * It is a good place to insert some optimizations for synchronized checkpoint. In the case of causal
+ * message logging, we can take advantage of this situation and garbage collect at this point.
+ */
 void initMlogLBStep(CkGroupID gid){
        DEBUGLB(CkPrintf("[%d] INIT MLOG STEP\n",CkMyPe()));
        countLBMigratedAway = 0;
@@ -3175,18 +1727,120 @@ void initMlogLBStep(CkGroupID gid){
                CmiAssert(globalLBID.idx == gid.idx);
        }
        globalLBID = gid;
+#if SYNCHRONIZED_CHECKPOINT
+       garbageCollectMlog();
+#endif
+}
+
+/**
+ * Pups a location
+ */
+void pupLocation(CkLocation *loc, CkLocMgr *locMgr, PUP::er &p){
+       CkArrayIndexMax idx = loc->getIndex();
+       CkGroupID gID = locMgr->ckGetGroupID();
+       p|gID;      // store loc mgr's GID as well for easier restore
+       p|idx;
+       p|*loc;
+};
+
+/**
+ * Sends back the immigrant recovering object to their origin PE.
+ */
+void sendBackImmigrantRecObjs(){
+       CkLocation *loc;
+       CkLocMgr *locMgr;
+       CkArrayIndexMax idx;
+       CkLocRec_local *rec;
+       PUP::sizer psizer;
+       int targetPE;
+       CkVec<CkMigratable *> eltList;
+       CkReductionMgr *reductionMgr;
+       // looping through all elements in immigrant recovery objects vector
+       for(int i=0; i<CpvAccess(_numImmigrantRecObjs); i++){
+
+               // getting the components of each location
+               loc = (*CpvAccess(_immigrantRecObjs))[i];
+               idx = loc->getIndex();
+               rec = loc->getLocalRecord();
+               locMgr = loc->getManager();
+       locMgr->migratableList((CkLocRec_local *)rec,eltList);
+               targetPE = eltList[i]->mlogData->immigrantSourcePE;
+
+               // decrement counter at array manager
+               reductionMgr = (CkReductionMgr*)CkpvAccess(_groupTable)->find(eltList[i]->mlogData->objID.data.array.id).getObj();
+               reductionMgr->decNumImmigrantRecObjs();
+
+               CkPrintf("[%d] Sending back object to %d: ",CkMyPe(),targetPE);
+               idx.print();
+
+               // let everybody else know the object is leaving
+               locMgr->callMethod(rec,&CkMigratable::ckAboutToMigrate);
+                       
+               //pack up this location and send it across
+               pupLocation(loc,locMgr,psizer);
+               int totalSize = psizer.size() + sizeof(DistributeObjectMsg);
+               char *msg = (char *)CmiAlloc(totalSize);
+               DistributeObjectMsg *distributeMsg = (DistributeObjectMsg *)msg;
+               distributeMsg->PE = CkMyPe();
+               char *buf = &msg[sizeof(DistributeObjectMsg)];
+               PUP::toMem pmem(buf);
+               pmem.becomeDeleting();
+               pupLocation(loc,locMgr,pmem);
+               
+               locMgr->setDuringMigration(CmiTrue);
+               delete rec;
+               locMgr->setDuringMigration(CmiFalse);
+               locMgr->inform(idx,targetPE);
+
+               // sending the object
+               CmiSetHandler(msg,_sendBackLocationHandlerIdx);
+               CmiSyncSendAndFree(targetPE,totalSize,msg);
+
+               // freeing memory
+               delete loc;
+
+               CmiAssert(locMgr->lastKnown(idx) == targetPE);
+               
+       }
+
+       // cleaning up all data structures
+       CpvAccess(_immigrantRecObjs)->removeAll();
+       CpvAccess(_numImmigrantRecObjs) = 0;
+
+}
+
+/**
+ * Restores objects after parallel recovery, either by sending back the immigrant objects or 
+ * by waiting for all emigrant objects to be back.
+ */
+void restoreParallelRecovery(void (*_fnPtr)(void *),void *_centralLb){
+       resumeLbFnPtr = _fnPtr;
+       centralLb = _centralLb;
+
+       // sending back the immigrant recovering objects
+       if(CpvAccess(_numImmigrantRecObjs) > 0){
+               sendBackImmigrantRecObjs();     
+       }
+
+       // checking whether it needs to wait for emigrant recovery objects
+       if(CpvAccess(_numEmigrantRecObjs) > 0)
+               return;
+
+       // otherwise, load balancing process is finished
+       (*resumeLbFnPtr)(centralLb);
 }
 
 void startLoadBalancingMlog(void (*_fnPtr)(void *),void *_centralLb){
        DEBUGLB(printf("[%d] start Load balancing section of message logging \n",CmiMyPe()));
        DEBUG_TEAM(printf("[%d] start Load balancing section of message logging \n",CmiMyPe()));
-       
+
        resumeLbFnPtr = _fnPtr;
        centralLb = _centralLb;
        migrationDoneCalled = 1;
        if(countLBToMigrate == countLBMigratedAway){
                DEBUGLB(printf("[%d] calling startMlogCheckpoint in startLoadBalancingMlog countLBToMigrate %d countLBMigratedAway %d \n",CmiMyPe(),countLBToMigrate,countLBMigratedAway));
-               startMlogCheckpoint(NULL,CmiWallTimer());       
+               startMlogCheckpoint(NULL,CmiWallTimer());
        }
 };
 
@@ -3201,59 +1855,6 @@ void finishedCheckpointLoadBalancing(){
        
 };
 
-
-void sendMlogLocation(int targetPE,envelope *env){
-       void *_msg = EnvToUsr(env);
-       CkArrayElementMigrateMessage *msg = (CkArrayElementMigrateMessage *)_msg;
-
-
-       int existing = 0;
-       //if this object is already in the retainedobjectlust destined for this
-       //processor it should not be sent
-       
-       for(int i=0;i<retainedObjectList.size();i++){
-               MigrationRecord &migRecord = retainedObjectList[i]->migRecord;
-               if(migRecord.gID == msg->gid && migRecord.idx == msg->idx){
-                       DEBUG(CmiPrintf("[%d] gid %d idx %s being sent to %d exists in retainedObjectList with toPE %d\n",CmiMyPe(),msg->gid.idx,idx2str(msg->idx),targetPE,migRecord.toPE));
-                       existing = 1;
-                       break;
-               }
-       }
-
-       if(existing){
-               return;
-       }
-       
-       
-       countLBToMigrate++;
-       
-       MigrationNotice migMsg;
-       migMsg.migRecord.gID = msg->gid;
-       migMsg.migRecord.idx = msg->idx;
-       migMsg.migRecord.fromPE = CkMyPe();
-       migMsg.migRecord.toPE =  targetPE;
-       
-       DEBUGLB(printf("[%d] Sending array to proc %d gid %d idx %s\n",CmiMyPe(),targetPE,msg->gid.idx,idx2str(msg->idx)));
-       
-       RetainedMigratedObject  *retainedObject = new RetainedMigratedObject;
-       retainedObject->migRecord = migMsg.migRecord;
-       retainedObject->acked  = 0;
-       
-       CkPackMessage(&env);
-       
-       migMsg.record = retainedObject;
-       retainedObject->msg = env;
-       int size = retainedObject->size = env->getTotalsize();
-       
-       retainedObjectList.push_back(retainedObject);
-       
-       CmiSetHandler((void *)&migMsg,_receiveMigrationNoticeHandlerIdx);
-       CmiSyncSend(getCheckPointPE(),sizeof(migMsg),(char *)&migMsg);
-       
-       DEBUGLB(printf("[%d] Location in message of size %d being sent to PE %d\n",CkMyPe(),size,targetPE));
-
-}
-
 void _receiveMigrationNoticeHandler(MigrationNotice *msg){
        msg->migRecord.ackFrom = msg->migRecord.ackTo = 0;
        migratedNoticeList.push_back(msg->migRecord);
@@ -3297,18 +1898,9 @@ void _receiveMlogLocationHandler(void *buf){
        mgr->immigrate(msg);
 };
 
-
-void resumeFromSyncRestart(void *data,ChareMlogData *mlogData){
-/*     if(mlogData->objID.type == TypeArray){
-               CkMigratable *elt = (CkMigratable *)mlogData->objID.getObject();
-       //      TODO: make sure later that atSync has been called and it needs 
-       //      to be resumed from sync
-       //
-               CpvAccess(_currentObj) = elt;
-               elt->ResumeFromSync();
-       }*/
-}
-
+/**
+ * @brief Processor 0 sends a broadcast to every other processor after checkpoint barrier.
+ */
 inline void checkAndSendCheckpointBarrierAcks(CheckpointBarrierMsg *msg){
        if(checkpointBarrierCount == CmiNumPes()){
                CmiSetHandler(msg,_checkpointBarrierAckHandlerIdx);
@@ -3318,6 +1910,9 @@ inline void checkAndSendCheckpointBarrierAcks(CheckpointBarrierMsg *msg){
        }
 }
 
+/**
+ * @brief Processor 0 receives a contribution from every other processor after checkpoint.
+ */ 
 void _checkpointBarrierHandler(CheckpointBarrierMsg *msg){
        DEBUG(CmiPrintf("[%d] msg->checkpointCount %d pe %d checkpointCount %d checkpointBarrierCount %d \n",CmiMyPe(),msg->checkpointCount,msg->fromPE,checkpointCount,checkpointBarrierCount));
        if(msg->checkpointCount == checkpointCount){
@@ -3332,23 +1927,55 @@ void _checkpointBarrierHandler(CheckpointBarrierMsg *msg){
                        CmiAbort("msg->checkpointCount and checkpointCount differ by more than 1");
                }
        }
+
+       // deleting the received message
        CmiFree(msg);
 }
 
 void _checkpointBarrierAckHandler(CheckpointBarrierMsg *msg){
        DEBUG(CmiPrintf("[%d] _checkpointBarrierAckHandler \n",CmiMyPe()));
        DEBUGLB(CkPrintf("[%d] Reaching this point\n",CkMyPe()));
-       sendRemoveLogRequests();
+
+       // resuming LB function pointer
        (*resumeLbFnPtr)(centralLb);
+
+       // deleting message
        CmiFree(msg);
 }
 
+/**
+ * @brief Function to remove all messages in the message log of a particular chare.
+ */
+void garbageCollectMlogForChare(void *data, ChareMlogData *mlogData){
+       int total;
+       MlogEntry *logEntry;
+       CkQ<MlogEntry *> *mlog = mlogData->getMlog();
+
+       // traversing the whole message log and removing all elements
+       total = mlog->length();
+       for(int i=0; i<total; i++){
+               logEntry = mlog->deq();
+               delete logEntry;
+       }
+
+}
+
+/**
+ * @brief Garbage collects the message log and other data structures.
+ */
+void garbageCollectMlog(){
+       DEBUG(CkPrintf("[%d] Garbage collecting message log and data structures\n", CkMyPe()));
+
+       // removing all messages in message log for every chare
+       forAllCharesDo(garbageCollectMlogForChare, NULL);
+}
+
 /**
        method that informs an array elements home processor of its current location
        It is a converse method to bypass the charm++ message logging framework
 */
 
-void informLocationHome(CkGroupID locMgrID,CkArrayIndex idx,int homePE,int currentPE){
+void informLocationHome(CkGroupID locMgrID,CkArrayIndexMax idx,int homePE,int currentPE){
        double _startTime = CmiWallTimer();
        CurrentLocationMsg msg;
        msg.mgrID = locMgrID;
@@ -3413,30 +2040,79 @@ void _getGlobalStepHandler(LBStepMsg *msg){
        CmiSyncSend(msg->fromPE,sizeof(LBStepMsg),(char *)msg);
 };
 
+/**
+ * @brief Receives the global step handler from PE 0
+ */
 void _recvGlobalStepHandler(LBStepMsg *msg){
        
-       restartDecisionNumber=msg->step;
-       RestartRequest *dummyAck = (RestartRequest *)CmiAlloc(sizeof(RestartRequest));
-       _updateHomeAckHandler(dummyAck);
+       // updating restart decision number
+       restartDecisionNumber = msg->step;
+       CmiFree(msg);
+
+       CmiPrintf("[%d] recvGlobalStepHandler \n",CmiMyPe());
+
+       // continuing with restart process; send out the request to resend logged messages to all other processors
+       CkVec<CkObjID> objectVec;
+       forAllCharesDo(createObjIDList, (void *)&objectVec);
+       int numberObjects = objectVec.size();
+       
+       //      resendMsg layout: |ResendRequest|Array of CkObjID|
+       int totalSize = sizeof(ResendRequest) + numberObjects * sizeof(CkObjID);
+       char *resendMsg = (char *)CmiAlloc(totalSize);  
+
+       ResendRequest *resendReq = (ResendRequest *)resendMsg;
+       resendReq->PE = CkMyPe(); 
+       resendReq->numberObjects = numberObjects;
+       char *objList = &resendMsg[sizeof(ResendRequest)];
+       memcpy(objList,objectVec.getVec(),numberObjects * sizeof(CkObjID));     
+
+       CentralLB *lb = (CentralLB *)CkpvAccess(_groupTable)->find(globalLBID).getObj();
+       CpvAccess(_currentObj) = lb;
+       lb->ReceiveDummyMigration(restartDecisionNumber);
+
+       CmiSetHandler(resendMsg,_resendMessagesHandlerIdx);
+       for(int i=0;i<CkNumPes();i++){
+               if(i != CkMyPe()){
+                       CmiSyncSend(i,totalSize,resendMsg);
+               }
+       }
+       _resendMessagesHandler(resendMsg);
+       CmiFree(resendMsg);
+
+       /* test for parallel restart migrate away object**/
+       if(fastRecovery){
+               distributeRestartedObjects();
+               printf("[%d] Redistribution of objects done at %.6lf \n",CkMyPe(),CmiWallTimer());
+       }
+
 };
 
 /**
  * @brief Function to wrap up performance information.
  */
 void _messageLoggingExit(){
-/*     if(CkMyPe() == 0){
-               if(countBuffered != 0){
-                       printf("[%d] countLocal %d countBuffered %d countPiggy %d Effeciency blocking %.2lf \n",CkMyPe(),countLocal,countBuffered,countPiggy,countLocal/(double )(countBuffered*_maxBufferedMessages));
-               }
+       
+       // printing the signature for causal message logging
+       if(CkMyPe() == 0)
+               printf("[%d] FastMessageLoggingExit \n",CmiMyPe());
 
-//             printf("[%d] totalSearchRestoredTime = %.6lf totalSearchRestoredCount %.1lf \n",CkMyPe(),totalSearchRestoredTime,totalSearchRestoredCount);     
+#if COLLECT_STATS_MSGS
+#if COLLECT_STATS_MSGS_TOTAL
+       printf("[%d] TOTAL MESSAGES SENT: %d\n",CmiMyPe(),totalMsgsTarget);
+       printf("[%d] TOTAL MESSAGES SENT SIZE: %.2f MB\n",CmiMyPe(),totalMsgsSize/(float)MEGABYTE);
+#else
+       printf("[%d] TARGETS: ",CmiMyPe());
+       for(int i=0; i<CmiNumPes(); i++){
+#if COLLECT_STATS_MSG_COUNT
+               printf("%d ",numMsgsTarget[i]);
+#else
+               printf("%d ",sizeMsgsTarget[i]);
+#endif
        }
-       printf("[%d] countHashCollisions %d countHashRefs %d \n",CkMyPe(),countHashCollisions,countHashRefs);*/
-       printf("[%d] _messageLoggingExit \n",CmiMyPe());
+       printf("\n");
+#endif
+#endif
 
-       //TML: printing some statistics for group approach
-       //if(teamSize > 1)
-               CkPrintf("[%d] Logged messages = %.0f, log size =  %.2f MB\n",CkMyPe(),MLOGFT_totalMessages,MLOGFT_totalLogSize/(float)MEGABYTE);
 
 }
 
@@ -3474,7 +2150,7 @@ void* CkObjID::getObject(){
        
                                        if(aid.ckLocalBranch() == NULL){ return NULL;}
        
-                                       CProxyElement_ArrayBase aProxy(aid,data.array.idx);
+                                       CProxyElement_ArrayBase aProxy(aid,data.array.idx.asChild());
        
                                        return aProxy.ckLocal();
                                }
@@ -3498,7 +2174,7 @@ int CkObjID::guessPE(){
                                        if(aid.ckLocalBranch() == NULL){
                                                return -1;
                                        }
-                                       return aid.ckLocalBranch()->lastKnown(data.array.idx);
+                                       return aid.ckLocalBranch()->lastKnown(data.array.idx.asChild());
                                }
                        default:
                                CkAssert(0);
@@ -3522,7 +2198,7 @@ char *CkObjID::toString(char *buf) const {
                        break;
                case TypeArray:
                        {
-                               const CkArrayIndex &idx = data.array.idx;
+                               const CkArrayIndexMax &idx = data.array.idx.asChild();
                                const int *indexData = idx.data();
                                sprintf(buf,"Array |%d %d %d| id %d \0",indexData[0],indexData[1],indexData[2],data.array.id.idx);
                                break;
@@ -3548,14 +2224,14 @@ void CkObjID::updatePosition(int PE){
                                                char str[100];
                                                CkLocMgr *mgr = aid.ckLocalBranch()->getLocMgr();
 //                                             CmiPrintf("[%d] location for object %s is %d\n",CmiMyPe(),toString(str),PE);
-                                               CkLocRec *rec = mgr->elementNrec(data.array.idx);
+                                               CkLocRec *rec = mgr->elementNrec(data.array.idx.asChild());
                                                if(rec != NULL){
                                                        if(rec->type() == CkLocRec::local){
                                                                CmiPrintf("[%d] local object %s can not exist on another processor %d\n",CmiMyPe(),str,PE);
                                                                return;
                                                        }
                                                }
-                                               mgr->inform(data.array.idx,PE);
+                                               mgr->inform(data.array.idx.asChild(),PE);
                                        }       
                                }
 
@@ -3573,12 +2249,9 @@ void CkObjID::updatePosition(int PE){
        }
 }
 
-
-
 void MlogEntry::pup(PUP::er &p){
        p | destPE;
        p | _infoIdx;
-       p | unackedLocal;
        int size;
        if(!p.isUnpacking()){
 /*             CkAssert(env);
@@ -3602,142 +2275,26 @@ void MlogEntry::pup(PUP::er &p){
        }
        if(size > 0){
                p((char *)env,size);
-       
-               if(p.isUnpacking()){
-                       env->localMlogEntry = NULL;
-               }
-       }
-};
-
-void RestoredLocalMap::pup(PUP::er &p){
-       p | minSN;
-       p | maxSN;
-       p | count;
-       if(p.isUnpacking()){
-               TNArray = new MCount[count];
        }
-       p(TNArray,count);
 };
 
 
-
-
 /**********************************
        * The methods of the message logging
        * data structure stored in each chare
        ********************************/
 
 MCount ChareMlogData::nextSN(const CkObjID &recver){
-/*     MCount SN = snTable.get(recver);
-       snTable.put(recver) = SN+1;
-       return SN+1;*/
-       double _startTime = CmiWallTimer();
-       MCount *SN = snTable.getPointer(recver);
+       MCount *SN = ssnTable.getPointer(recver);
        if(SN==NULL){
-               snTable.put(recver) = 1;
+               ssnTable.put(recver) = 1;
                return 1;
        }else{
                (*SN)++;
                return *SN;
        }
-//     traceUserBracketEvent(34,_startTime,CkWallTimer());
-};
-
-
-MCount ChareMlogData::newTN(){
-       MCount TN;
-       if(currentHoles > 0){
-               int holeidx = numberHoles-currentHoles;
-               TN = ticketHoles[holeidx];
-               currentHoles--;
-               if(currentHoles == 0){
-                       delete []ticketHoles;
-                       numberHoles = 0;
-               }
-       }else{
-               TN = ++tCount;
-       }       
-       return TN;
-};
-
-/**
- * Inserts a ticket in the ticketTable if it is not already there.
- */
-inline void ChareMlogData::verifyTicket(CkObjID &sender, MCount SN, MCount TN){
-       Ticket ticket;
-
-       SNToTicket *ticketRow = ticketTable.get(sender);
-       if(ticketRow != NULL){
-               Ticket earlierTicket = ticketRow->get(SN);
-               if(earlierTicket.TN != 0){
-                       CkAssert(earlierTicket.TN == TN);
-                       return;
-               }
-       }else{
-               ticketRow = new SNToTicket();
-               ticketTable.put(sender) = ticketRow;
-       }
-       ticket.TN = TN;
-       ticketRow->put(SN) = ticket;
-}
-
-/**
- * Generates the next ticket for a request.
- */
-inline Ticket ChareMlogData::next_ticket(CkObjID &sender,MCount SN){
-       DEBUG(char senderName[100];)
-       DEBUG(char recverName[100];)
-       double _startTime =CmiWallTimer();
-       Ticket ticket;
-
-       // if a ticket is requested during restart, 0 is returned to make the requester to ask for it later.
-       if(restartFlag){
-               ticket.TN = 0;
-               return ticket;
-       }
-/*     SNToTicket &ticketRow = ticketTable.put(sender);
-       Ticket earlierTicket = ticketRow.get(SN);
-       if(earlierTicket.TN == 0){
-               //This SN has not been ever alloted a ticket
-               ticket.TN = newTN();
-               ticketRow.put(SN)=ticket;
-       }else{
-               ticket.TN = earlierTicket.TN;
-       }*/
-       
-
-       SNToTicket *ticketRow = ticketTable.get(sender);
-       if(ticketRow != NULL){
-               Ticket earlierTicket = ticketRow->get(SN);
-               if(earlierTicket.TN == 0){
-                       ticket.TN = newTN();
-                       ticketRow->put(SN) = ticket;
-                       DEBUG(CkAssert((ticketRow->get(SN)).TN == ticket.TN));
-               }else{
-                       ticket.TN = earlierTicket.TN;
-                       if(ticket.TN > tCount){
-                               DEBUG(CmiPrintf("[%d] next_ticket old row ticket sender %s recver %s SN %d TN %d tCount %d\n",CkMyPe(),sender.toString(senderName),objID.toString(recverName),SN,ticket.TN,tCount));
-                       }
-                               CmiAssert(ticket.TN <= tCount);
-               }
-               DEBUG(CmiPrintf("[%d] next_ticket old row ticket sender %s recver %s SN %d TN %d tCount %d\n",CkMyPe(),sender.toString(senderName),objID.toString(recverName),SN,ticket.TN,tCount));
-       }else{
-               SNToTicket *newRow = new SNToTicket;            
-               ticket.TN = newTN();
-               newRow->put(SN) = ticket;
-               ticketTable.put(sender) = newRow;
-               DEBUG(printf("[%d] next_ticket new row ticket sender %s recver %s SN %d TN %d\n",CkMyPe(),sender.toString(senderName),objID.toString(recverName),SN,ticket.TN));
-       }
-/*TODO: check if the message for this SN has already been received
-       in the table of received SNs 
-       If it was received before the last checkpoint mark it as old
-       other wise received
-       */
-       ticket.state = NEW_TICKET;
-//     traceUserBracketEvent(34,_startTime,CkWallTimer());
-       return ticket;  
 };
-
 /**
  * Adds an entry into the message log.
  */
@@ -3750,79 +2307,17 @@ void ChareMlogData::addLogEntry(MlogEntry *entry){
        mlog.enq(entry);
 };
 
-double totalSearchRestoredTime=0;
-double totalSearchRestoredCount=0;
-
 /**
- * Searches the restoredlocal map to see if the combination of sender and sequence number
- * shows up in the map. Returns the ticket if found, or 0 otherwise.
+ * Checks whether a ssn has been already received. The collateral effect is the ssn get added to the list.
  */
-MCount ChareMlogData::searchRestoredLocalQ(CkObjID &sender,CkObjID &recver,MCount SN){
-       double start= CkWallTimer();
-       MCount TN=0;    
-       if(mapTable.numObjects() > 0){
-               RestoredLocalMap *map = mapTable.get(sender);
-               if(map){
-                       int index = SN - map->minSN;
-                       if(index < map->count){
-                               TN = map->TNArray[index];
-                       }
-               }
-       }
-       
-       DEBUG(char senderName[100]);
-       DEBUG(char recverName[100]);
-       DEBUG(if(TN != 0){ CmiPrintf("[%d] searchRestoredLocalQ found match sender %s recver %s SN %d TN %d\n",CmiMyPe(),sender.toString(senderName),recver.toString(recverName),SN,TN);});
-
-       totalSearchRestoredTime += CkWallTimer()-start;
-       totalSearchRestoredCount++;
-       return TN;
-}
-
-void ChareMlogData::addToRestoredLocalQ(LocalMessageLog *logEntry){
-       restoredLocalMsgLog.push_back(*logEntry);
-}
-
-void sortRestoredLocalMsgLog(void *_dummy,ChareMlogData *mlogData){
-       mlogData->sortRestoredLocalMsgLog();
-}
-
-void ChareMlogData::sortRestoredLocalMsgLog(){
-       //sort it ->its bloddy bubble sort
-       
-       for(int i=0;i<restoredLocalMsgLog.size();i++){
-               LocalMessageLog &logEntry = restoredLocalMsgLog[i];
-               RestoredLocalMap *map = mapTable.get(logEntry.sender);
-               if(map == NULL){
-                       map = new RestoredLocalMap;
-                       mapTable.put(logEntry.sender)=map;
-               }
-               map->count++;
-               if(map->minSN == 0){
-                       map->minSN = logEntry.SN;
-               }else{
-                       if(logEntry.SN < map->minSN){
-                               map->minSN = logEntry.SN;
-                       }
-               }
-               if(logEntry.SN > map->maxSN){
-                       map->maxSN = logEntry.SN;
-               }
-
-       }
-       for(int i=0;i< restoredLocalMsgLog.size();i++){
-               LocalMessageLog &logEntry = restoredLocalMsgLog[i];
-               RestoredLocalMap *map = mapTable.get(logEntry.sender);
-               CkAssert(map != NULL);
-               if(map->TNArray == NULL){
-                       map->TNArray = new MCount[map->maxSN-map->minSN+1];                     
-                       CkAssert(map->count == map->maxSN-map->minSN+1);
-                       map->count = 0;
-               }
-               map->TNArray[map->count] = logEntry.TN;
-               map->count++;
-       }
-       restoredLocalMsgLog.free();
+int ChareMlogData::checkAndStoreSsn(const CkObjID &sender, MCount ssn){
+       RSSN *rssn;
+       rssn = receivedSsnTable.get(sender);
+       if(rssn == NULL){
+               rssn = new RSSN();
+               receivedSsnTable.put(sender) = rssn;
+       }
+       return rssn->checkAndStore(ssn);
 }
 
 /**
@@ -3831,21 +2326,13 @@ void ChareMlogData::sortRestoredLocalMsgLog(){
  * Then, we only support one failure at a time. Read Sayantan's thesis, sections 4.2 and 4.3 for more details.
  */
 void ChareMlogData::pup(PUP::er &p){
-       int tCountAux;
        int startSize=0;
        char nameStr[100];
        if(p.isSizing()){
                PUP::sizer *sizep = (PUP::sizer *)&p;
                startSize = sizep->size();
        }
-       double _startTime = CkWallTimer();
-       
        p | objID;
-       if(teamRecoveryFlag)
-               p | tCountAux;
-       else
-               p | tCount;
-       p | tProcessed;
        if(p.isUnpacking()){
                DEBUG(CmiPrintf("[%d] Obj %s being unpacked with tCount %d tProcessed %d \n",CmiMyPe(),objID.toString(nameStr),tCount,tProcessed));
        }
@@ -3853,156 +2340,36 @@ void ChareMlogData::pup(PUP::er &p){
        p | resumeCount;
        DEBUG(CmiPrintf("[%d] Obj %s toResumeOrNot %d resumeCount %d \n",CmiMyPe(),objID.toString(nameStr),toResumeOrNot,resumeCount));
        
-
-       /*pack the receivedTN vector*/
-       int lengthReceivedTNs;
-       if(!p.isUnpacking()){
-               if(receivedTNs == NULL){
-                       lengthReceivedTNs = -1;
-               }else{
-                       lengthReceivedTNs = receivedTNs->size();                
-               }
-       }
-       p | lengthReceivedTNs;
-       if(p.isUnpacking()){
-               if(lengthReceivedTNs == -1){
-                       receivedTNs = NULL;
-               }else{
-                       receivedTNs = new CkVec<MCount>;
-                       for(int i=0;i<lengthReceivedTNs;i++){
-                               MCount tempTicket;
-                               p | tempTicket;
-                               CkAssert(tempTicket > 0);
-                               receivedTNs->push_back(tempTicket);
-                       }
-               }
-       }else{
-               for(int i=0;i<lengthReceivedTNs;i++){
-                       p | (*receivedTNs)[i];
-               }
-       }
-       
-       
-       p | currentHoles;
-       p | numberHoles;
-       if(p.isUnpacking()){
-               if(numberHoles > 0){
-                       ticketHoles = new MCount[numberHoles];                  
-               }else{
-                       ticketHoles = NULL;
-               }
-       }
-       if(numberHoles > 0){
-               p(ticketHoles,numberHoles);
-       }
-       
-       snTable.pup(p);
-
-       // pupping only the unacked local messages in the message log
-       int length = 0;
-       MlogEntry *entry;
-       if(!p.isUnpacking()){
-               for(int i=0; i<mlog.length(); i++){
-                       entry = mlog[i];
-                       if(entry->unackedLocal)
-                               length++;
-               }
-       }
-       p | length;
-       if(p.isUnpacking()){
-               for(int i=0; i<length; i++){
-                       entry = new MlogEntry();
-                       mlog.enq(entry);
-                       entry->pup(p);
-               }
-       }else{
-               for(int i=0; i<mlog.length(); i++){
-                       entry = mlog[i];
-                       if(entry->unackedLocal){
-                               entry->pup(p);
-                       }
-               }
-       }
-
-/*     int length;
-       if(!p.isUnpacking()){           
-               length = mlog.length(); 
-               if(length > 0)
-                       DEBUG(printf("[%d] Mlog length %d \n",CkMyPe(),length));
-       }
-       p | length;
-       for(int i=0;i<length;i++){
-               MlogEntry *entry;
-               if(p.isUnpacking()){
-                       entry = new MlogEntry();
-                       mlog.enq(entry);
-               }else{
-                       entry = mlog[i];
-               }
-               entry->pup(p);
-       }*/
+       ssnTable.pup(p);
        
-       p | restoredLocalMsgLog;
-       p | resendReplyRecvd;
-       p | restartFlag;
-
-       // pup the mapTable
-       int tableSize;
+       // pupping receivedSsnTable
+       int rssnTableSize;
        if(!p.isUnpacking()){
-               tableSize = mapTable.numObjects();
+               rssnTableSize = receivedSsnTable.numObjects();
        }
-       p | tableSize;
+       p | rssnTableSize;
        if(!p.isUnpacking()){
-               CkHashtableIterator *iter = mapTable.iterator();
+               CkHashtableIterator *iter = receivedSsnTable.iterator();
                while(iter->hasNext()){
                        CkObjID *objID;
-                       RestoredLocalMap **map = (RestoredLocalMap **) iter->next((void **)&objID);
+                       RSSN **row = (RSSN **)iter->next((void **)&objID);
                        p | (*objID);
-                       (*map)->pup(p);
+                       (*row)->pup(p);
                }
-               // releasing memory for iterator
                delete iter;
        }else{
-               for(int i=0;i<tableSize;i++){
+               for(int i=0; i<rssnTableSize; i++){
                        CkObjID objID;
                        p | objID;
-                       RestoredLocalMap *map = new RestoredLocalMap;
-                       map->pup(p);
-                       mapTable.put(objID) = map;
+                       RSSN *row = new RSSN;
+                       row->pup(p);
+                       receivedSsnTable.put(objID) = row;
                }
        }
-
-       //pup the ticketTable
-       {
-               int ticketTableSize;
-               if(!p.isUnpacking()){
-                       ticketTableSize = ticketTable.numObjects();
-               }
-               p | ticketTableSize;
-               if(!p.isUnpacking()){
-                       CkHashtableIterator *iter = ticketTable.iterator();
-                       while(iter->hasNext()){
-                               CkObjID *objID;
-                               SNToTicket **ticketRow = (SNToTicket **)iter->next((void **)&objID);
-                               p | (*objID);
-                               (*ticketRow)->pup(p);
-                       }
-                       //releasing memory for iterator
-                       delete iter;
-               }else{
-                       for(int i=0;i<ticketTableSize;i++){
-                               CkObjID objID;
-                               p | objID;
-                               SNToTicket *ticketRow = new SNToTicket;
-                               ticketRow->pup(p);
-                               if(!teamRecoveryFlag)
-                                       ticketTable.put(objID) = ticketRow;
-                               else
-                                       delete ticketRow;
-                       }
-               }
-       }       
        
+       p | resendReplyRecvd;
+       p | restartFlag;
+
        if(p.isSizing()){
                PUP::sizer *sizep = (PUP::sizer *)&p;
                int pupSize = sizep->size()-startSize;
@@ -4020,13 +2387,19 @@ void ChareMlogData::pup(PUP::er &p){
 
 /**
  * Getting the pe number of the current processor's buddy.
+ * In the team-based approach each processor might checkpoint in the next team, but currently
+ * teams are only meant to reduce memory overhead.
+ * Note: function getReverseCheckPointPE performs the reverse map. It must be changed accordingly.
  */
 int getCheckPointPE(){
-       //TML: assigning a team-based buddy
-       if(teamSize != 1){
-               return (CmiMyPe() + teamSize) % CmiNumPes();
-       }
-       return (CmiNumPes() -1 - CmiMyPe());
+       return (CmiMyPe() + 1) % CmiNumPes();
+}
+
+/**
+ * Getting the pe that checkpoints on this pe.
+ */
+int getReverseCheckPointPE(){
+       return (CmiMyPe() - 1 + CmiNumPes()) % CmiNumPes();
 }
 
 //assume it is a packed envelope
index 4c44bbe6cccb4a6a9ba3a31065fecf8d804d07aa..ba560277ecd43c45af439e96904c37b20d710e74 100644 (file)
@@ -10,6 +10,7 @@
 #endif
 
 CpvExtern(Chare *,_currentObj);
+CpvExtern(int, _numImmigrantRecObjs);
 
 //states of a ticket sent as a reply to a request
 #define NEW_TICKET 1
@@ -24,124 +25,102 @@ CpvExtern(Chare *,_currentObj);
 //array on which we print the formatted string representing an object id
 extern char objString[100];
 
-/**
- * @brief
- */
-class Ticket {
-public:
-       MCount TN;
-       int state;
-       Ticket(){
-               TN = 0;
-               state = 0;
-       }
-       Ticket(int x){
-               TN = x;
-               state = 0;
-       }
-};
-PUPbytes(Ticket)
-class MlogEntry;
+// defines the initial size of _bufferedDets
+#define INITIAL_BUFFERED_DETERMINANTS 1024
+
+// constant to define the type of checkpoint used (synchronized or not)
+#define SYNCHRONIZED_CHECKPOINT 1
 
-/**
- * Log entry for local messages, can also be sent as a message.
- * A message is local if:
- * 1) It is sent between objects in the same processor.
- * 2) It is sent between objects residing in processors of the same group
- * (whenever group-based message logging is used).
- */
-typedef struct{
-       char header[CmiMsgHeaderSizeBytes];
-       CkObjID sender;
-       CkObjID recver;
-       MCount SN;
-       MCount TN;
-       MlogEntry *entry;
-       int senderPE;
-       int recverPE;
-} LocalMessageLog;
-PUPbytes(LocalMessageLog)
 
 class MlogEntry;
+
 class RestoredLocalMap;
 
-#define INITSIZE_SNTOTICKET 100
+#define RSSN_INITIAL_SIZE 16
 
 /**
- * @brief Class that maps SN (sequence numbers) to TN (ticket numbers)
- * for a particular object.
+ * @brief Class that stores all received-sender-sequence-numbers (rssn) from another object.
  */
-class SNToTicket{
-       private:
-               Ticket initial[INITSIZE_SNTOTICKET];
-               Ticket *ticketVec;
-               MCount startSN;
-               int currentSize;
-               MCount finishSN;
-       public:
-               SNToTicket(){
-                       currentSize = INITSIZE_SNTOTICKET;
-                       ticketVec = &initial[0];
-                       bzero(ticketVec,sizeof(Ticket)*currentSize);
-                       startSN = 0;
-                       finishSN = 0;
-               }
-               /**
-                * Gets the finishSN value.
-                */ 
-               inline MCount getFinishSN(){
-                       return finishSN;
-               }
-               /**
-                * Gets the startSN value.
-                */      
-               inline MCount getStartSN(){
-                       return startSN;
+class RSSN{
+private:
+       MCount *data;
+       int currentSize, start, end;
+public:
+
+       // Constructor
+       RSSN(){
+               currentSize = RSSN_INITIAL_SIZE;
+               start = 0;
+               end = 0;
+               data = new MCount[RSSN_INITIAL_SIZE];
+               bzero(data,sizeof(MCount)*currentSize);
+       }
+
+       // Checks if a particular SSN is already in the data; if not, stores it         
+       // return value: 0 (sucess, value stored), 1 (value already there)
+       int checkAndStore(MCount ssn){
+               int index, oldCS, num, i;
+               MCount *old;
+
+               // checking if ssn can be inserted, most common case
+               if(start == end && ssn == (data[start] + 1)){
+                       data[start] = ssn;
+                       return 0;
                }
-               //assume indices start from 1.. true for MCounts
-               inline Ticket &put(MCount SN){
-                       if(SN > finishSN) finishSN = SN;
-                       if(startSN == 0){
-                               startSN = SN;                           
-                       }
-                       int index = SN-startSN;
-                       if(index >= currentSize){
-                               int oldSize = currentSize;
-                               Ticket *old = ticketVec;
-                               
-                               currentSize = index*2;
-                               ticketVec = new Ticket[currentSize];
-                               memcpy(ticketVec,old,sizeof(Ticket)*oldSize);
-                               if(old != &initial[0]){                                 
-                                       delete [] old;
-                               }
+
+               // checking if ssn was already received
+               if(ssn <= data[start]) return 1;
+
+               // checking if data needs to be extended
+               if(ssn-data[start] >= currentSize){
+                       old = data;
+                       oldCS = currentSize;
+                       currentSize *= 2;
+                       data = new MCount[currentSize];
+                       bzero(data,sizeof(MCount)*currentSize);
+                       for(i=start, num=0; i!=end; i=(i+1)%oldCS,num++){
+                               data[num] = old[i];
                        }
-                       return ticketVec[index];
+                       start = 0;
+                       end = num-1;
+                       delete[] old;
                }
 
-               inline Ticket get(MCount SN){
-                       int index = SN-startSN;
-                       CmiAssert(index >= 0);
-                       if(index >= currentSize){
-                               Ticket tn;
-                               return tn;
-                       }else{
-                               return ticketVec[index];
-                       }
+               // adding ssn into data
+               num = end - start;
+               if(num < 0) num += currentSize;
+               num++;
+               index = (start+ssn-data[start])%currentSize;
+               data[index] = ssn;
+               if((ssn-data[start]) >= num) end = index;
+
+               // compressing ssn
+               index = start + 1;
+               while(data[index]){
+                       data[start] = 0;
+                       start = index;
+                       index = (index + 1)%currentSize;
+                       if(index == end) break;
                }
+               return 0;
+       }
 
-               inline void pup(PUP::er &p){
-                       p | startSN;
-                       p | currentSize;
-                       if(p.isUnpacking()){
-                               if(currentSize > INITSIZE_SNTOTICKET){
-                                       ticketVec = new Ticket[currentSize];
-                               }
+       // PUP method
+       inline void pup(PUP::er &p){
+               p | start;
+               p | end;
+               p | currentSize;
+               if(p.isUnpacking()){
+                       if(currentSize > RSSN_INITIAL_SIZE){
+                               delete[] data;
+                               data = new MCount[currentSize];
                        }
-                       for(int i=0;i<currentSize;i++){
-                               p | ticketVec[i];
-                       }
-               }       
+               }
+               for(int i=0;i<currentSize;i++){
+                       p | data[i];
+               }
+       }
+
 };
 
 
@@ -158,90 +137,66 @@ class ChareMlogData{
 public:
        // Object unique ID.
        CkObjID objID;
-       // Counts how many tickets have been handed out.
-       MCount tCount; 
-       // Stores the highest ticket that has been processed.
-       MCount tProcessed;
-       
-       //TODO: pup receivedTNs
-       CkVec<MCount> *receivedTNs; //used to store TNs received by senders during a restart
-       MCount *ticketHoles;
-       int numberHoles;
-       int currentHoles;
-       CkVec<LocalMessageLog> restoredLocalMsgLog;
-       int maxRestoredLocalTN;
-       int resendReplyRecvd;// variable that keeps a count of the processors that have replied to a requests to resend messages. 
-       int restartFlag; /*0 -> Normal state .. 1-> just after restart. tickets should not be handed out at this time */
-    int teamRecoveryFlag; // 0 -> normal state .. 1 -> recovery of a team member       
-       CkHashtableT<CkHashtableAdaptorT<CkObjID>,RestoredLocalMap *> mapTable;
-       //TML: teamTable, stores the SN to TN mapping for messages intra team
-       CkHashtableT<CkHashtableAdaptorT<CkObjID>,SNToTicket *> teamTable;
-
+       // variable that keeps a count of the processors that have replied to a requests to resend messages. 
+       int resendReplyRecvd;
+       // 0 -> Normal state .. 1-> just after restart. tickets should not be handed out at this time 
+       int restartFlag;
+       // 0 -> normal state .. 1 -> recovery of a team member 
+    int teamRecoveryFlag;      
        int toResumeOrNot;
        int resumeCount;
+       int immigrantRecFlag;
+       int immigrantSourcePE;
 
 private:
-
-       // SNTable, stores the number of messages sent (sequence numbers) to other objects.
-       CkHashtableT<CkHashtableAdaptorT<CkObjID>,MCount> snTable;
-       // TNTable, stores the ticket associated with a particular combination <ObjectID,SN>.
-       CkHashtableT<CkHashtableAdaptorT<CkObjID>,SNToTicket *> ticketTable;
+       // ssnTable, stores the number of messages sent (sequence numbers) to other objects.
+       CkHashtableT<CkHashtableAdaptorT<CkObjID>, MCount> ssnTable;
+       // receivedSsnTable, stores the list of ssn received from other objects.
+       CkHashtableT<CkHashtableAdaptorT<CkObjID>, RSSN *> receivedSsnTable;
        // Log of messages sent.
        CkQ<MlogEntry *> mlog;
-       
-               
-       inline MCount newTN();
 
 public:
        /**
         * Default constructor.
         */ 
-       ChareMlogData():ticketTable(1000,0.3),snTable(100,0.4),teamTable(100,0.4){
-               tCount = 0;
-               tProcessed = 0;
-               numberHoles = 0;
-               ticketHoles = NULL;
-               currentHoles = 0;
+       ChareMlogData():ssnTable(100,0.4),receivedSsnTable(100,0.4){
                restartFlag=0;
                teamRecoveryFlag=0;
-               receivedTNs = NULL;
                resendReplyRecvd=0;
-               maxRestoredLocalTN=0;
                toResumeOrNot=0;
                resumeCount=0;
+               immigrantRecFlag = 0;
        };
        inline MCount nextSN(const CkObjID &recver);
-       inline Ticket next_ticket(CkObjID &sender,MCount SN);
-       inline void verifyTicket(CkObjID &sender,MCount SN, MCount TN);
+       int checkAndStoreSsn(const CkObjID &sender, MCount ssn);
        void addLogEntry(MlogEntry *entry);
        virtual void pup(PUP::er &p);
        CkQ<MlogEntry *> *getMlog(){ return &mlog;};
-       MCount searchRestoredLocalQ(CkObjID &sender,CkObjID &recver,MCount SN);
-       void addToRestoredLocalQ(LocalMessageLog *logEntry);
-       void sortRestoredLocalMsgLog();
 };
 
 /**
- * @brief Entry in a message log
+ * @brief Entry in a message log. It also includes the index of the buffered
+ * determinants array and the number of appended determinants.
+ * @note: this message appended numBufDets counting downwards from indexBufDets.
+ * In other words, if indexBufDets == 5 and numBufDets = 3, it means that
+ * determinants bufDets[2], bufDets[3] and bufDets[4] were piggybacked.
  */
 class MlogEntry{
 public:
        envelope *env;
        int destPE;
        int _infoIdx;
-       char unackedLocal;
        
        MlogEntry(envelope *_env,int _destPE,int __infoIdx){
                env = _env;
                destPE = _destPE;
                _infoIdx = __infoIdx;
-               unackedLocal = 0;
        }
        MlogEntry(){
                env = 0;
                destPE = -1;
                _infoIdx = 0;
-               unackedLocal = 0;
        }
        ~MlogEntry(){
                if(env){
@@ -251,15 +206,6 @@ public:
        virtual void pup(PUP::er &p);
 };
 
-/**
- * @brief 
- */
-class LocationID{
-public:
-       CkArrayIndex idx;
-       CkGroupID gid;
-};
-
 /**
  * @brief
  */
@@ -275,75 +221,18 @@ public:
        };
 };
 
-/**
- *  @brief Class for storing metadata of local messages.
- *  It maps sequence numbers to ticket numbers.
- *  It is used after a restart to maintain the same ticket numbers.
- */
-class RestoredLocalMap {
-public:
-       MCount minSN,maxSN,count;
-       MCount *TNArray;
-       RestoredLocalMap(){
-               minSN=maxSN=count=0;
-               TNArray=NULL;
-       };
-       RestoredLocalMap(int i){
-               minSN=maxSN=count=0;
-               TNArray=NULL;
-       };
-
-       virtual void pup(PUP::er &p);
-};
-
-
-typedef struct {
-       char header[CmiMsgHeaderSizeBytes];
-       CkObjID sender;
-       CkObjID recver;
-       MlogEntry *logEntry;
-       MCount SN;
-       MCount TN;
-       int senderPE;
-} TicketRequest;
-CpvExtern(CkQ<TicketRequest *> *,_delayedTicketRequests);
-CpvExtern(CkQ<MlogEntry *> *,_delayedLocalTicketRequests);
-
-typedef struct{
-       TicketRequest request;
-       Ticket ticket;
-       int recverPE;
-} TicketReply;
-
-
-CpvExtern(CkQ<LocalMessageLog> *,_localMessageLog); // used on buddy to store local message logs
-
-CpvExtern(CkQ<LocalMessageLog>*,_bufferedLocalMessageLogs);
-extern int _maxBufferedMessages; //Number of local message logs  to be buffered
-
-CpvExtern(char**,_bufferedTicketRequests);
-extern int _maxBufferedTicketRequests; //Number of ticket requests to be buffered
-
-
-
-typedef struct {
-       char header[CmiMsgHeaderSizeBytes];
-       int numberLogs;
-} BufferedLocalLogHeader;
-
-typedef BufferedLocalLogHeader BufferedTicketRequestHeader;
-
-typedef struct{
-       char header[CmiMsgHeaderSizeBytes];
-       MlogEntry *entry;               
-} LocalMessageLogAck;
-
 typedef struct{
        char header[CmiMsgHeaderSizeBytes];
        int PE;
        int dataSize;
 } CheckPointDataMsg;
 
+typedef struct{
+    char header[CmiMsgHeaderSizeBytes];
+    int PE;
+} DistributeObjectMsg;
+
+
 /*typedef struct{
        char header[CmiMsgHeaderSizeBytes];
        int PE;
@@ -352,10 +241,6 @@ typedef struct{
 
 typedef CheckPointDataMsg CheckPointAck;
 
-typedef struct{
-       CkObjID recver;
-       MCount tProcessed;
-} TProcessedLog;
 
 /**
  * Struct to request a particular action during restart.
@@ -394,17 +279,22 @@ typedef struct {
        int numTNs;
 } ReceivedTNData;
 
+// Structure to forward determinants in parallel restart
+typedef struct {
+       char header[CmiMsgHeaderSizeBytes];
+       CkObjID recver;
+       int numDets;
+} ReceivedDetData;
+
 typedef struct{
        int PE;
        int numberObjects;
-       TProcessedLog *listObjects;
-       MCount *maxTickets;
-       CkVec<MCount> *ticketVecs;
+       CkObjID *listObjects;
 } ResendData;
 
 typedef struct {
        CkGroupID gID;
-       CkArrayIndex idx;
+       CkArrayIndexMax idx;
        int fromPE,toPE;
        char ackFrom,ackTo;
 } MigrationRecord;
@@ -445,7 +335,7 @@ typedef struct {
 typedef struct {
        char header[CmiMsgHeaderSizeBytes];
        CkGroupID mgrID;
-       CkArrayIndex idx;
+       CkArrayIndexMax idx;
        int locationPE;
        int fromPE;
 } CurrentLocationMsg;
@@ -468,7 +358,7 @@ typedef struct {
        int count;// if just count
        /**if object **/
        CkGroupID mgrID;
-       CkArrayIndex idx;
+       CkArrayIndexMax idx;
        int locationPE;
 } DummyMigrationMsg;
 
@@ -481,48 +371,21 @@ typedef void (*MlogFn)(void *,ChareMlogData *);
 void _messageLoggingInit();
 
 //Methods for sending ticket requests
-void sendTicketGroupRequest(envelope *env,int destPE,int _infoIdx);
-void sendTicketArrayRequest(envelope *env,int destPE,int _infoIdx);
-void sendTicketNodeGroupRequest(envelope *env,int destNode,int _infoIdx);
-void generateCommonTicketRequest(CkObjID &recver,envelope *env,int destPE,int _infoIdx);
-void sendTicketRequest(CkObjID &sender,CkObjID &recver,int destPE,MlogEntry *entry,MCount SN,MCount TN,int resend);
-void ticketLogLocalMessage(MlogEntry *entry);
-void sendLocalMessageCopy(MlogEntry *entry);
-void sendBufferedLocalMessageCopy();
-void checkBufferedLocalMessageCopy(void *_dummy,double curWallTime);
-void sendBufferedTicketRequests(int destPE);
-void checkBufferedTicketRequests(void *_destPE,double curWallTime);
-
-
-
-
-//handler idxs
-extern int _ticketRequestHandlerIdx;
-extern int _ticketHandlerIdx;
-extern int _localMessageCopyHandlerIdx;
-extern int _localMessageAckHandlerIdx;
-extern int _bufferedLocalMessageCopyHandlerIdx;
-extern int _bufferedLocalMessageAckHandlerIdx;
-extern int _bufferedTicketRequestHandlerIdx;
-extern int _bufferedTicketHandlerIdx;
+void sendGroupMsg(envelope *env,int destPE,int _infoIdx);
+void sendArrayMsg(envelope *env,int destPE,int _infoIdx);
+void sendChareMsg(envelope *env,int destPE,int _infoIdx, const CkChareID *pCid);
+void sendNodeGroupMsg(envelope *env,int destNode,int _infoIdx);
+void sendCommonMsg(CkObjID &recver,envelope *env,int destPE,int _infoIdx);
+void sendRemoteMsg(CkObjID &sender,CkObjID &recver,int destPE,MlogEntry *entry,MCount SN,int resend);
+void sendLocalMsg(envelope *env, int _infoIdx);
 
 //handler functions
-void _ticketRequestHandler(TicketRequest *);
-void _ticketHandler(TicketReply *);
-void _localMessageCopyHandler(LocalMessageLog *);
-void _localMessageAckHandler(LocalMessageLogAck *);
 void _pingHandler(CkPingMsg *msg);
-void _bufferedLocalMessageCopyHandler(BufferedLocalLogHeader *recvdHeader,int freeHeader=1);
-void _bufferedLocalMessageAckHandler(BufferedLocalLogHeader *recvdHeader);
-void _bufferedTicketRequestHandler(BufferedTicketRequestHeader *recvdHeader);
-void _bufferedTicketHandler(BufferedTicketRequestHeader *recvdHeader);
-
 
 //methods for sending messages
 extern void _skipCldEnqueue(int pe,envelope *env, int infoFn);
 extern void _noCldNodeEnqueue(int node, envelope *env);
 void generalCldEnqueue(int destPE,envelope *env,int _infoIdx);
-void retryTicketRequest(void *_ticketRequest,double curWallTime);
 
 //methods to process received messages with respect to mlog
 int preProcessReceivedMessage(envelope *env,Chare **objPointer,MlogEntry **localLogEntry);
@@ -542,6 +405,7 @@ void _checkpointRequestHandler(CheckpointRequest *request);
 void _storeCheckpointHandler(char *msg);
 void _checkpointAckHandler(CheckPointAck *ackMsg);
 void _removeProcessedLogHandler(char *requestMsg);
+void garbageCollectMlog();
 
 //handler idxs for checkpoint
 extern int _checkpointRequestHandlerIdx;
@@ -555,11 +419,9 @@ extern int _removeProcessedLogHandlerIdx;
 //methods for restart
 void CkMlogRestart(const char * dummy, CkArgMsg * dummyMsg);
 void CkMlogRestartDouble(void *,double);
-void processReceivedTN(Chare *obj,int vecsize,MCount *listTNs);
 void initializeRestart(void *data,ChareMlogData *mlogData);
 void distributeRestartedObjects();
-void sortRestoredLocalMsgLog(void *_dummy,ChareMlogData *mlogData);
-void sendDummyMigration(int restartPE,CkGroupID lbID,CkGroupID locMgrID,CkArrayIndex &idx,int locationPE);
+void sendDummyMigration(int restartPE,CkGroupID lbID,CkGroupID locMgrID,CkArrayIndexMax &idx,int locationPE);
 
 //TML: function for locally calling the restart
 void CkMlogRestartLocal();
@@ -568,9 +430,12 @@ void CkMlogRestartLocal();
 void _getCheckpointHandler(RestartRequest *restartMsg);
 void _recvCheckpointHandler(char *_restartData);
 void _resendMessagesHandler(char *msg);
-void _resendReplyHandler(char *msg);
+void _sendDetsHandler(char *msg);
+void _sendDetsReplyHandler(char *msg);
 void _receivedTNDataHandler(ReceivedTNData *msg);
+void _receivedDetDataHandler(ReceivedDetData *msg);
 void _distributedLocationHandler(char *receivedMsg);
+void _sendBackLocationHandler(char *receivedMsg);
 void _updateHomeRequestHandler(RestartRequest *updateRequest);
 void _updateHomeAckHandler(RestartRequest *updateHomeAck);
 void _verifyAckRequestHandler(VerifyAckMsg *verifyRequest);
@@ -582,13 +447,14 @@ void _restartHandler(RestartRequest *restartMsg);
 void _getRestartCheckpointHandler(RestartRequest *restartMsg);
 void _recvRestartCheckpointHandler(char *_restartData);
 
-
 //handler idxs for restart
 extern int _getCheckpointHandlerIdx;
 extern int _recvCheckpointHandlerIdx;
 extern int _resendMessagesHandlerIdx;
-extern int _resendReplyHandlerIdx;
+extern int _sendDetsHandlerIdx;
+extern int _sendDetsReplyHandlerIdx;
 extern int _receivedTNDataHandlerIdx;
+extern int _receivedDetDataHandlerIdx;
 extern int _distributedLocationHandlerIdx;
 extern int _updateHomeRequestHandlerIdx;
 extern int _updateHomeAckHandlerIdx;
@@ -596,7 +462,6 @@ extern int _verifyAckRequestHandlerIdx;
 extern int _verifyAckHandlerIdx;
 extern int _dummyMigrationHandlerIdx;
 
-
 /// Load Balancing
 
 //methods for load balancing
@@ -604,6 +469,7 @@ void startLoadBalancingMlog(void (*fnPtr)(void *),void *_centralLb);
 void finishedCheckpointLoadBalancing();
 void sendMlogLocation(int targetPE,envelope *env);
 void resumeFromSyncRestart(void *data,ChareMlogData *mlogData);
+void restoreParallelRecovery(void (*fnPtr)(void *),void *_centralLb);
 
 //handlers for Load Balancing
 void _receiveMlogLocationHandler(void *buf);
@@ -614,7 +480,6 @@ void _recvGlobalStepHandler(LBStepMsg *msg);
 void _checkpointBarrierHandler(CheckpointBarrierMsg *msg);
 void _checkpointBarrierAckHandler(CheckpointBarrierMsg *msg);
 
-
 //globals used for loadBalancing
 extern int onGoingLoadBalancing;
 extern void *centralLb;
@@ -640,7 +505,7 @@ extern void _initDone(void);
 extern void _resetNodeBocInitVec(void);
 
 //methods for updating location
-void informLocationHome(CkGroupID mgrID,CkArrayIndex idx,int homePE,int currentPE);
+void informLocationHome(CkGroupID mgrID,CkArrayIndexMax idx,int homePE,int currentPE);
 
 //handlers for updating locations
 void _receiveLocationHandler(CurrentLocationMsg *data);
@@ -650,8 +515,5 @@ extern int _receiveLocationHandlerIdx;
 
 
 extern "C" void CmiDeliverRemoteMsgHandlerRange(int lowerHandler,int higherHandler);
-inline void processRemoteMlogMessages(){
-       CmiDeliverRemoteMsgHandlerRange(_ticketRequestHandlerIdx,_receiveLocationHandlerIdx);
-}
 
 #endif
index c47120c3d7ccde9520f18d876d222aa6b9686736..8c01c56a176019118b113ad1d6bebf7174cd62e9 100644 (file)
@@ -192,19 +192,7 @@ CkReductionMgr::CkReductionMgr()//Constructor
   gcount=lcount=0;
   nContrib=nRemote=0;
   maxStartRequest=0;
-#if (defined(_FAULT_MLOG_) && _MLOG_REDUCE_P2P_)
-    if(CkMyPe() != 0){
-        perProcessorCounts = NULL;
-    }else{
-        perProcessorCounts = new int[CmiNumPes()];
-        for(int i=0;i<CmiNumPes();i++){
-            perProcessorCounts[i] = -1;
-        }
-    }
-    totalCount = 0;
-    processorCount = 0;
-#endif
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        numImmigrantRecObjs = 0;
        numEmigrantRecObjs = 0;
 #endif
@@ -223,7 +211,7 @@ CkReductionMgr::CkReductionMgr(CkMigrateMessage *m) :CkGroupInitCallback(m)
   nContrib=nRemote=0;
   maxStartRequest=0;
   DEBR((AA"In reductionMgr migratable constructor at %d \n"AB,this));
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        numImmigrantRecObjs = 0;
        numEmigrantRecObjs = 0;
 #endif
@@ -404,7 +392,7 @@ void CkReductionMgr::contribute(contributorInfo *ci,CkReductionMsg *m)
   m->sourceFlag=-1;//A single contribution
   m->gcount=0;
 
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 
        // if object is an immigrant recovery object, we send the contribution to the source PE
        if(CpvAccess(_currentObj)->mlogData->immigrantRecFlag){
@@ -428,7 +416,7 @@ void CkReductionMgr::contribute(contributorInfo *ci,CkReductionMsg *m)
 #endif
 }
 
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 void CkReductionMgr::contributeViaMessage(CkReductionMsg *m){
        //if(CkMyPe() == 2) CkPrintf("[%d] ---> Contributing Via Message\n",CkMyPe());
        
@@ -454,7 +442,6 @@ void CkReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
  }
  DEBR((AA" Group ReductionStarting called for redNo %d\n"AB,m->num));
  int srcPE = (UsrToEnv(m))->getSrcPe();
-#if (!defined(_FAULT_MLOG_) || !_MLOG_REDUCE_P2P_)
   if (isPresent(m->num) && !inProgress)
   {
     DEBR((AA"Starting reduction #%d at parent's request\n"AB,m->num));
@@ -472,18 +459,6 @@ void CkReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
   else //is Past
     DEBR((AA"Ignoring parent's late request to start #%d\n"AB,m->num));
   delete m;
-#else
-    if(redNo == 0){
-        if(lcount == 0){
-            DEBR(("[%d] Group %d Sending dummy contribute to get totalCount\n",CmiMyPe(),thisgroup.idx));
-            CkReductionMsg *dummy = CkReductionMsg::buildNew(0,NULL);
-            dummy->fromPE = CmiMyPe();
-            dummy->sourceProcessorCount = 0;
-            dummy->redNo = 0;
-            thisProxy[0].contributeViaMessage(dummy);
-        }
-    }
-#endif
 }
 
 //Sent to root of reduction tree with reduction contribution
@@ -548,26 +523,6 @@ void CkReductionMgr::startReduction(int number,int srcPE)
 
   if(disableNotifyChildrenStart) return;
  
-#if (defined(_FAULT_MLOG_) && _MLOG_REDUCE_P2P_) 
-  if(CmiMyPe() == 0 && redNo == 0){
-            for(int j=0;j<CkNumPes();j++){
-                if(j != CkMyPe() && j != srcPE){
-                        thisProxy[j].ReductionStarting(new CkReductionNumberMsg(number));
-                }
-            }
-            if(lcount == 0){
-                CkReductionMsg *dummy = CkReductionMsg::buildNew(0,NULL);
-                dummy->fromPE = CmiMyPe();
-                dummy->sourceProcessorCount = 0;
-                dummy->redNo = 0;
-                thisProxy[0].contributeViaMessage(dummy);
-            }
-    }   else{
-        thisProxy[0].ReductionStarting(new CkReductionNumberMsg(number));
-    }
-
-
-#else
   //Sent start requests to our kids (in case they don't already know)
 #if GROUP_LEVEL_REDUCTION
   for (int k=0;k<treeKids();k++)
@@ -577,7 +532,6 @@ void CkReductionMgr::startReduction(int number,int srcPE)
   }
 #else
   nodeProxy[CkMyNode()].ckLocalBranch()->startNodeGroupReduction(number,thisgroup);
-#endif
 #endif
        
        /*
@@ -668,7 +622,7 @@ void CkReductionMgr::finishReduction(void)
        return;
   }
   //CkPrintf("[%d]finishReduction called for redNo %d with nContrib %d at %.6f\n",CkMyPe(),redNo, nContrib,CmiWallTimer());
-#if (defined(_FAULT_CAUSAL_))
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        if (nContrib<(lcount+adj(redNo).lcount) - numImmigrantRecObjs + numEmigrantRecObjs){
         DEBR((AA"Need more local messages %d %d\n"AB,nContrib,(lcount+adj(redNo).lcount)));
                return;//Need more local messages
@@ -688,14 +642,13 @@ void CkReductionMgr::finishReduction(void)
   DEBR((AA"Reducing data... %d %d\n"AB,nContrib,(lcount+adj(redNo).lcount)));
   CkReductionMsg *result=reduceMessages();
   result->redNo=redNo;
-#if (!defined(_FAULT_MLOG_) || !_MLOG_REDUCE_P2P_)
 
 #if GROUP_LEVEL_REDUCTION
   if (hasParent())
   {//Pass data up tree to parent
     DEBR((AA"Passing reduced data up to parent node %d.\n"AB,treeParent()));
     DEBR((AA"Message gcount is %d+%d+%d.\n"AB,result->gcount,gcount,adj(redNo).gcount));
-#if (defined(_FAULT_CAUSAL_))
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
     result->gcount+=gcount+adj(redNo).gcount;
 #else
     result->gcount+=gcount+adj(redNo).gcount;
@@ -741,29 +694,12 @@ void CkReductionMgr::finishReduction(void)
   CkArrayReductionMgr *nodeMgr=nodeProxy[CkMyNode()].ckLocalBranch();
   nodeMgr->contributeArrayReduction(result);
 #endif
-#else                // _FAULT_MLOG_
-  DEBR(("~~~~~~~~~~~~~~~~~ About to call callback from end of SIMPLIFIED GROUP REDUCTION %d at %.6f\n",redNo,CmiWallTimer()));
-
-    CkSetRefNum(result, result->getUserFlag());
-    if (!result->callback.isInvalid())
-        result->callback.send(result);
-    else if (!storedCallback.isInvalid())
-        storedCallback.send(result);
-    else{
-      DEBR(("No reduction client for group %d \n",thisgroup.idx));
-        CkAbort("No reduction client!\n"
-            "You must register a client with either SetReductionClient or during contribute.\n");
-    }
-
-    DEBR(("[%d,%d]------------END OF SIMPLIFIED GROUP REDUCTION %d for group %d at %.6f\n",CkMyNode(),CkMyPe(),redNo,thisgroup.idx,CkWallTimer()));
-
-#endif               // _FAULT_MLOG_
 
   //House Keeping Operations will have to check later what needs to be changed
   redNo++;
   //Shift the count adjustment vector down one slot (to match new redNo)
   int i;
-#if (!defined(_FAULT_MLOG_) || !_MLOG_REDUCE_P2P_) && !GROUP_LEVEL_REDUCTION
+#if !GROUP_LEVEL_REDUCTION
     /* nodegroup reduction will adjust adjVec in endArrayReduction on PE 0 */
   if(CkMyPe()!=0)
 #endif
@@ -798,13 +734,11 @@ void CkReductionMgr::finishReduction(void)
   }
 #endif
 
-#if (!defined(_FAULT_MLOG_) || !_MLOG_REDUCE_P2P_)
   if(maxStartRequest >= redNo){
          startReduction(redNo,CkMyPe());
          finishReduction();
   }
  
-#endif
 
 }
 
@@ -992,11 +926,9 @@ void CkReductionMgr::pup(PUP::er &p)
   // we can not pup because inserting array elems will add the counters again
 //  p|lcount;
 //  p|gcount;
-#if (defined(_FAULT_MLOG_) && _MLOG_REDUCE_P2P_)
 //  p|lcount;
 //  //  p|gcount;
 //  //  printf("[%d] nodeProxy nodeGroup %d pupped in group %d \n",CkMyPe(),(nodeProxy.ckGetGroupID()).idx,thisgroup.idx);
-#endif
   if(p.isUnpacking()){
     thisProxy = thisgroup;
     maxStartRequest=0;
@@ -1754,6 +1686,9 @@ void CkNodeReductionMgr::contributeWithCounter(contributorInfo *ci,CkReductionMs
 void CkNodeReductionMgr::ReductionStarting(CkReductionNumberMsg *m)
 {
   CmiLock(lockEverything);
+       /*
+               FAULT_EVAC
+       */
   if(blocked){
        delete m;
        CmiUnlock(lockEverything);
@@ -1919,9 +1854,14 @@ void CkNodeReductionMgr::addContribution(CkReductionMsg *m)
 }
 
 void CkNodeReductionMgr::LateMigrantMsg(CkReductionMsg *m){
+        CmiLock(lockEverything);   
+       /*
+               FAULT_EVAC
+       */
        if(blocked){
                DEBR(("[%d] This node is blocked, so local message is being buffered as no %d\n",CkMyNode(),bufferedMsgs.length()));
                bufferedMsgs.enq(m);
+                CmiUnlock(lockEverything);   
                return;
        }
        
@@ -1935,6 +1875,7 @@ void CkNodeReductionMgr::LateMigrantMsg(CkReductionMsg *m){
                msgs.enq(m);
                finishReduction();
        }
+        CmiUnlock(lockEverything);   
 }
 
 
index d4b06851510432f54155bccd1d2494608207b62f..a8e79c30924b92f8fd29f07588e375277d88b72f 100644 (file)
@@ -544,7 +544,7 @@ public:
        */
        int getGCount(){return gcount;};
         static void sanitycheck();
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        void decGCount(){gcount--;}
        void incNumImmigrantRecObjs(){
                numImmigrantRecObjs++;
@@ -563,7 +563,7 @@ public:
 
 private:
 
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        int numImmigrantRecObjs;
        int numEmigrantRecObjs;
 #endif
index 37248a4f26dc08430c49078a77d8a54ab5e94640..c1138ea3a47d2aeb8456ea508037ab38d95e1089 100644 (file)
@@ -26,7 +26,7 @@
 // silly ancient name: for backward compatability only.
 #define PW(x) CkPriobitsToInts(x) 
 
-#if defined(_FAULT_CAUSAL_)
+#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 #define CK_FREE_MSG_MLOG       0x1
 #define CK_BYPASS_DET_MLOG     0x2
 #endif
@@ -186,7 +186,9 @@ public:
     CkObjID sender;
     CkObjID recver;
     MCount SN;
+#if defined(_FAULT_CAUSAL_)
     MCount TN;
+#endif
     int incarnation;
        int flags;
 #endif
@@ -226,7 +228,7 @@ private:</