Adding test cases for fault tolerant Charm++. These tests do not interfere with the...
authorEsteban Meneses <emenese2@illinois.edu>
Mon, 2 Apr 2012 22:38:51 +0000 (17:38 -0500)
committerEsteban Meneses <emenese2@illinois.edu>
Mon, 2 Apr 2012 22:38:51 +0000 (17:38 -0500)
18 files changed:
tests/ampi/Makefile
tests/ampi/jacobi3d/Makefile [new file with mode: 0644]
tests/ampi/jacobi3d/jacobi.C [new file with mode: 0644]
tests/ampi/jacobi3d/kill_01.txt [new file with mode: 0644]
tests/ampi/jacobi3d/kill_02.txt [new file with mode: 0644]
tests/charm++/Makefile
tests/charm++/jacobi3d-sdag/Makefile [new file with mode: 0644]
tests/charm++/jacobi3d-sdag/jacobi3d.C [new file with mode: 0644]
tests/charm++/jacobi3d-sdag/jacobi3d.ci [new file with mode: 0644]
tests/charm++/jacobi3d-sdag/kill_01.txt [new file with mode: 0644]
tests/charm++/jacobi3d-sdag/kill_02.txt [new file with mode: 0644]
tests/charm++/jacobi3d-sdag/kill_03.txt [new file with mode: 0644]
tests/charm++/jacobi3d/Makefile [new file with mode: 0644]
tests/charm++/jacobi3d/jacobi3d.C [new file with mode: 0644]
tests/charm++/jacobi3d/jacobi3d.ci [new file with mode: 0644]
tests/charm++/jacobi3d/kill_01.txt [new file with mode: 0644]
tests/charm++/jacobi3d/kill_02.txt [new file with mode: 0644]
tests/charm++/jacobi3d/kill_03.txt [new file with mode: 0644]

index 0891b5b2c3aa0c3f4851c8e5ff59f4e43d1e99d3..b5cebc3e0752b715eb79ebc5c47b0d5f4dc0ee52 100644 (file)
@@ -16,6 +16,11 @@ bgtest:
                (cd $$d && $(MAKE) bgtest OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)' || exit 1) || exit 1; \
        done
 
+fttest:
+       for d in jacobi3d; do \
+        (cd $$d && $(MAKE) fttest OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)' || exit 1) || exit 1; \
+    done
+
 clean:
        for d in $(DIRS); do (cd $$d; $(MAKE) clean OPTS='$(OPTS)'); done
        rm -f TAGS #*#
diff --git a/tests/ampi/jacobi3d/Makefile b/tests/ampi/jacobi3d/Makefile
new file mode 100644 (file)
index 0000000..01fbdb9
--- /dev/null
@@ -0,0 +1,19 @@
+CHARMC=../../../bin/ampicxx $(OPTS) $(MOPTS)
+
+OBJS = jacobi.o
+
+all: jacobi
+
+jacobi: $(OBJS)
+       $(CHARMC) -o jacobi $(OBJS) -lm -memory isomalloc
+
+jacobi.o: jacobi.C
+       $(CHARMC) -language ampic -swapglobals jacobi.C 
+
+fttest: jacobi
+       ./charmrun ./jacobi 2 2 2 200 +vp8 +p8 +isomalloc_sync +killFile kill_01.txt
+       ./charmrun ./jacobi 4 2 2 200 +vp16 +p8 +isomalloc_sync +killFile kill_02.txt
+
+clean:
+       rm -f *.o jacobi jacobi.o *~ moduleinit.C charmrun conv-host jacobi-cpp jacobi.iso jacobi-get jacobi.tls
+       rm -rf 40 80 120
diff --git a/tests/ampi/jacobi3d/jacobi.C b/tests/ampi/jacobi3d/jacobi.C
new file mode 100644 (file)
index 0000000..9284607
--- /dev/null
@@ -0,0 +1,237 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "mpi.h"
+
+#if CMK_BLUEGENE_CHARM
+extern void BgPrintf(char *);
+#define BGPRINTF(x)  if (thisIndex == 0) BgPrintf(x);
+#else
+#define BGPRINTF(x)
+#endif
+
+#define DIMX 128
+#define DIMY 128
+#define DIMZ 128
+#define CKPT_FREQ 100
+
+int NX, NY, NZ;
+
+class chunk {
+  public:
+    double t[DIMX+2][DIMY+2][DIMZ+2];
+    int xidx, yidx, zidx;
+    int xm, xp, ym, yp, zm, zp;
+    double sbxm[DIMY*DIMZ];
+    double sbxp[DIMY*DIMZ];
+    double sbym[DIMX*DIMZ];
+    double sbyp[DIMX*DIMZ];
+    double sbzm[DIMX*DIMY];
+    double sbzp[DIMX*DIMY];
+    double rbxm[DIMY*DIMZ];
+    double rbxp[DIMY*DIMZ];
+    double rbym[DIMX*DIMZ];
+    double rbyp[DIMX*DIMZ];
+    double rbzm[DIMX*DIMY];
+    double rbzp[DIMX*DIMY];
+};
+
+#ifdef AMPI
+void chunk_pup(pup_er p, void *d)
+{
+  chunk **cpp = (chunk **) d;
+  if(pup_isUnpacking(p))
+    *cpp = new chunk;
+  chunk *cp = *cpp;
+  pup_doubles(p, &cp->t[0][0][0], (DIMX+2)*(DIMY+2)*(DIMZ+2));
+  pup_int(p, &cp->xidx);
+  pup_int(p, &cp->yidx);
+  pup_int(p, &cp->zidx);
+  pup_int(p, &cp->xp);
+  pup_int(p, &cp->xm);
+  pup_int(p, &cp->yp);
+  pup_int(p, &cp->ym);
+  pup_int(p, &cp->zp);
+  pup_int(p, &cp->zm);
+  pup_doubles(p, cp->sbxm, (DIMY*DIMZ));
+  pup_doubles(p, cp->sbxp, (DIMY*DIMZ));
+  pup_doubles(p, cp->rbxm, (DIMY*DIMZ));
+  pup_doubles(p, cp->rbxp, (DIMY*DIMZ));
+  pup_doubles(p, cp->sbym, (DIMX*DIMZ));
+  pup_doubles(p, cp->sbyp, (DIMX*DIMZ));
+  pup_doubles(p, cp->rbym, (DIMX*DIMZ));
+  pup_doubles(p, cp->rbyp, (DIMX*DIMZ));
+  pup_doubles(p, cp->sbzm, (DIMX*DIMY));
+  pup_doubles(p, cp->sbzp, (DIMX*DIMY));
+  pup_doubles(p, cp->rbzm, (DIMX*DIMY));
+  pup_doubles(p, cp->rbzp, (DIMX*DIMY));
+  if(pup_isDeleting(p))
+    delete cp;
+}
+#endif
+
+#define abs(x) ((x)<0.0 ? -(x) : (x))
+
+int index1d(int ix, int iy, int iz)
+{
+  return NY*NZ*ix + NZ*iy + iz;
+}
+
+void index3d(int index, int& ix, int& iy, int& iz)
+{
+  ix = index/(NY*NZ);
+  iy = (index%(NY*NZ))/NZ;
+  iz = index%NZ;
+}
+
+static void copyout(double *d, double t[DIMX+2][DIMY+2][DIMZ+2],
+                    int sx, int ex, int sy, int ey, int sz, int ez)
+{
+  int i, j, k;
+  int l = 0;
+  for(i=sx; i<=ex; i++)
+    for(j=sy; j<=ey; j++)
+      for(k=sz; k<=ez; k++, l++)
+        d[l] = t[i][j][k];
+}
+
+static void copyin(double *d, double t[DIMX+2][DIMY+2][DIMZ+2],
+                    int sx, int ex, int sy, int ey, int sz, int ez)
+{
+  int i, j, k;
+  int l = 0;
+  for(i=sx; i<=ex; i++)
+    for(j=sy; j<=ey; j++)
+      for(k=sz; k<=ez; k++, l++)
+        t[i][j][k] = d[l];
+}
+
+int main(int ac, char** av)
+{
+  int i,j,k,m,cidx;
+  int iter, niter;
+  MPI_Status status;
+  double error, tval, maxerr, tmpmaxerr, starttime, endtime, itertime;
+  chunk *cp;
+  int thisIndex, ierr, nblocks;
+
+  MPI_Init(&ac, &av);
+  MPI_Comm_rank(MPI_COMM_WORLD, &thisIndex);
+  MPI_Comm_size(MPI_COMM_WORLD, &nblocks);
+
+  if (ac < 4) {
+    if (thisIndex == 0)
+      printf("Usage: jacobi X Y Z [nIter].\n");
+    MPI_Finalize();
+  }
+  NX = atoi(av[1]);
+  NY = atoi(av[2]);
+  NZ = atoi(av[3]);
+  if (NX*NY*NZ != nblocks) {
+    if (thisIndex == 0) 
+      printf("%d x %d x %d != %d\n", NX,NY,NZ, nblocks);
+    MPI_Finalize();
+  }
+  if (ac == 5)
+    niter = atoi(av[4]);
+  else
+    niter = 20;
+
+  MPI_Bcast(&niter, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+#if CMK_AIX
+  cp = (chunk*)malloc(sizeof(chunk));
+#else
+  cp = new chunk;
+#endif
+#if defined(AMPI) && ! defined(NO_PUP)
+  MPI_Register((void*)&cp, (MPI_PupFn) chunk_pup);
+#endif
+
+  index3d(thisIndex, cp->xidx, cp->yidx, cp->zidx);
+  cp->xp = index1d((cp->xidx+1)%NX,cp->yidx,cp->zidx);
+  cp->xm = index1d((cp->xidx+NX-1)%NX,cp->yidx,cp->zidx);
+  cp->yp = index1d(cp->xidx,(cp->yidx+1)%NY,cp->zidx);
+  cp->ym = index1d(cp->xidx,(cp->yidx+NY-1)%NY,cp->zidx);
+  cp->zp = index1d(cp->xidx,cp->yidx,(cp->zidx+1)%NZ);
+  cp->zm = index1d(cp->xidx,cp->yidx,(cp->zidx+NZ-1)%NZ);
+  for(i=1; i<=DIMZ; i++)
+    for(j=1; j<=DIMY; j++)
+      for(k=1; k<=DIMX; k++)
+        cp->t[k][j][i] = DIMY*DIMX*(i-1) + DIMX*(j-2) + (k-1);
+
+  MPI_Barrier(MPI_COMM_WORLD);
+  starttime = MPI_Wtime();
+
+  maxerr = 0.0;
+  for(iter=1; iter<=niter; iter++) {
+    BGPRINTF("interation starts at %f\n");
+    maxerr = 0.0;
+    copyout(cp->sbxm, cp->t, 1, 1, 1, DIMY, 1, DIMZ);
+    copyout(cp->sbxp, cp->t, DIMX, DIMX, 1, DIMY, 1, DIMZ);
+    copyout(cp->sbym, cp->t, 1, DIMX, 1, 1, 1, DIMZ);
+    copyout(cp->sbyp, cp->t, 1, DIMX, DIMY, DIMY, 1, DIMZ);
+    copyout(cp->sbzm, cp->t, 1, DIMX, 1, DIMY, 1, 1);
+    copyout(cp->sbzp, cp->t, 1, DIMX, 1, DIMY, DIMZ, DIMZ);
+
+    MPI_Request rreq[6];
+    MPI_Status rsts[6];
+
+    MPI_Irecv(cp->rbxp, DIMY*DIMZ, MPI_DOUBLE, cp->xp, 0, MPI_COMM_WORLD, &rreq[0]);
+    MPI_Irecv(cp->rbxm, DIMY*DIMZ, MPI_DOUBLE, cp->xm, 1, MPI_COMM_WORLD, &rreq[1]);
+    MPI_Irecv(cp->rbyp, DIMX*DIMZ, MPI_DOUBLE, cp->yp, 2, MPI_COMM_WORLD, &rreq[2]);
+    MPI_Irecv(cp->rbym, DIMX*DIMZ, MPI_DOUBLE, cp->ym, 3, MPI_COMM_WORLD, &rreq[3]);
+    MPI_Irecv(cp->rbzm, DIMX*DIMY, MPI_DOUBLE, cp->zm, 5, MPI_COMM_WORLD, &rreq[4]);
+    MPI_Irecv(cp->rbzp, DIMX*DIMY, MPI_DOUBLE, cp->zp, 4, MPI_COMM_WORLD, &rreq[5]);
+
+    MPI_Send(cp->sbxm, DIMY*DIMZ, MPI_DOUBLE, cp->xm, 0, MPI_COMM_WORLD);
+    MPI_Send(cp->sbxp, DIMY*DIMZ, MPI_DOUBLE, cp->xp, 1, MPI_COMM_WORLD);
+    MPI_Send(cp->sbym, DIMX*DIMZ, MPI_DOUBLE, cp->ym, 2, MPI_COMM_WORLD);
+    MPI_Send(cp->sbyp, DIMX*DIMZ, MPI_DOUBLE, cp->yp, 3, MPI_COMM_WORLD);
+    MPI_Send(cp->sbzm, DIMX*DIMY, MPI_DOUBLE, cp->zm, 4, MPI_COMM_WORLD);
+    MPI_Send(cp->sbzp, DIMX*DIMY, MPI_DOUBLE, cp->zp, 5, MPI_COMM_WORLD);
+
+    MPI_Waitall(6, rreq, rsts);
+
+    copyin(cp->sbxm, cp->t, 0, 0, 1, DIMY, 1, DIMZ);
+    copyin(cp->sbxp, cp->t, DIMX+1, DIMX+1, 1, DIMY, 1, DIMZ);
+    copyin(cp->sbym, cp->t, 1, DIMX, 0, 0, 1, DIMZ);
+    copyin(cp->sbyp, cp->t, 1, DIMX, DIMY+1, DIMY+1, 1, DIMZ);
+    copyin(cp->sbzm, cp->t, 1, DIMX, 1, DIMY, 0, 0);
+    copyin(cp->sbzp, cp->t, 1, DIMX, 1, DIMY, DIMZ+1, DIMZ+1);
+    if(iter > 25 &&  iter < 85 && thisIndex == 35)
+      m = 9;
+    else
+      m = 1;
+    for(; m>0; m--)
+      for(i=1; i<=DIMZ; i++)
+        for(j=1; j<=DIMY; j++)
+          for(k=1; k<=DIMX; k++) {
+            tval = (cp->t[k][j][i] + cp->t[k][j][i+1] +
+                 cp->t[k][j][i-1] + cp->t[k][j+1][i]+ 
+                 cp->t[k][j-1][i] + cp->t[k+1][j][i] + cp->t[k-1][j][i])/7.0;
+            error = abs(tval-cp->t[k][j][i]);
+            cp->t[k][j][i] = tval;
+            if (error > maxerr) maxerr = error;
+          }
+    MPI_Allreduce(&maxerr, &tmpmaxerr, 1, MPI_DOUBLE, MPI_MAX, 
+                   MPI_COMM_WORLD);
+    maxerr = tmpmaxerr;
+    endtime = MPI_Wtime();
+    itertime = endtime - starttime;
+    double  it;
+    MPI_Allreduce(&itertime, &it, 1, MPI_DOUBLE, MPI_SUM,
+                   MPI_COMM_WORLD);
+    itertime = it/nblocks;
+    if (thisIndex == 0)
+      printf("iter %d elapsed time: %f time: %lf maxerr: %lf\n", iter, MPI_Wtime(), itertime, maxerr);
+    starttime = MPI_Wtime();
+#ifdef AMPI
+    if(iter%CKPT_FREQ == 50) {
+      AMPI_MemCheckpoint();
+      //MPI_Migrate();
+    }
+#endif
+  }
+  MPI_Finalize();
+  return 0;
+}
diff --git a/tests/ampi/jacobi3d/kill_01.txt b/tests/ampi/jacobi3d/kill_01.txt
new file mode 100644 (file)
index 0000000..fa80727
--- /dev/null
@@ -0,0 +1 @@
+3 40
diff --git a/tests/ampi/jacobi3d/kill_02.txt b/tests/ampi/jacobi3d/kill_02.txt
new file mode 100644 (file)
index 0000000..497174c
--- /dev/null
@@ -0,0 +1 @@
+3 70
index b15338af6ec1b203e25822643ecc20ef48bd50bc..451346bf31759034a7a2d4d0005315b5b847e40b 100644 (file)
@@ -16,6 +16,12 @@ bgtest:
                (cd $$d; $(MAKE) bgtest OPTS='$(OPTS)' || exit 1) || exit 1; \
        done
 
+fttest:
+       for d in jacobi3d jacobi3d-sdag; do \
+               (cd $$d; $(MAKE) fttest OPTS='$(OPTS)' || exit 1) || exit 1; \
+       done
+
+
 clean:
        for d in $(DIRS); do (cd $$d; $(MAKE) clean OPTS='$(OPTS)'); done
        rm -f TAGS #*#
diff --git a/tests/charm++/jacobi3d-sdag/Makefile b/tests/charm++/jacobi3d-sdag/Makefile
new file mode 100644 (file)
index 0000000..85e22f3
--- /dev/null
@@ -0,0 +1,31 @@
+CHARMC=../../../bin/charmc $(OPTS) $(MOPTS)
+
+OBJS = jacobi3d.o
+
+all: jacobi3d
+
+jacobi3d: $(OBJS)
+       $(CHARMC) -language charm++ -module DummyLB -o jacobi3d $(OBJS)
+#      $(CHARMC) -language charm++ -module EveryLB -memory paranoid -o jacobi3d $(OBJS)
+
+projections: $(OBJS)
+       $(CHARMC) -language charm++ -tracemode projections -lz -o jacobi3d.prj $(OBJS)
+
+summary: $(OBJS)
+       $(CHARMC) -language charm++ -tracemode summary -lz -o jacobi3d.sum $(OBJS)
+
+jacobi3d.decl.h: jacobi3d.ci
+       $(CHARMC)  jacobi3d.ci
+
+fttest: jacobi3d
+       ./charmrun ./jacobi3d 256 128 +p8
+       ./charmrun ./jacobi3d 256 128 +p8 +killFile kill_01.txt
+       ./charmrun ./jacobi3d 256 256 256 64 64 32 +p8 +killFile kill_02.txt
+       ./charmrun ./jacobi3d 256 256 256 64 64 32 +p8 +killFile kill_03.txt
+
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.o jacobi3d jacobi3d.prj charmrun *~
+
+jacobi3d.o: jacobi3d.C jacobi3d.decl.h
+       $(CHARMC) -c jacobi3d.C
diff --git a/tests/charm++/jacobi3d-sdag/jacobi3d.C b/tests/charm++/jacobi3d-sdag/jacobi3d.C
new file mode 100644 (file)
index 0000000..31f1d8f
--- /dev/null
@@ -0,0 +1,545 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+/** \file jacobi3d.C
+ *  Author: Abhinav S Bhatele
+ *  Date Created: June 01st, 2009
+ *
+ *  This does a topological placement for a 3d jacobi.
+ *
+ *     
+ *           *****************
+ *        *               *  *
+ *   ^ *****************     *
+ *   | *               *     *
+ *   | *               *     *
+ *   | *               *     *
+ *   Y *               *     *
+ *   | *               *     *
+ *   | *               *     *
+ *   | *               *  * 
+ *   ~ *****************    Z
+ *     <------ X ------> 
+ *
+ *   X: left, right --> wrap_x
+ *   Y: top, bottom --> wrap_y
+ *   Z: front, back --> wrap_z
+ */
+
+#include "jacobi3d.decl.h"
+#include "TopoManager.h"
+
+// See README for documentation
+
+/*readonly*/ CProxy_Main mainProxy;
+/*readonly*/ int arrayDimX;
+/*readonly*/ int arrayDimY;
+/*readonly*/ int arrayDimZ;
+/*readonly*/ int blockDimX;
+/*readonly*/ int blockDimY;
+/*readonly*/ int blockDimZ;
+
+// specify the number of worker chares in each dimension
+/*readonly*/ int num_chare_x;
+/*readonly*/ int num_chare_y;
+/*readonly*/ int num_chare_z;
+
+/*readonly*/ int globalBarrier;
+
+static unsigned long next = 1;
+
+int myrand(int numpes) {
+  next = next * 1103515245 + 12345;
+  return((unsigned)(next/65536) % numpes);
+}
+
+// We want to wrap entries around, and because mod operator % 
+// sometimes misbehaves on negative values. -1 maps to the highest value.
+#define wrap_x(a)      (((a)+num_chare_x)%num_chare_x)
+#define wrap_y(a)      (((a)+num_chare_y)%num_chare_y)
+#define wrap_z(a)      (((a)+num_chare_z)%num_chare_z)
+
+#define index(a, b, c) ( (a)*(blockDimY+2)*(blockDimZ+2) + (b)*(blockDimZ+2) + (c) )
+
+#define PRINT_FREQ      10
+#define CKP_FREQ               200
+#define MAX_ITER               1000
+#define WARM_ITER              5
+#define LEFT                   1
+#define RIGHT                  2
+#define TOP                    3
+#define BOTTOM                 4
+#define FRONT                  5
+#define BACK                   6
+#define DIVIDEBY7              0.14285714285714285714
+
+double startTime;
+double endTime;
+
+/** \class ghostMsg
+ *
+ */
+class ghostMsg: public CMessage_ghostMsg {
+  public:
+    int dir;
+    int height;
+    int width;
+    double* gh;
+
+    ghostMsg(int _d, int _h, int _w) : dir(_d), height(_h), width(_w) {
+    }
+};
+
+/** \class Main
+ *
+ */
+class Main : public CBase_Main {
+  public:
+    CProxy_Jacobi array;
+    int iterations;
+
+    Main(CkArgMsg* m) {
+      if ( (m->argc != 3) && (m->argc != 7) ) {
+        CkPrintf("%s [array_size] [block_size]\n", m->argv[0]);
+        CkPrintf("OR %s [array_size_X] [array_size_Y] [array_size_Z] [block_size_X] [block_size_Y] [block_size_Z]\n", m->argv[0]);
+        CkAbort("Abort");
+      }
+
+      // set iteration counter to zero
+      iterations = 0;
+
+      // store the main proxy
+      mainProxy = thisProxy;
+       
+      if(m->argc == 3) {
+       arrayDimX = arrayDimY = arrayDimZ = atoi(m->argv[1]);
+        blockDimX = blockDimY = blockDimZ = atoi(m->argv[2]); 
+      }
+      else if (m->argc == 7) {
+        arrayDimX = atoi(m->argv[1]);
+       arrayDimY = atoi(m->argv[2]);
+       arrayDimZ = atoi(m->argv[3]);
+        blockDimX = atoi(m->argv[4]); 
+       blockDimY = atoi(m->argv[5]); 
+       blockDimZ = atoi(m->argv[6]);
+      }
+
+      if (arrayDimX < blockDimX || arrayDimX % blockDimX != 0)
+        CkAbort("array_size_X % block_size_X != 0!");
+      if (arrayDimY < blockDimY || arrayDimY % blockDimY != 0)
+        CkAbort("array_size_Y % block_size_Y != 0!");
+      if (arrayDimZ < blockDimZ || arrayDimZ % blockDimZ != 0)
+        CkAbort("array_size_Z % block_size_Z != 0!");
+
+      num_chare_x = arrayDimX / blockDimX;
+      num_chare_y = arrayDimY / blockDimY;
+      num_chare_z = arrayDimZ / blockDimZ;
+
+      // print info
+      CkPrintf("\nSTENCIL COMPUTATION WITH NO BARRIERS\n");
+      CkPrintf("Running Jacobi on %d processors with (%d, %d, %d) chares\n", CkNumPes(), num_chare_x, num_chare_y, num_chare_z);
+      CkPrintf("Array Dimensions: %d %d %d\n", arrayDimX, arrayDimY, arrayDimZ);
+      CkPrintf("Block Dimensions: %d %d %d\n", blockDimX, blockDimY, blockDimZ);
+
+      // Create new array of worker chares
+#if USE_TOPOMAP
+      CProxy_JacobiMap map = CProxy_JacobiMap::ckNew(num_chare_x, num_chare_y, num_chare_z);
+      CkPrintf("Topology Mapping is being done ... \n");
+      CkArrayOptions opts(num_chare_x, num_chare_y, num_chare_z);
+      opts.setMap(map);
+      array = CProxy_Jacobi::ckNew(opts);
+#else
+      array = CProxy_Jacobi::ckNew(num_chare_x, num_chare_y, num_chare_z);
+#endif
+
+      TopoManager tmgr;
+      CkArray *jarr = array.ckLocalBranch();
+      int jmap[num_chare_x][num_chare_y][num_chare_z];
+
+      int hops=0, p;
+      for(int i=0; i<num_chare_x; i++)
+       for(int j=0; j<num_chare_y; j++)
+         for(int k=0; k<num_chare_z; k++) {
+           jmap[i][j][k] = jarr->procNum(CkArrayIndex3D(i, j, k));
+         }
+
+      for(int i=0; i<num_chare_x; i++)
+       for(int j=0; j<num_chare_y; j++)
+         for(int k=0; k<num_chare_z; k++) {
+           p = jmap[i][j][k];
+           hops += tmgr.getHopsBetweenRanks(p, jmap[wrap_x(i+1)][j][k]);
+           hops += tmgr.getHopsBetweenRanks(p, jmap[wrap_x(i-1)][j][k]);
+           hops += tmgr.getHopsBetweenRanks(p, jmap[i][wrap_y(j+1)][k]);
+           hops += tmgr.getHopsBetweenRanks(p, jmap[i][wrap_y(j-1)][k]);
+           hops += tmgr.getHopsBetweenRanks(p, jmap[i][j][wrap_z(k+1)]);
+           hops += tmgr.getHopsBetweenRanks(p, jmap[i][j][wrap_z(k-1)]);
+         }
+      CkPrintf("Total Hops: %d\n", hops);
+
+               startTime = CmiWallTimer();
+       
+               //Start the computation
+               array.doStep();
+       }
+
+    // Each worker reports back to here when it completes an iteration
+       void report() {
+               iterations += CKP_FREQ;
+               if (iterations < MAX_ITER) {
+                       CkCallback cb (CkIndex_Jacobi::doStep(), array);
+                       CkStartMemCheckpoint(cb);
+               } else {
+                       CkPrintf("Completed %d iterations\n", MAX_ITER-1);
+                       endTime = CmiWallTimer();
+                       CkPrintf("Time elapsed per iteration: %f\n", (endTime - startTime)/(MAX_ITER-1-WARM_ITER));
+                       CkExit();
+               }
+       }
+
+};
+
+/** \class Jacobi
+ *
+ */
+
+class Jacobi: public CBase_Jacobi {
+  Jacobi_SDAG_CODE
+
+  public:
+    int iterations;
+    int imsg;
+
+    double *temperature;
+    double *new_temperature;
+       double timing,average;
+
+    // Constructor, initialize values
+    Jacobi() {
+      __sdag_init();
+
+      int i, j, k;
+      // allocate a three dimensional array
+      temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+      new_temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+
+      for(i=0; i<blockDimX+2; ++i) {
+       for(j=0; j<blockDimY+2; ++j) {
+         for(k=0; k<blockDimZ+2; ++k) {
+           temperature[index(i, j, k)] = 0.0;
+         }
+       } 
+      }
+
+      iterations = 0;
+      imsg = 0;
+      constrainBC();
+       
+               usesAtSync = CmiTrue;
+
+    }
+
+    Jacobi(CkMigrateMessage* m): CBase_Jacobi(m) {
+               __sdag_init();
+       }
+
+    ~Jacobi() { 
+      delete [] temperature; 
+      delete [] new_temperature; 
+    }
+
+       // Pupping function for migration and fault tolerance
+       // Condition: assuming the 3D Chare Arrays are NOT used
+       void pup(PUP::er &p){
+               
+               // calling parent's pup
+               CBase_Jacobi::pup(p);
+       
+               __sdag_pup(p);
+       
+               // pupping properties of this class
+               p | iterations;
+               p | imsg;
+
+               // if unpacking, allocate the memory space
+               if(p.isUnpacking()){
+                       temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+                       new_temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+       
+               }
+
+               // pupping the arrays
+               p((char *)temperature, (blockDimX+2) * (blockDimY+2) * (blockDimZ+2) * sizeof(double));
+               //p((char *) new_temperature, (blockDimX+2) * (blockDimY+2) * (blockDimZ+2) * sizeof(double));
+
+       }
+
+
+    // Send ghost faces to the six neighbors
+    void begin_iteration(void) {
+      if (thisIndex.x == 0 && thisIndex.y == 0 && thisIndex.z == 0) {
+//          CkPrintf("Start of iteration %d\n", iterations);
+                       if(iterations % PRINT_FREQ == 0){
+                               average = timing;
+                               timing = CmiWallTimer();
+                               average = (timing - average)/(double)PRINT_FREQ;
+                               CkPrintf("time=%.2f it=%d avg=%.4f\n",timing,iterations,average);
+                       }
+      }
+      iterations++;
+
+      // Copy different faces into messages
+      ghostMsg *leftMsg = new (blockDimY*blockDimZ) ghostMsg(RIGHT, blockDimY, blockDimZ);
+      ghostMsg *rightMsg = new (blockDimY*blockDimZ) ghostMsg(LEFT, blockDimY, blockDimZ);
+      ghostMsg *topMsg = new (blockDimX*blockDimZ) ghostMsg(BOTTOM, blockDimX, blockDimZ);
+      ghostMsg *bottomMsg = new (blockDimX*blockDimZ) ghostMsg(TOP, blockDimX, blockDimZ);
+      ghostMsg *frontMsg = new (blockDimX*blockDimY) ghostMsg(BACK, blockDimX, blockDimY);
+      ghostMsg *backMsg = new (blockDimX*blockDimY) ghostMsg(FRONT, blockDimX, blockDimY);
+
+      CkSetRefNum(leftMsg, iterations);
+      CkSetRefNum(rightMsg, iterations);
+      CkSetRefNum(topMsg, iterations);
+      CkSetRefNum(bottomMsg, iterations);
+      CkSetRefNum(frontMsg, iterations);
+      CkSetRefNum(backMsg, iterations);
+
+      for(int j=0; j<blockDimY; ++j) 
+       for(int k=0; k<blockDimZ; ++k) {
+         leftMsg->gh[k*blockDimY+j] = temperature[index(1, j+1, k+1)];
+         rightMsg->gh[k*blockDimY+j] = temperature[index(blockDimX, j+1, k+1)];
+      }
+
+      for(int i=0; i<blockDimX; ++i) 
+       for(int k=0; k<blockDimZ; ++k) {
+         topMsg->gh[k*blockDimX+i] = temperature[index(i+1, 1, k+1)];
+         bottomMsg->gh[k*blockDimX+i] = temperature[index(i+1, blockDimY, k+1)];
+      }
+
+      for(int i=0; i<blockDimX; ++i) 
+       for(int j=0; j<blockDimY; ++j) {
+         frontMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, 1)];
+         backMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, blockDimZ)];
+      }
+
+      // Send my left face
+      thisProxy(wrap_x(thisIndex.x-1), thisIndex.y, thisIndex.z).receiveGhosts(leftMsg);
+      // Send my right face
+      thisProxy(wrap_x(thisIndex.x+1), thisIndex.y, thisIndex.z).receiveGhosts(rightMsg);
+      // Send my top face
+      thisProxy(thisIndex.x, wrap_y(thisIndex.y-1), thisIndex.z).receiveGhosts(topMsg);
+      // Send my bottom face
+      thisProxy(thisIndex.x, wrap_y(thisIndex.y+1), thisIndex.z).receiveGhosts(bottomMsg);
+      // Send my front face
+      thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z-1)).receiveGhosts(frontMsg);
+      // Send my back face
+      thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z+1)).receiveGhosts(backMsg);
+    }
+
+    void processGhosts(ghostMsg *gmsg) {
+      int height = gmsg->height;
+      int width = gmsg->width;
+
+      switch(gmsg->dir) {
+       case LEFT:
+         for(int j=0; j<height; ++j) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(0, j+1, k+1)] = gmsg->gh[k*height+j];
+         }
+         break;
+       case RIGHT:
+         for(int j=0; j<height; ++j) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(blockDimX+1, j+1, k+1)] = gmsg->gh[k*height+j];
+         }
+         break;
+       case TOP:
+         for(int i=0; i<height; ++i) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(i+1, 0, k+1)] = gmsg->gh[k*height+i];
+         }
+         break;
+       case BOTTOM:
+         for(int i=0; i<height; ++i) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(i+1, blockDimY+1, k+1)] = gmsg->gh[k*height+i];
+         }
+         break;
+       case FRONT:
+         for(int i=0; i<height; ++i) 
+           for(int j=0; j<width; ++j) {
+             temperature[index(i+1, j+1, blockDimZ+1)] = gmsg->gh[j*height+i];
+         }
+         break;
+       case BACK:
+         for(int i=0; i<height; ++i) 
+           for(int j=0; j<width; ++j) {
+             temperature[index(i+1, j+1, 0)] = gmsg->gh[j*height+i];
+         }
+         break;
+       default:
+          CkAbort("ERROR\n");
+      }
+
+      delete gmsg;
+    }
+
+
+       void check_and_compute() {
+               compute_kernel();
+
+               // calculate error
+               // not being done right now since we are doing a fixed no. of iterations
+
+      double *tmp;
+               tmp = temperature;
+               temperature = new_temperature;
+               new_temperature = tmp;
+
+               constrainBC();
+
+               if (iterations % CKP_FREQ == 0 || iterations > MAX_ITER){
+                       contribute(0, 0, CkReduction::concat, CkCallback(CkIndex_Main::report(), mainProxy));
+               } else {
+                       doStep();
+               }
+       }
+
+       void ResumeFromSync(){
+               doStep();
+       }
+
+
+    // Check to see if we have received all neighbor values yet
+    // If all neighbor values have been received, we update our values and proceed
+    void compute_kernel() {
+#pragma unroll    
+      for(int i=1; i<blockDimX+1; ++i) {
+       for(int j=1; j<blockDimY+1; ++j) {
+         for(int k=1; k<blockDimZ+1; ++k) {
+           // update my value based on the surrounding values
+           new_temperature[index(i, j, k)] = (temperature[index(i-1, j, k)] 
+                                           +  temperature[index(i+1, j, k)]
+                                           +  temperature[index(i, j-1, k)]
+                                           +  temperature[index(i, j+1, k)]
+                                           +  temperature[index(i, j, k-1)]
+                                           +  temperature[index(i, j, k+1)]
+                                           +  temperature[index(i, j, k)] ) * DIVIDEBY7;
+         }
+       }
+      }
+    }
+
+    // Enforce some boundary conditions
+    void constrainBC() {
+      // Heat left, top and front faces of each chare's block
+      for(int i=1; i<blockDimX+1; ++i)
+       for(int k=1; k<blockDimZ+1; ++k)
+         temperature[index(i, 1, k)] = 255.0;
+      for(int j=1; j<blockDimY+1; ++j)
+       for(int k=1; k<blockDimZ+1; ++k)
+         temperature[index(1, j, k)] = 255.0;
+      for(int i=1; i<blockDimX+1; ++i)
+       for(int j=1; j<blockDimY+1; ++j)
+         temperature[index(i, j, 1)] = 255.0;
+    }
+
+};
+
+/** \class JacobiMap
+ *
+ */
+
+class JacobiMap : public CkArrayMap {
+  public:
+    int X, Y, Z;
+    int *mapping;
+
+    JacobiMap(int x, int y, int z) {
+      X = x; Y = y; Z = z;
+      mapping = new int[X*Y*Z];
+
+      // we are assuming that the no. of chares in each dimension is a 
+      // multiple of the torus dimension
+
+      TopoManager tmgr;
+      int dimNX, dimNY, dimNZ, dimNT;
+
+      dimNX = tmgr.getDimNX();
+      dimNY = tmgr.getDimNY();
+      dimNZ = tmgr.getDimNZ();
+      dimNT = tmgr.getDimNT();
+
+      // we are assuming that the no. of chares in each dimension is a 
+      // multiple of the torus dimension
+      int numCharesPerPe = X*Y*Z/CkNumPes();
+
+      int numCharesPerPeX = X / dimNX;
+      int numCharesPerPeY = Y / dimNY;
+      int numCharesPerPeZ = Z / dimNZ;
+      int pe = 0, pes = CkNumPes();
+
+#if USE_BLOCK_RNDMAP
+      int used[pes];
+      for(int i=0; i<pes; i++)
+       used[i] = 0;
+#endif
+
+      if(dimNT < 2) {  // one core per node
+       if(CkMyPe()==0) CkPrintf("%d %d %d %d : %d %d %d \n", dimNX, dimNY, dimNZ, dimNT, numCharesPerPeX, numCharesPerPeY, numCharesPerPeZ); 
+       for(int i=0; i<dimNX; i++)
+         for(int j=0; j<dimNY; j++)
+           for(int k=0; k<dimNZ; k++)
+           {
+#if USE_BLOCK_RNDMAP
+             pe = myrand(pes); 
+             while(used[pe]!=0) {
+               pe = myrand(pes); 
+             }
+             used[pe] = 1;
+#endif
+
+             for(int ci=i*numCharesPerPeX; ci<(i+1)*numCharesPerPeX; ci++)
+               for(int cj=j*numCharesPerPeY; cj<(j+1)*numCharesPerPeY; cj++)
+                 for(int ck=k*numCharesPerPeZ; ck<(k+1)*numCharesPerPeZ; ck++) {
+#if USE_TOPOMAP
+                   mapping[ci*Y*Z + cj*Z + ck] = tmgr.coordinatesToRank(i, j, k);
+#elif USE_BLOCK_RNDMAP
+                   mapping[ci*Y*Z + cj*Z + ck] = pe;
+#endif
+                 }
+           }
+      } else {         // multiple cores per node
+       // In this case, we split the chares in the X dimension among the
+       // cores on the same node. The strange thing I figured out is that
+       // doing this in the Z dimension is not as good.
+       numCharesPerPeX /= dimNT;
+       if(CkMyPe()==0) CkPrintf("%d %d %d %d : %d %d %d \n", dimNX, dimNY, dimNZ, dimNT, numCharesPerPeX, numCharesPerPeY, numCharesPerPeZ);
+
+       for(int i=0; i<dimNX; i++)
+         for(int j=0; j<dimNY; j++)
+           for(int k=0; k<dimNZ; k++)
+             for(int l=0; l<dimNT; l++)
+               for(int ci=(dimNT*i+l)*numCharesPerPeX; ci<(dimNT*i+l+1)*numCharesPerPeX; ci++)
+                 for(int cj=j*numCharesPerPeY; cj<(j+1)*numCharesPerPeY; cj++)
+                   for(int ck=k*numCharesPerPeZ; ck<(k+1)*numCharesPerPeZ; ck++) {
+                     mapping[ci*Y*Z + cj*Z + ck] = tmgr.coordinatesToRank(i, j, k, l);
+                   }
+      } // end of if
+
+      if(CkMyPe() == 0) CkPrintf("Map generated ... \n");
+    }
+
+    ~JacobiMap() { 
+      delete [] mapping;
+    }
+
+    int procNum(int, const CkArrayIndex &idx) {
+      int *index = (int *)idx.data();
+      return mapping[index[0]*Y*Z + index[1]*Z + index[2]]; 
+    }
+};
+
+#include "jacobi3d.def.h"
diff --git a/tests/charm++/jacobi3d-sdag/jacobi3d.ci b/tests/charm++/jacobi3d-sdag/jacobi3d.ci
new file mode 100644 (file)
index 0000000..5ee6132
--- /dev/null
@@ -0,0 +1,52 @@
+mainmodule jacobi3d {
+
+  readonly CProxy_Main mainProxy;
+  readonly int arrayDimX;
+  readonly int arrayDimY;
+  readonly int arrayDimZ;
+  readonly int blockDimX;
+  readonly int blockDimY;
+  readonly int blockDimZ;
+
+  readonly int num_chare_x;
+  readonly int num_chare_y;
+  readonly int num_chare_z;
+
+  readonly int globalBarrier;
+
+  message ghostMsg {
+    double gh[];
+  };
+
+  mainchare Main {
+    entry Main(CkArgMsg *m);
+    entry void report();
+  };
+
+  array [3D] Jacobi {
+    // Normal Charm++ entry methods
+    entry Jacobi(void);
+    entry void begin_iteration(void);
+    entry void receiveGhosts(ghostMsg *gmsg);
+    entry void processGhosts(ghostMsg *gmsg);
+
+    entry void doStep() {
+      atomic "begin_iteration" {
+       begin_iteration();
+      }
+      for(imsg = 0; imsg < 6; imsg++) {
+       // "iterations" keeps track of messages across steps
+       when receiveGhosts[iterations] (ghostMsg *gmsg)
+         atomic "process ghosts" { processGhosts(gmsg); }
+      }
+      atomic "doWork" {
+       check_and_compute();
+      }
+    };
+  };
+
+  group JacobiMap : CkArrayMap {
+    entry JacobiMap(int x, int y, int z);
+  };
+
+};
diff --git a/tests/charm++/jacobi3d-sdag/kill_01.txt b/tests/charm++/jacobi3d-sdag/kill_01.txt
new file mode 100644 (file)
index 0000000..44e9006
--- /dev/null
@@ -0,0 +1 @@
+3 50
diff --git a/tests/charm++/jacobi3d-sdag/kill_02.txt b/tests/charm++/jacobi3d-sdag/kill_02.txt
new file mode 100644 (file)
index 0000000..abecdc0
--- /dev/null
@@ -0,0 +1 @@
+3 33
diff --git a/tests/charm++/jacobi3d-sdag/kill_03.txt b/tests/charm++/jacobi3d-sdag/kill_03.txt
new file mode 100644 (file)
index 0000000..18a7a02
--- /dev/null
@@ -0,0 +1,2 @@
+3 33
+6 95
diff --git a/tests/charm++/jacobi3d/Makefile b/tests/charm++/jacobi3d/Makefile
new file mode 100644 (file)
index 0000000..508d0b3
--- /dev/null
@@ -0,0 +1,29 @@
+CHARMC=../../../bin/charmc $(OPTS) $(MOPTS)
+
+OBJS = jacobi3d.o
+
+all: jacobi3d
+
+jacobi3d: $(OBJS)
+       $(CHARMC) -language charm++ -o jacobi3d $(OBJS)
+
+projections: $(OBJS)
+       $(CHARMC) -language charm++ -tracemode projections -lz -o jacobi3d.prj $(OBJS)
+
+summary: $(OBJS)
+       $(CHARMC) -language charm++ -tracemode summary -lz -o jacobi3d.sum $(OBJS)
+
+jacobi3d.decl.h: jacobi3d.ci
+       $(CHARMC)  jacobi3d.ci
+
+fttest: jacobi3d
+       ./charmrun ./jacobi3d 256 128 +p8
+       ./charmrun ./jacobi3d 256 128 +p8 +killFile kill_01.txt
+       ./charmrun ./jacobi3d 256 256 256 64 64 32 +p8 +killFile kill_02.txt
+       ./charmrun ./jacobi3d 256 256 256 64 64 32 +p8 +killFile kill_03.txt
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.o jacobi3d jacobi3d.prj charmrun *~
+
+jacobi3d.o: jacobi3d.C jacobi3d.decl.h
+       $(CHARMC) -c jacobi3d.C
diff --git a/tests/charm++/jacobi3d/jacobi3d.C b/tests/charm++/jacobi3d/jacobi3d.C
new file mode 100644 (file)
index 0000000..735bed7
--- /dev/null
@@ -0,0 +1,448 @@
+/*****************************************************************************
+ * $Source: /cvsroot/charm/examples/bigsim/sdag/jacobi3d/jacobi3d.C,v $
+ * $Author: bhatele $
+ * $Date: 2009-06-04 21:50:17 $
+ * $Revision: 1.5 $
+ *****************************************************************************/
+
+/** \file jacobi3d.C
+ *  Author: Abhinav S Bhatele
+ *  Date Created: June 01st, 2009
+ *
+ *  This does a topological placement for a 3d jacobi.
+ *
+ *     
+ *           *****************
+ *        *               *  *
+ *   ^ *****************     *
+ *   | *               *     *
+ *   | *               *     *
+ *   | *               *     *
+ *   Y *               *     *
+ *   | *               *     *
+ *   | *               *     *
+ *   | *               *  * 
+ *   ~ *****************    Z
+ *     <------ X ------> 
+ *
+ *   X: left, right --> wrap_x
+ *   Y: top, bottom --> wrap_y
+ *   Z: front, back --> wrap_z
+ */
+
+#include "jacobi3d.decl.h"
+#include <vector>
+#include <utility>
+
+#define CKP_FREQ 200
+#define MAX_ITER               1000
+#define PRINT_FREQ     10
+
+// See README for documentation
+
+/*readonly*/ CProxy_Main mainProxy;
+/*readonly*/ int arrayDimX;
+/*readonly*/ int arrayDimY;
+/*readonly*/ int arrayDimZ;
+/*readonly*/ int blockDimX;
+/*readonly*/ int blockDimY;
+/*readonly*/ int blockDimZ;
+
+// specify the number of worker chares in each dimension
+/*readonly*/ int num_chare_x;
+/*readonly*/ int num_chare_y;
+/*readonly*/ int num_chare_z;
+
+/*readonly*/ int globalBarrier;
+
+static unsigned long next = 1;
+
+int myrand(int numpes) {
+  next = next * 1103515245 + 12345;
+  return((unsigned)(next/65536) % numpes);
+}
+
+// We want to wrap entries around, and because mod operator % 
+// sometimes misbehaves on negative values. -1 maps to the highest value.
+#define wrap_x(a)      (((a)+num_chare_x)%num_chare_x)
+#define wrap_y(a)      (((a)+num_chare_y)%num_chare_y)
+#define wrap_z(a)      (((a)+num_chare_z)%num_chare_z)
+
+#define index(a, b, c) ((a)*(blockDimY+2)*(blockDimZ+2) + (b)*(blockDimZ+2) + (c))
+
+#define WARM_ITER              5
+#define LEFT                   1
+#define RIGHT                  2
+#define TOP                    3
+#define BOTTOM                 4
+#define FRONT                  5
+#define BACK                   6
+#define DIVIDEBY7              0.14285714285714285714
+
+double startTime;
+double endTime;
+
+/** \class ghostMsg
+ *
+ */
+class ghostMsg: public CMessage_ghostMsg {
+  public:
+    int dir;
+    int height;
+    int width;
+    double* gh;
+
+    ghostMsg(int _d, int _h, int _w) : dir(_d), height(_h), width(_w) {
+    }
+};
+
+/** \class Main
+ *
+ */
+class Main : public CBase_Main {
+  public:
+    CProxy_Jacobi array;
+    int iterations;
+       std::vector<std::pair<double,int> > times;
+
+    Main(CkArgMsg* m) {
+      if ( (m->argc != 3) && (m->argc != 7) ) {
+        CkPrintf("%s [array_size] [block_size]\n", m->argv[0]);
+        CkPrintf("OR %s [array_size_X] [array_size_Y] [array_size_Z] [block_size_X] [block_size_Y] [block_size_Z]\n", m->argv[0]);
+        CkAbort("Abort");
+      }
+
+      // set iteration counter to zero
+      iterations = 0;
+
+      // store the main proxy
+      mainProxy = thisProxy;
+       
+      if(m->argc == 3) {
+       arrayDimX = arrayDimY = arrayDimZ = atoi(m->argv[1]);
+        blockDimX = blockDimY = blockDimZ = atoi(m->argv[2]); 
+      }
+      else if (m->argc == 7) {
+        arrayDimX = atoi(m->argv[1]);
+       arrayDimY = atoi(m->argv[2]);
+       arrayDimZ = atoi(m->argv[3]);
+        blockDimX = atoi(m->argv[4]); 
+       blockDimY = atoi(m->argv[5]); 
+       blockDimZ = atoi(m->argv[6]);
+      }
+
+      if (arrayDimX < blockDimX || arrayDimX % blockDimX != 0)
+        CkAbort("array_size_X % block_size_X != 0!");
+      if (arrayDimY < blockDimY || arrayDimY % blockDimY != 0)
+        CkAbort("array_size_Y % block_size_Y != 0!");
+      if (arrayDimZ < blockDimZ || arrayDimZ % blockDimZ != 0)
+        CkAbort("array_size_Z % block_size_Z != 0!");
+
+      num_chare_x = arrayDimX / blockDimX;
+      num_chare_y = arrayDimY / blockDimY;
+      num_chare_z = arrayDimZ / blockDimZ;
+
+      // print info
+      CkPrintf("\nSTENCIL COMPUTATION WITH BARRIERS\n");
+      CkPrintf("Running Jacobi on %d processors with (%d, %d, %d) chares\n", CkNumPes(), num_chare_x, num_chare_y, num_chare_z);
+      CkPrintf("Array Dimensions: %d %d %d\n", arrayDimX, arrayDimY, arrayDimZ);
+      CkPrintf("Block Dimensions: %d %d %d\n", blockDimX, blockDimY, blockDimZ);
+
+      // Create new array of worker chares
+      array = CProxy_Jacobi::ckNew(num_chare_x, num_chare_y, num_chare_z);
+
+      CkArray *jarr = array.ckLocalBranch();
+      int jmap[num_chare_x][num_chare_y][num_chare_z];
+
+      int hops=0, p;
+      for(int i=0; i<num_chare_x; i++)
+       for(int j=0; j<num_chare_y; j++)
+         for(int k=0; k<num_chare_z; k++) {
+           jmap[i][j][k] = jarr->procNum(CkArrayIndex3D(i, j, k));
+         }
+
+               //Start the computation
+               startTime = CmiWallTimer();
+
+               //Registering the callback
+               CkCallback *cb = new CkCallback(CkIndex_Main::report(NULL), mainProxy);
+               array.ckSetReductionClient(cb);
+
+               array.doStep();
+    }
+
+    // Each worker reports back to here when it completes an iteration
+       void report(CkReductionMsg *msg) {
+               int *value = (int *)msg->getData();
+       iterations = value[0];
+               if (iterations < MAX_ITER) {
+                       times.push_back(std::make_pair(CmiWallTimer() - startTime,iterations));
+                       if(iterations != 0 && iterations % CKP_FREQ == 0){
+                               CkCallback cb (CkIndex_Jacobi::doStep(), array);
+                               CkStartMemCheckpoint(cb);               
+                       }else{
+                               array.doStep();
+                       }
+       } else {
+                       CkPrintf("Completed %d iterations\n", MAX_ITER-1);
+                       endTime = CmiWallTimer();
+//                     CkPrintf("Time elapsed per iteration: %f\n", (endTime - startTime)/(MAX_ITER-1));
+//                     for(int i = 1; i < times.size(); i++)
+//                             CkPrintf("time=%.2f it=%d\n",times[i].first,times[i].second);
+                       CkExit();
+               }
+       }
+
+};
+
+/** \class Jacobi
+ *
+ */
+
+class Jacobi: public CBase_Jacobi {
+  // Jacobi_SDAG_CODE
+
+  public:
+    int iterations;
+    int imsg;
+
+    double *temperature;
+    double *new_temperature;
+
+    // Constructor, initialize values
+    Jacobi() {
+
+       int i, j, k;
+       
+               // allocate a three dimensional array
+               temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+       new_temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+
+
+               for(i=0; i<blockDimX+2; ++i) {
+                       for(j=0; j<blockDimY+2; ++j) {
+                               for(k=0; k<blockDimZ+2; ++k) {
+                               temperature[index(i, j, k)] = 0.0;
+                               }
+                       } 
+       }
+
+               iterations = 0;
+       imsg = 0;
+       constrainBC();
+    }
+
+    Jacobi(CkMigrateMessage* m): CBase_Jacobi(m) {}
+
+    ~Jacobi() { 
+      delete [] temperature; 
+      delete [] new_temperature; 
+    }
+
+
+       // Pupping function for migration and fault tolerance
+       // Condition: assuming the 3D Chare Arrays are NOT used
+       void pup(PUP::er &p){
+       
+               // calling parent's pup
+               CBase_Jacobi::pup(p);
+               
+               // pupping properties of this class
+               p | iterations;
+               p | imsg;
+
+               // if unpacking, allocate the memory space
+               if(p.isUnpacking()){
+                       temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+                       new_temperature = new double[(blockDimX+2) * (blockDimY+2) * (blockDimZ+2)];
+       
+               }
+
+               // pupping the arrays
+               p((char *)temperature, (blockDimX+2) * (blockDimY+2) * (blockDimZ+2) * sizeof(double));
+               //p((char *) new_temperature, (blockDimX+2) * (blockDimY+2) * (blockDimZ+2) * sizeof(double));
+       }
+
+    // Send ghost faces to the six neighbors
+    void doStep(void) {
+      if (thisIndex.x == 0 && thisIndex.y == 0 && thisIndex.z == 0 && iterations % PRINT_FREQ == 0) {
+          CkPrintf("Start of iteration %d at %f\n", iterations,CmiWallTimer());
+          //BgPrintf("BgPrint> Start of iteration at %f\n");
+      }
+      iterations++;
+       imsg++;
+               
+
+      // Copy different faces into messages
+      ghostMsg *leftMsg = new (blockDimY*blockDimZ) ghostMsg(RIGHT, blockDimY, blockDimZ);
+      ghostMsg *rightMsg = new (blockDimY*blockDimZ) ghostMsg(LEFT, blockDimY, blockDimZ);
+      ghostMsg *topMsg = new (blockDimX*blockDimZ) ghostMsg(BOTTOM, blockDimX, blockDimZ);
+      ghostMsg *bottomMsg = new (blockDimX*blockDimZ) ghostMsg(TOP, blockDimX, blockDimZ);
+      ghostMsg *frontMsg = new (blockDimX*blockDimY) ghostMsg(BACK, blockDimX, blockDimY);
+      ghostMsg *backMsg = new (blockDimX*blockDimY) ghostMsg(FRONT, blockDimX, blockDimY);
+
+               
+       
+      CkSetRefNum(leftMsg, iterations);
+      CkSetRefNum(rightMsg, iterations);
+      CkSetRefNum(topMsg, iterations);
+      CkSetRefNum(bottomMsg, iterations);
+      CkSetRefNum(frontMsg, iterations);
+      CkSetRefNum(backMsg, iterations);
+
+
+     for(int j=0; j<blockDimY; ++j) 
+       for(int k=0; k<blockDimZ; ++k) {
+         leftMsg->gh[k*blockDimY+j] = temperature[index(1, j+1, k+1)];
+         rightMsg->gh[k*blockDimY+j] = temperature[index(blockDimX, j+1, k+1)];
+      }
+
+               
+               
+
+      for(int i=0; i<blockDimX; ++i) 
+       for(int k=0; k<blockDimZ; ++k) {
+         topMsg->gh[k*blockDimX+i] = temperature[index(i+1, 1, k+1)];
+         bottomMsg->gh[k*blockDimX+i] = temperature[index(i+1, blockDimY, k+1)];
+      }
+
+               
+
+      for(int i=0; i<blockDimX; ++i) 
+       for(int j=0; j<blockDimY; ++j) {
+         frontMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, 1)];
+         backMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, blockDimZ)];
+      }
+
+               
+
+      // Send my left face
+      thisProxy(wrap_x(thisIndex.x-1), thisIndex.y, thisIndex.z).receiveGhosts(leftMsg);
+      // Send my right face
+      thisProxy(wrap_x(thisIndex.x+1), thisIndex.y, thisIndex.z).receiveGhosts(rightMsg);
+      // Send my top face
+      thisProxy(thisIndex.x, wrap_y(thisIndex.y-1), thisIndex.z).receiveGhosts(topMsg);
+      // Send my bottom face
+      thisProxy(thisIndex.x, wrap_y(thisIndex.y+1), thisIndex.z).receiveGhosts(bottomMsg);
+      // Send my front face
+      thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z-1)).receiveGhosts(frontMsg);
+      // Send my back face
+      thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z+1)).receiveGhosts(backMsg);
+       
+        if(imsg == 7){
+        imsg = 0;
+        check_and_compute();
+    }  
+
+    }
+
+    void receiveGhosts(ghostMsg *gmsg) {
+      int height = gmsg->height;
+      int width = gmsg->width;
+
+//             CkPrintf("[%d] Receiving data %d %d from %d...\n",CkMyPe(), height, width, gmsg->dir);
+               
+
+      switch(gmsg->dir) {
+       case LEFT:
+         for(int j=0; j<height; ++j) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(0, j+1, k+1)] = gmsg->gh[k*height+j];
+         }
+         break;
+       case RIGHT:
+         for(int j=0; j<height; ++j) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(blockDimX+1, j+1, k+1)] = gmsg->gh[k*height+j];
+         }
+         break;
+       case TOP:
+         for(int i=0; i<height; ++i) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(i+1, 0, k+1)] = gmsg->gh[k*height+i];
+         }
+         break;
+       case BOTTOM:
+         for(int i=0; i<height; ++i) 
+           for(int k=0; k<width; ++k) {
+             temperature[index(i+1, blockDimY+1, k+1)] = gmsg->gh[k*height+i];
+         }
+         break;
+       case FRONT:
+         for(int i=0; i<height; ++i) 
+           for(int j=0; j<width; ++j) {
+             temperature[index(i+1, j+1, blockDimZ+1)] = gmsg->gh[j*height+i];
+         }
+         break;
+       case BACK:
+         for(int i=0; i<height; ++i) 
+           for(int j=0; j<width; ++j) {
+             temperature[index(i+1, j+1, 0)] = gmsg->gh[j*height+i];
+         }
+         break;
+       default:
+          CkAbort("ERROR\n");
+      } 
+
+      delete gmsg;
+
+
+               imsg++;
+               if(imsg == 7){
+                       imsg = 0;
+                       check_and_compute();
+               }
+    }
+
+
+       void check_and_compute() {
+       compute_kernel();
+
+               // interchanging the two different arrays
+               double *tmp;
+       tmp = temperature;
+       temperature = new_temperature;
+       new_temperature = tmp;
+
+               constrainBC();
+               contribute(sizeof(int), &iterations, CkReduction::max_int);
+       }
+
+    // Check to see if we have received all neighbor values yet
+    // If all neighbor values have been received, we update our values and proceed
+       void compute_kernel() {
+
+               for(int i=1; i<blockDimX+1; ++i) {
+                       for(int j=1; j<blockDimY+1; ++j) {
+                               for(int k=1; k<blockDimZ+1; ++k) {
+
+                               // update my value based on the surrounding values
+                               new_temperature[index(i, j, k)] = (temperature[index(i-1, j, k)] 
+                                           +  temperature[index(i+1, j, k)]
+                                           +  temperature[index(i, j-1, k)]
+                                           +  temperature[index(i, j+1, k)]
+                                           +  temperature[index(i, j, k-1)]
+                                           +  temperature[index(i, j, k+1)]
+                                           +  temperature[index(i, j, k)] ) * DIVIDEBY7; 
+                               }
+                       }
+       } 
+    }
+
+    // Enforce some boundary conditions
+       void constrainBC() {
+       // Heat left, top and front faces of each chare's block
+       for(int i=1; i<blockDimX+1; ++i)
+                       for(int k=1; k<blockDimZ+1; ++k)
+                               temperature[index(i, 1, k)] = 255.0;
+       for(int j=1; j<blockDimY+1; ++j)
+                       for(int k=1; k<blockDimZ+1; ++k)
+                               temperature[index(1, j, k)] = 255.0;
+       for(int i=1; i<blockDimX+1; ++i)
+                       for(int j=1; j<blockDimY+1; ++j)
+                               temperature[index(i, j, 1)] = 255.0; 
+       }
+
+};
+
+#include "jacobi3d.def.h"
diff --git a/tests/charm++/jacobi3d/jacobi3d.ci b/tests/charm++/jacobi3d/jacobi3d.ci
new file mode 100644 (file)
index 0000000..5ff410b
--- /dev/null
@@ -0,0 +1,33 @@
+mainmodule jacobi3d {
+
+  readonly CProxy_Main mainProxy;
+  readonly int arrayDimX;
+  readonly int arrayDimY;
+  readonly int arrayDimZ;
+  readonly int blockDimX;
+  readonly int blockDimY;
+  readonly int blockDimZ;
+
+  readonly int num_chare_x;
+  readonly int num_chare_y;
+  readonly int num_chare_z;
+
+  readonly int globalBarrier;
+
+  message ghostMsg {
+    double gh[];
+  };
+
+  mainchare Main {
+    entry Main(CkArgMsg *m);
+    entry void report(CkReductionMsg *msg);
+  };
+
+  array [3D] Jacobi {
+    // Normal Charm++ entry methods
+    entry Jacobi(void);
+    entry void receiveGhosts(ghostMsg *gmsg);
+    entry void doStep();
+  };
+
+};
diff --git a/tests/charm++/jacobi3d/kill_01.txt b/tests/charm++/jacobi3d/kill_01.txt
new file mode 100644 (file)
index 0000000..44e9006
--- /dev/null
@@ -0,0 +1 @@
+3 50
diff --git a/tests/charm++/jacobi3d/kill_02.txt b/tests/charm++/jacobi3d/kill_02.txt
new file mode 100644 (file)
index 0000000..4dc7abb
--- /dev/null
@@ -0,0 +1 @@
+3 35
diff --git a/tests/charm++/jacobi3d/kill_03.txt b/tests/charm++/jacobi3d/kill_03.txt
new file mode 100644 (file)
index 0000000..1ea779d
--- /dev/null
@@ -0,0 +1,2 @@
+3 35
+6 100