Feature #195: Remove ckdirect examples 96/4596/7
authorNitin Bhat <nbhat4@illinois.edu>
Fri, 21 Sep 2018 14:01:06 +0000 (09:01 -0500)
committerNitin Bhat <nbhat4@illinois.edu>
Mon, 26 Nov 2018 19:00:59 +0000 (13:00 -0600)
Change-Id: I15b8a9cbb69f0e64dae3768f76a6915e9ec5ada0

examples/charm++/ckdirect/matmul3d/Makefile [deleted file]
examples/charm++/ckdirect/matmul3d/matmul3d.C [deleted file]
examples/charm++/ckdirect/matmul3d/matmul3d.ci [deleted file]
examples/charm++/ckdirect/matmul3d/matmul3d.h [deleted file]
examples/charm++/ckdirect/stencil3d/Makefile [deleted file]
examples/charm++/ckdirect/stencil3d/stencil3d.C [deleted file]
examples/charm++/ckdirect/stencil3d/stencil3d.ci [deleted file]

diff --git a/examples/charm++/ckdirect/matmul3d/Makefile b/examples/charm++/ckdirect/matmul3d/Makefile
deleted file mode 100644 (file)
index cd94947..0000000
+++ /dev/null
@@ -1,33 +0,0 @@
-CHARMC=../../../../bin/charmc $(OPTS)
-
-MATH_LIB        = -lesslbg -lesslsmpbg -lxlf90_r  \
-                 -lmass -lmassv -lxlfmath -lxlomp_ser -lxlsmp -lpthread
-
-LOCAL_LIBS      = -L/soft/apps/LAPACK \
-                 -L/opt/ibmcmp/xlf/bg/11.1/bglib \
-                 -L/opt/ibmcmp/xlsmp/bg/1.7/bglib \
-                 -L/bgsys/ibm_essl/sles10/prod/opt/ibmmath/lib \
-                 -L/bgsys/drivers/ppcfloor/gnu-linux/powerpc-bgp-linux/lib \
-                 -L/gpfs/home/bohm/zlib/lib
-
-OBJS = matmul3d.o
-
-all: matmul3d
-
-matmul3d: $(OBJS)
-       $(CHARMC) -O3 -language charm++ $(LOCAL_LIBS) $(MATH_LIB) -o matmul3d $(OBJS)
-
-projections: $(OBJS)
-       $(CHARMC) -O3 -language charm++ -tracemode projections -lz -o matmul3d.prj $(OBJS)
-
-summary: $(OBJS)
-       $(CHARMC) -O3 -language charm++ -tracemode summary -lz -o matmul3d.sum $(OBJS)
-
-matmul3d.decl.h: matmul3d.ci
-       $(CHARMC)  matmul3d.ci
-
-clean:
-       rm -f *.decl.h *.def.h conv-host *.o matmul3d matmul3d.prj charmrun *~
-
-matmul3d.o: matmul3d.C matmul3d.h matmul3d.decl.h
-       $(CHARMC) -c -O3 -I/bgsys/ibm_essl/sles10/prod/opt/ibmmath/include matmul3d.C
diff --git a/examples/charm++/ckdirect/matmul3d/matmul3d.C b/examples/charm++/ckdirect/matmul3d/matmul3d.C
deleted file mode 100644 (file)
index ef6f518..0000000
+++ /dev/null
@@ -1,583 +0,0 @@
-/** \file matmul3d.C
- *  Author: Abhinav S Bhatele
- *  Date Created: April 01st, 2008
- *
- */
-
-#include <stdlib.h>
-#include "ckdirect.h"
-#include "matmul3d.decl.h"
-#include "matmul3d.h"
-#include "TopoManager.h"
-#if CMK_BLUEGENEP || CMK_VERSION_BLUEGENE
-#include "essl.h"
-#endif
-#include "rand48_replacement.h"
-
-Main::Main(CkArgMsg* m) {
-  if ( (m->argc != 3) && (m->argc != 7) && (m->argc != 10) ) {
-    CkPrintf("%s [array_size] [block_size]\n", m->argv[0]);
-    CkPrintf("OR %s [array_size_X] [array_size_Y] [array_size_Z] [block_size_X] [block_size_Y] [block_size_Z]\n", m->argv[0]);
-    CkPrintf("OR %s [array_size_X] [array_size_Y] [array_size_Z] [block_size_X] [block_size_Y] [block_size_Z] [torus_dim_X] [torus_dim_Y] [torus_dim_Z] \n", m->argv[0]);
-    CkAbort("Abort");
-  }
-
-  // store the main proxy
-  mainProxy = thisProxy;
-
-  // get the size of the global array, size of each chare and size of the torus [optional]
-  if(m->argc == 3) {
-    arrayDimX = arrayDimY = arrayDimZ = atoi(m->argv[1]);
-    blockDimX = blockDimY = blockDimZ = atoi(m->argv[2]);
-  }
-  else if (m->argc == 7) {
-    arrayDimX = atoi(m->argv[1]);
-    arrayDimY = atoi(m->argv[2]);
-    arrayDimZ = atoi(m->argv[3]);
-    blockDimX = atoi(m->argv[4]);
-    blockDimY = atoi(m->argv[5]);
-    blockDimZ = atoi(m->argv[6]);
-  } else {
-    arrayDimX = atoi(m->argv[1]);
-    arrayDimY = atoi(m->argv[2]);
-    arrayDimZ = atoi(m->argv[3]);
-    blockDimX = atoi(m->argv[4]);
-    blockDimY = atoi(m->argv[5]);
-    blockDimZ = atoi(m->argv[6]);
-    torusDimX = atoi(m->argv[7]);
-    torusDimY = atoi(m->argv[8]);
-    torusDimZ = atoi(m->argv[9]);
-  }
-
-  if (arrayDimX < blockDimX || arrayDimX % blockDimX != 0)
-    CkAbort("array_size_X % block_size_X != 0!");
-  if (arrayDimY < blockDimY || arrayDimY % blockDimY != 0)
-    CkAbort("array_size_Y % block_size_Y != 0!");
-  if (arrayDimZ < blockDimZ || arrayDimZ % blockDimZ != 0)
-    CkAbort("array_size_Z % block_size_Z != 0!");
-
-  num_chare_x = arrayDimX / blockDimX;
-  num_chare_y = arrayDimY / blockDimY;
-  num_chare_z = arrayDimZ / blockDimZ;
-  num_chares = num_chare_x * num_chare_y * num_chare_z;
-
-  subBlockDimXz = blockDimX/num_chare_z;
-  subBlockDimYx = blockDimY/num_chare_x;
-  subBlockDimXy = blockDimX/num_chare_y;
-
-  // print info
-  CkPrintf("Running Matrix Multiplication on %d processors with (%d, %d, %d) chares\n", CkNumPes(), num_chare_x, num_chare_y, num_chare_z);
-  CkPrintf("Array Dimensions: %d %d %d\n", arrayDimX, arrayDimY, arrayDimZ);
-  CkPrintf("Block Dimensions: %d %d %d\n", blockDimX, blockDimY, blockDimZ);
-
-  // Create new array of worker chares
-#if USE_TOPOMAP
-  CkPrintf("Topology Mapping is being done ...\n");
-  CProxy_ComputeMap map = CProxy_ComputeMap::ckNew(num_chare_x, num_chare_y, num_chare_z, 1, 1, 1);
-  CkArrayOptions opts(num_chare_x, num_chare_y, num_chare_z);
-  opts.setMap(map);
-  compute = CProxy_Compute::ckNew(opts);
-#elif USE_BLOCKMAP
-  CkPrintf("Block Mapping is being done ...\n");
-  CProxy_ComputeMap map = CProxy_ComputeMap::ckNew(num_chare_x, num_chare_y, num_chare_z, torusDimX, torusDimY, torusDimZ);
-  CkArrayOptions opts(num_chare_x, num_chare_y, num_chare_z);
-  opts.setMap(map);
-  compute = CProxy_Compute::ckNew(opts);
-#else
-  compute = CProxy_Compute::ckNew(num_chare_x, num_chare_y, num_chare_z);
-#endif
-
-  // CkPrintf("Total Hops: %d\n", hops);
-
-  // Start the computation
-  numIterations = 0;
-  startTime = CkWallTimer();
-#if USE_CKDIRECT
-  compute.setupChannels();
-#else
-  compute.beginCopying();
-#endif
-}
-
-void Main::done() {
-  numIterations++;
-  if(numIterations == 1) {
-    firstTime = CkWallTimer();
-#if USE_CKDIRECT
-    CkPrintf("FIRST ITER TIME %f secs\n", firstTime - setupTime);
-#else
-    CkPrintf("FIRST ITER TIME %f secs\n", firstTime - startTime);
-#endif
-    compute.resetArrays();
-  } else {
-    if(numIterations == NUM_ITER) {
-      endTime[numIterations-2] = CkWallTimer() - firstTime;
-      double sum = 0;
-      for(int i=0; i<NUM_ITER-1; i++)
-       sum += endTime[i];
-#if USE_CKDIRECT
-      CkPrintf("AVG TIME %f secs\n", sum/(NUM_ITER-1));
-#else
-      CkPrintf("AVG TIME %f secs\n", sum/(NUM_ITER-1));
-#endif
-      CkExit();
-    } else {
-      endTime[numIterations-2] = CkWallTimer() - firstTime;
-      compute.resetArrays();
-    }
-  }
-}
-
-void Main::resetDone() {
-  firstTime = CkWallTimer();
-  compute.beginCopying();
-}
-
-void Main::setupDone() {
-  setupTime = CkWallTimer();
-  CkPrintf("SETUP TIME %f secs\n", setupTime - startTime);
-  compute.beginCopying();
-}
-
-// Constructor, initialize values
-Compute::Compute() {
-  // each compute will only hold blockDimX/num_chare_z no. of rows to begin with
-  A = new float[blockDimX*blockDimY];
-  B = new float[blockDimY*blockDimZ];
-  C = new float[blockDimX*blockDimZ];
-#if USE_CKDIRECT
-  tmpC = new float[blockDimX*blockDimZ];
-#endif
-
-  int indexX = thisIndex.x;
-  int indexY = thisIndex.y;
-  int indexZ = thisIndex.z;
-
-  float tmp;
-
-  for(int i=indexZ*subBlockDimXz; i<(indexZ+1)*subBlockDimXz; i++)
-    for(int j=0; j<blockDimY; j++) {
-      tmp = (float)drand48();
-      while(tmp > MAX_LIMIT || tmp < (-1)*MAX_LIMIT)
-        tmp = (float)drand48();
-
-      A[i*blockDimY + j] = tmp;
-  }
-
-  for(int j=indexX*subBlockDimYx; j<(indexX+1)*subBlockDimYx; j++)
-    for(int k=0; k<blockDimZ; k++) {
-      tmp = (float)drand48();
-      while(tmp > MAX_LIMIT || tmp < (-1)*MAX_LIMIT)
-        tmp = (float)drand48();
-
-      B[j*blockDimZ + k] = tmp;
-  }
-
-  for(int i=0; i<blockDimX; i++)
-    for(int k=0; k<blockDimZ; k++) {
-      C[i*blockDimZ + k] = 0.0;
-#if USE_CKDIRECT
-      tmpC[i*blockDimZ + k] = 0.0;
-#endif
-    }
-
-  // counters to keep track of how many messages have been received
-  countA = 0;
-  countB = 0;
-  countC = 0;
-
-#if USE_CKDIRECT
-  sHandles = new infiDirectUserHandle[num_chare_x + num_chare_y + num_chare_z];
-  rHandles = new infiDirectUserHandle[num_chare_x + num_chare_y + num_chare_z];
-#endif
-}
-
-Compute::Compute(CkMigrateMessage* m) {
-
-}
-
-Compute::~Compute() {
-  delete [] A;
-  delete [] B;
-  delete [] C;
-#if USE_CKDIRECT
-  delete [] tmpC;
-#endif
-}
-
-void Compute::beginCopying() {
-  sendA();
-  sendB();
-}
-
-void Compute::resetArrays() {
-  int indexX = thisIndex.x;
-  int indexY = thisIndex.y;
-  int indexZ = thisIndex.z;
-  
-  float tmp;
-  
-  for(int i=indexZ*subBlockDimXz; i<(indexZ+1)*subBlockDimXz; i++)
-    for(int j=0; j<blockDimY; j++) {
-      tmp = (float)drand48(); 
-      while(tmp > MAX_LIMIT || tmp < (-1)*MAX_LIMIT)
-        tmp = (float)drand48();
-
-      A[i*blockDimY + j] = tmp;
-  }
-
-  for(int j=indexX*subBlockDimYx; j<(indexX+1)*subBlockDimYx; j++)
-    for(int k=0; k<blockDimZ; k++) {
-      tmp = (float)drand48();
-      while(tmp > MAX_LIMIT || tmp < (-1)*MAX_LIMIT)
-        tmp = (float)drand48();
-
-      B[j*blockDimZ + k] = tmp;
-  }
-
-  for(int i=0; i<blockDimX; i++)
-    for(int k=0; k<blockDimZ; k++) {
-      C[i*blockDimZ + k] = 0.0;
-#if USE_CKDIRECT
-      tmpC[i*blockDimZ + k] = 0.0;
-#endif
-    }
-
-  contribute(CkCallback(CkReductionTarget(Main, resetDone), mainProxy));
-}
-
-void Compute::sendA() {
-  int indexZ = thisIndex.z;
-
-  for(int k=0; k<num_chare_z; k++)
-    if(k != indexZ) {
-#if USE_CKDIRECT
-      CkDirect_put(&sHandles[num_chare_x + num_chare_y + k]);
-#else
-      // use a local pointer for chares on the same processor
-      Compute* c = compute(thisIndex.x, thisIndex.y, k).ckLocal();
-      if(c != NULL)
-       c->receiveA(indexZ, &A[indexZ*subBlockDimXz*blockDimY], subBlockDimXz * blockDimY);
-      else
-       compute(thisIndex.x, thisIndex.y, k).receiveA(indexZ, &A[indexZ*subBlockDimXz*blockDimY], subBlockDimXz * blockDimY);
-#endif
-    }
-}
-
-void Compute::sendB() {
-  int indexX = thisIndex.x;
-
-  for(int i=0; i<num_chare_x; i++)
-    if(i != indexX) {
-#if USE_CKDIRECT
-      CkDirect_put(&sHandles[i]);
-#else
-      // use a local pointer for chares on the same processor
-      Compute* c = compute(i, thisIndex.y, thisIndex.z).ckLocal();
-      if(c != NULL)
-       c->receiveB(indexX, &B[indexX*subBlockDimYx*blockDimZ], subBlockDimYx * blockDimZ);
-      else
-       compute(i, thisIndex.y, thisIndex.z).receiveB(indexX, &B[indexX*subBlockDimYx*blockDimZ], subBlockDimYx * blockDimZ);
-#endif
-    }
-}
-
-void Compute::sendC() {
-  int indexY = thisIndex.y;
-  for(int j=0; j<num_chare_y; j++) {
-    if(j != indexY) {
-#if USE_CKDIRECT
-      CkDirect_put(&sHandles[num_chare_x + j]);
-#else
-      // use a local pointer for chares on the same processor
-      Compute *c = compute(thisIndex.x, j, thisIndex.z).ckLocal();
-      if(c != NULL)
-       c->receiveC(&C[j*subBlockDimXy*blockDimZ], subBlockDimXy * blockDimZ, 1);
-      else
-       compute(thisIndex.x, j, thisIndex.z).receiveC(&C[j*subBlockDimXy*blockDimZ], subBlockDimXy * blockDimZ, 1);
-#endif
-    }
-  }
-}
-
-void Compute::receiveA(int indexZ, float *data, int size) {
-  for(int i=0; i<subBlockDimXz; i++)
-    for(int j=0; j<blockDimY; j++)
-      A[indexZ*subBlockDimXz*blockDimY + i*blockDimY + j] = data[i*blockDimY + j];
-  countA++;
-  if(countA == num_chare_z-1 && countB == num_chare_x-1)
-    doWork();
-}
-
-void Compute::receiveB(int indexX, float *data, int size) {
-  for(int j=0; j<subBlockDimYx; j++)
-    for(int k=0; k<blockDimZ; k++)
-      B[indexX*subBlockDimYx*blockDimZ + j*blockDimZ + k] = data[j*blockDimZ + k];
-  countB++;
-  if(countA == num_chare_z-1 && countB == num_chare_x-1)
-    doWork();
-}
-
-void Compute::receiveC(float *data, int size, int who) {
-  int indexY = thisIndex.y;
-  if(who) {
-    for(int i=0; i<subBlockDimXy; i++)
-      for(int k=0; k<blockDimZ; k++)
-       C[indexY*subBlockDimXy*blockDimZ + i*blockDimZ + k] += data[i*blockDimZ + k];
-  }
-  countC++;
-  if(countC == num_chare_y) {
-    /*char name[30];
-    sprintf(name, "%s_%d_%d_%d", "C", thisIndex.x, thisIndex.y, thisIndex.z);
-    FILE *fp = fopen(name, "w");
-    for(int i=0; i<subBlockDimXy; i++) {
-      for(int k=0; k<blockDimZ; k++)
-       fprintf(fp, "%f ", C[indexY*subBlockDimXy*blockDimZ + i*blockDimZ + k]);
-      fprintf(fp, "\n");
-    }
-    fclose(fp);*/
-
-    // counters to keep track of how many messages have been received
-    countA = 0;
-    countB = 0;
-    countC = 0;
-
-    contribute(0, 0, CkReduction::concat, CkCallback(CkIndex_Main::done(), mainProxy));
-    // mainProxy.done();
-  }
-}
-
-void Compute::receiveC() {
-  int indexX = thisIndex.x;
-  int indexY = thisIndex.y;
-  int indexZ = thisIndex.z;
-
-  // copy C from tmpC to the correct location
-  for(int j=0; j<num_chare_y; j++) {
-    if( j != indexY) {
-      for(int i=0; i<subBlockDimXy; i++)
-       for(int k=0; k<blockDimZ; k++)
-         C[indexY*subBlockDimXy*blockDimZ + i*blockDimZ + k] += tmpC[j*subBlockDimXy*blockDimZ + i*blockDimZ + k];
-    }
-  }
-  /*char name[30];
-  sprintf(name, "%s_%d_%d_%d", "C", thisIndex.x, thisIndex.y, thisIndex.z);
-  FILE *fp = fopen(name, "w");
-  for(int i=0; i<subBlockDimXy; i++) {
-    for(int k=0; k<blockDimZ; k++)
-      fprintf(fp, "%f ", C[indexY*subBlockDimXy*blockDimZ + i*blockDimZ + k]);
-    fprintf(fp, "\n");
-  }
-  fclose(fp);
-  CkPrintf("%d_%d_%d\n", thisIndex.x, thisIndex.y, thisIndex.z);
-  for(int i=0; i<subBlockDimXy; i++) {
-    for(int k=0; k<blockDimZ; k++)
-      CkPrintf("%f ", C[indexY*subBlockDimXy*blockDimZ + i*blockDimZ + k]);
-    CkPrintf("\n");
-  }*/
-
-  // call ready for the buffers
-  for(int i=0; i<num_chare_x; i++)
-    if(i != indexX)
-      CkDirect_ready(&rHandles[i]);
-
-  for(int j=0; j<num_chare_y; j++)
-    if(j != indexY)
-      CkDirect_ready(&rHandles[num_chare_x + j]);
-  
-  for(int k=0; k<num_chare_z; k++)
-    if(k != indexZ)
-      CkDirect_ready(&rHandles[num_chare_x + num_chare_y + k]);
-
-  // counters to keep track of how many messages have been received
-  countA = 0;
-  countB = 0;
-  countC = 0;
-
-  contribute(0, 0, CkReduction::concat, CkCallback(CkIndex_Main::done(), mainProxy));
-  // mainProxy.done();
-}
-
-void Compute::doWork() {
-  if(countA == num_chare_z-1 && countB == num_chare_x-1) {
-
-#if CMK_BLUEGENEP || CMK_VERSION_BLUEGENE
-    const char trans = 'N';
-    const double alpha = 1.0;
-    const double beta = 0.0;
-    
-    sgemm(&trans, &trans, blockDimX, blockDimZ, blockDimY, alpha, A, blockDimX, B, blockDimY, beta, C, blockDimX);
-#else
-    for(int i=0; i<blockDimX; i++)
-      for(int j=0; j<blockDimY; j++)
-       for(int k=0; k<blockDimZ; k++)
-         C[i*blockDimZ+k] += A[i*blockDimY+j] * B[j*blockDimZ+k];
-#endif
-
-#if USE_CKDIRECT
-    callBackRcvdC((void *)this);
-#else
-    receiveC(&C[(thisIndex.y)*subBlockDimXy*blockDimZ], subBlockDimXy*blockDimZ, 0);
-#endif
-    sendC();
-  }
-}
-
-void Compute::setupChannels() {
-  int mype = CkMyPe();
-  int x = thisIndex.x;
-  int y = thisIndex.y;
-  int z = thisIndex.z;
-
-  for(int k=0; k<num_chare_z; k++)
-    if(k != z)
-      thisProxy(x, y, k).notifyReceiver(mype, thisIndex, SENDA);
-
-  for(int i=0; i<num_chare_x; i++)
-    if(i != x)
-      thisProxy(i, y, z).notifyReceiver(mype, thisIndex, SENDB);
-
-  for(int j=0; j<num_chare_y; j++)
-    if(j != y)
-      thisProxy(x, j, z).notifyReceiver(mype, thisIndex, SENDC);
-
-}
-
-void Compute::notifyReceiver(int pe, CkIndex3D index, int arr) {
-  // --- B --- | --- C --- | --- A ---
-  if(arr == SENDA) {
-    rHandles[num_chare_x + num_chare_y + index.z] = CkDirect_createHandle(pe, &A[(index.z)*subBlockDimXz*blockDimY], sizeof(float)*subBlockDimXz*blockDimY, Compute::callBackRcvdA, (void *)this, OOB);
-    thisProxy(index.x, index.y, index.z).recvHandle(rHandles[num_chare_x + num_chare_y + index.z], thisIndex.z, arr);
-  }
-
-  if(arr == SENDB) {
-    rHandles[index.x] = CkDirect_createHandle(pe, &B[(index.x)*subBlockDimYx*blockDimZ], sizeof(float)*subBlockDimYx*blockDimZ, Compute::callBackRcvdB, (void *)this, OOB);
-    thisProxy(index.x, index.y, index.z).recvHandle(rHandles[index.x], thisIndex.x, arr);
-  }
-
-  if(arr == SENDC) {
-    rHandles[num_chare_x + index.y] = CkDirect_createHandle(pe, &tmpC[(index.y)*subBlockDimXy*blockDimZ], sizeof(float)*subBlockDimXy*blockDimZ, Compute::callBackRcvdC, (void *)this, OOB);
-    thisProxy(index.x, index.y, index.z).recvHandle(rHandles[num_chare_x + index.y], thisIndex.y, arr);
-  }
-}
-
-void Compute::recvHandle(infiDirectUserHandle shdl, int index, int arr) {
-  // --- B --- | --- C --- | --- A ---
-  if(arr == SENDA) {
-    sHandles[num_chare_x + num_chare_y + index] = shdl;
-    CkDirect_assocLocalBuffer(&sHandles[num_chare_x + num_chare_y + index], &A[thisIndex.z*subBlockDimXz*blockDimY], sizeof(float)*subBlockDimXz*blockDimY);
-    countA++;
-  }
-
-  if(arr == SENDB) {
-    sHandles[index] = shdl;
-    CkDirect_assocLocalBuffer(&sHandles[index], &B[thisIndex.x*subBlockDimYx*blockDimZ], sizeof(float)*subBlockDimYx*blockDimZ);
-    countB++;
-  }
-
-  if(arr == SENDC) {
-    sHandles[num_chare_x + index] = shdl;
-    CkDirect_assocLocalBuffer(&sHandles[num_chare_x + index], &C[index*subBlockDimXy*blockDimZ], sizeof(float)*subBlockDimXy*blockDimZ);
-    countC++;
-  }
-
-  if(countA == num_chare_z-1 && countB == num_chare_x-1 && countC == num_chare_y-1) {
-    countA = 0;
-    countB = 0;
-    countC = 0;
-    contribute(0, 0, CkReduction::concat, CkCallback(CkIndex_Main::setupDone(), mainProxy));
-    // mainProxy.setupDone();
-  }
-}
-
-
-void Compute::callBackRcvdA(void *arg) {
-  Compute *obj = (Compute *)arg;
-  obj->countA++;
-  if(obj->countA == num_chare_z-1 && obj->countB == num_chare_x-1)
-    obj->thisProxy(obj->thisIndex.x, obj->thisIndex.y, obj->thisIndex.z).doWork();
-}
-
-void Compute::callBackRcvdB(void *arg) {
-  Compute *obj = (Compute *)arg;
-  obj->countB++;
-  if(obj->countA == num_chare_z-1 && obj->countB == num_chare_x-1)
-    obj->thisProxy(obj->thisIndex.x, obj->thisIndex.y, obj->thisIndex.z).doWork();
-}
-
-void Compute::callBackRcvdC(void *arg) {
-  Compute *obj = (Compute *)arg;
-  obj->countC++;
-  if(obj->countC == num_chare_y)
-    obj->thisProxy(obj->thisIndex.x, obj->thisIndex.y, obj->thisIndex.z).receiveC();
-}
-
-ComputeMap::ComputeMap(int x, int y, int z, int tx, int ty, int tz) {
-  X = x;
-  Y = y;
-  Z = z;
-  mapping = new int[X*Y*Z];
-
-  TopoManager tmgr;
-  int dimX, dimY, dimZ, dimT;
-
-#if USE_TOPOMAP
-  dimX = tmgr.getDimNX();
-  dimY = tmgr.getDimNY();
-  dimZ = tmgr.getDimNZ();
-  dimT = tmgr.getDimNT();
-#elif USE_BLOCKMAP
-  dimX = tx;
-  dimY = ty;
-  dimZ = tz;
-  dimT = 1;
-#endif
-
-  // we are assuming that the no. of chares in each dimension is a 
-  // multiple of the torus dimension
-  int numCharesPerPe = X*Y*Z/CkNumPes();
-
-  int numCharesPerPeX = X / dimX;
-  int numCharesPerPeY = Y / dimY;
-  int numCharesPerPeZ = Z / dimZ;
-
-  if(dimT < 2) {    // one core per node
-    if(CkMyPe()==0) CkPrintf("DATA: %d %d %d %d : %d %d %d\n", dimX, dimY, dimZ, dimT, numCharesPerPeX, numCharesPerPeY, numCharesPerPeZ);
-    for(int i=0; i<dimX; i++)
-      for(int j=0; j<dimY; j++)
-        for(int k=0; k<dimZ; k++)
-          for(int ci=i*numCharesPerPeX; ci<(i+1)*numCharesPerPeX; ci++)
-            for(int cj=j*numCharesPerPeY; cj<(j+1)*numCharesPerPeY; cj++)
-              for(int ck=k*numCharesPerPeZ; ck<(k+1)*numCharesPerPeZ; ck++) {
-#if USE_TOPOMAP
-                mapping[ci*Y*Z + cj*Z + ck] = tmgr.coordinatesToRank(i, j, k);
-#elif USE_BLOCKMAP
-                mapping[ci*Y*Z + cj*Z + ck] = i + j*dimX + k*dimX*dimY;
-#endif
-              }
-  } else {          // multiple cores per node
-    // In this case, we split the chares in the X dimension among the
-    // cores on the same node.
-    numCharesPerPeX /= dimT;
-      if(CkMyPe()==0) CkPrintf("%d %d %d : %d %d %d %d : %d %d %d \n", x, y, z, dimX, dimY, dimZ, dimT, numCharesPerPeX, numCharesPerPeY, numCharesPerPeZ);
-      for(int i=0; i<dimX; i++)
-        for(int j=0; j<dimY; j++)
-          for(int k=0; k<dimZ; k++)
-            for(int l=0; l<dimT; l++)
-              for(int ci=(dimT*i+l)*numCharesPerPeX; ci<(dimT*i+l+1)*numCharesPerPeX; ci++)
-                for(int cj=j*numCharesPerPeY; cj<(j+1)*numCharesPerPeY; cj++)
-                  for(int ck=k*numCharesPerPeZ; ck<(k+1)*numCharesPerPeZ; ck++) {
-                    mapping[ci*Y*Z + cj*Z + ck] = tmgr.coordinatesToRank(i, j, k, l);
-                  }
-  }
-}
-
-ComputeMap::~ComputeMap() {
-  delete [] mapping;
-}
-
-int ComputeMap::procNum(int, const CkArrayIndex &idx) {
-  int *index = (int *)idx.data();
-  return mapping[index[0]*Y*Z + index[1]*Z + index[2]];
-}
-
diff --git a/examples/charm++/ckdirect/matmul3d/matmul3d.ci b/examples/charm++/ckdirect/matmul3d/matmul3d.ci
deleted file mode 100644 (file)
index 7a5faf6..0000000
+++ /dev/null
@@ -1,48 +0,0 @@
-mainmodule matmul3d {
-  readonly CProxy_Main mainProxy;
-  readonly CProxy_Compute compute;
-  readonly int arrayDimX;
-  readonly int arrayDimY;
-  readonly int arrayDimZ;
-  readonly int blockDimX;
-  readonly int blockDimY;
-  readonly int blockDimZ;
-  readonly int torusDimX;
-  readonly int torusDimY;
-  readonly int torusDimZ;
-
-  readonly int num_chare_x;
-  readonly int num_chare_y;
-  readonly int num_chare_z;
-  readonly int num_chares;
-  readonly int subBlockDimXz;
-  readonly int subBlockDimYx;
-  readonly int subBlockDimXy;
-
-  mainchare Main {
-    entry Main(CkArgMsg *m);
-    entry void done();
-    entry [reductiontarget] void resetDone();
-    entry void setupDone();
-  };
-
-  array [3D] Compute {
-    entry Compute(void);
-    entry void beginCopying();
-    entry void resetArrays();
-    entry void receiveA(int indexZ, float data[size], int size);
-    entry void receiveB(int indexX, float data[size], int size);
-    entry void receiveC(float data[size], int size, int who);
-    entry void doWork();
-
-    entry void setupChannels();
-    entry void notifyReceiver(int pe, CkIndex3D index, int arr);
-    entry void recvHandle(infiDirectUserHandle shdl, int index, int arr);
-    entry void receiveC();
-  };
-
-  group ComputeMap : CkArrayMap {
-    entry ComputeMap(int, int, int, int, int, int);
-  };
-
-};
diff --git a/examples/charm++/ckdirect/matmul3d/matmul3d.h b/examples/charm++/ckdirect/matmul3d/matmul3d.h
deleted file mode 100644 (file)
index 1dda088..0000000
+++ /dev/null
@@ -1,117 +0,0 @@
-/** \file matmul3d.h
- *  Author: Abhinav S Bhatele
- *  Date Created: April 01st, 2008
- *
- */
-
-#include "ckdirect.h"
-
-// Read-only global variables
-
-/*readonly*/ CProxy_Main mainProxy;
-/*readonly*/ CProxy_Compute compute;
-/*readonly*/ int arrayDimX;
-/*readonly*/ int arrayDimY;
-/*readonly*/ int arrayDimZ;
-/*readonly*/ int blockDimX;
-/*readonly*/ int blockDimY;
-/*readonly*/ int blockDimZ;
-/*readonly*/ int torusDimX;
-/*readonly*/ int torusDimY;
-/*readonly*/ int torusDimZ;
-
-/*readonly*/ int num_chare_x;
-/*readonly*/ int num_chare_y;
-/*readonly*/ int num_chare_z;
-/*readonly*/ int num_chares;
-/*readonly*/ int subBlockDimXz;
-/*readonly*/ int subBlockDimYx;
-/*readonly*/ int subBlockDimXy;
-
-static unsigned long next = 1;
-
-int myrand(int numpes) {
-  next = next * 1103515245 + 12345;
-  return((unsigned)(next/65536) % numpes);
-}
-
-#define SENDA          1
-#define SENDB          2
-#define SENDC          3
-
-#define USE_TOPOMAP    0
-#define USE_BLOCKMAP   1
-
-#define USE_CKDIRECT   0
-
-#define OOB            -111111111111.0
-
-#define NUM_ITER       21
-
-#define MAX_LIMIT      9999999999.0
-
-double startTime;
-double setupTime;
-double firstTime;
-double endTime[NUM_ITER-1];
-
-/** \class Main
- *
- */
-class Main : public CBase_Main {
-  public:
-    int numIterations;
-
-    Main(CkArgMsg* m);
-    void done();
-    void resetDone();
-    void setupDone();
-};
-
-/** \class Compute
- *
- */
-class Compute: public CBase_Compute {
-  public:
-    float *A, *B, *C, *tmpC;
-    int countA, countB, countC;
-    infiDirectUserHandle *sHandles;
-    infiDirectUserHandle *rHandles;
-    
-    Compute();
-    Compute(CkMigrateMessage* m);
-    ~Compute();
-
-    void beginCopying();
-    void resetArrays();
-    void sendA();
-    void sendB();
-    void sendC();
-    void receiveA(int indexZ, float *data, int size);
-    void receiveB(int indexX, float *data, int size);
-    void receiveC(float *data, int size, int who);
-    void doWork();
-
-    void setupChannels();
-    void notifyReceiver(int pe, CkIndex3D index, int arr);
-    void recvHandle(infiDirectUserHandle shdl, int index, int arr);
-    void receiveC();
-    static void callBackRcvdA(void *arg);
-    static void callBackRcvdB(void *arg);
-    static void callBackRcvdC(void *arg);
-};
-
-/** \class ComputeMap
- *
- */
-class ComputeMap: public CBase_ComputeMap {
-  public:
-    int X, Y, Z;
-    int *mapping;
-
-    ComputeMap(int x, int y, int z, int tx, int ty, int tz);
-    ~ComputeMap();
-    int procNum(int, const CkArrayIndex &idx);
-};
-
-#include "matmul3d.def.h"
diff --git a/examples/charm++/ckdirect/stencil3d/Makefile b/examples/charm++/ckdirect/stencil3d/Makefile
deleted file mode 100644 (file)
index 3f6afb4..0000000
+++ /dev/null
@@ -1,31 +0,0 @@
--include ../../../common.mk
-OPTS   = -O3
-CHARMC = ../../../../bin/charmc $(OPTS)
-SRC    = ../../../../src
-
-OBJS = stencil3d.o
-
-all: stencil3d-ckdirect stencil3d-msg
-
-stencil3d-ckdirect: stencil3d-ckdirect.o
-       $(CHARMC) -language charm++ -o stencil3d-ckdirect stencil3d-ckdirect.o
-
-stencil3d-msg: stencil3d-msg.o
-       $(CHARMC) -language charm++ -o stencil3d-msg stencil3d-msg.o
-
-stencil3d.decl.h: stencil3d.ci
-       $(CHARMC)  stencil3d.ci
-
-clean:
-       rm -f *.decl.h *.def.h conv-host *.o stencil3d charmrun stencil3d-ckdirect stencil3d-msg
-
-stencil3d-ckdirect.o: stencil3d.C stencil3d.decl.h
-       $(CHARMC) -o stencil3d-ckdirect.o -DUSE_CKDIRECT stencil3d.C 
-
-stencil3d-msg.o: stencil3d.C stencil3d.decl.h
-       $(CHARMC) -o stencil3d-msg.o -DUSE_MESSAGES stencil3d.C 
-
-# these test values are for sanity checking.  Wider and larger favors RDMA
-test: all
-       $(call run, ./stencil3d-msg +p4 20 10 100 2 2 2 10 )
-       $(call run, ./stencil3d-ckdirect +p4 20 10 100 2 2 2 10 )
diff --git a/examples/charm++/ckdirect/stencil3d/stencil3d.C b/examples/charm++/ckdirect/stencil3d/stencil3d.C
deleted file mode 100644 (file)
index f912064..0000000
+++ /dev/null
@@ -1,1313 +0,0 @@
-#include "ckdirect.h" 
-
-#if defined ARR_CHECK && !defined CMK_PARANOID
-#define CMK_PARANOID
-#endif
-
-#include "stencil3d.decl.h"
-#define OOB 9999999999.0
-#define NBRS 6
-
-CProxy_Main mainProxy;
-
-int dimx, dimy, dimz;
-int charesx, charesy, charesz;
-int num_iterations;
-
-class Main : public CBase_Main {
-  
-  double startSetup, start, end;
-  public:
-
-  Main(CkArgMsg *msg){
-    mainProxy = thisProxy;
-
-    if(msg->argc == 8){
-      dimx = atoi(msg->argv[1]);
-      dimy = atoi(msg->argv[2]);
-      dimz = atoi(msg->argv[3]);
-      charesx = atoi(msg->argv[4]);
-      charesy = atoi(msg->argv[5]);
-      charesz = atoi(msg->argv[6]);
-      num_iterations = atoi(msg->argv[7]);
-    }
-    else{
-      CkAbort("arguments: dimx dimy dimz charesx charesy charesz iter\n");
-    }
-    
-    delete msg;
-    CProxy_StencilPoint array = 
-        CProxy_StencilPoint::ckNew(charesx*charesy*charesz);
-    CkPrintf("main: dim: %d,%d,%d charedim: %d,%d,%d num_iterations: %d\n", dimx, dimy, dimz, charesx, charesy, charesz, num_iterations);
-    startSetup = CkWallTimer();
-#ifdef USE_CKDIRECT
-    array.setupChannels();
-#else
-    array.sendData();
-#endif
-  }
-
-  void doneSetup(){
-    start = CkWallTimer();
-  }
-  
-  void done(CkReductionMsg *msg){
-    end = CkWallTimer();
-    CkPrintf("Total time: %f sec\n", end-startSetup);
-    CkPrintf("Computation time per iteration: %f sec\n", (end-start)/(num_iterations-1));
-    CkPrintf("Total computations: %f\n", *(double *)msg->getData());
-    CkExit();
-  }
-};
-
-class StencilMsg : public CMessage_StencilMsg {
-  public:
-  float *arr;
-  int size;
-  int which;
-};
-
-class StencilPoint : public CBase_StencilPoint{
-  // this chare's place in the array of chares
-  int row, col, plane;
-  int whichLocal;
-
-  // this chare has a domain with dimensions 
-  int rows, cols, planes;
-  int payload[3];
-  
-  int iterations;
-  
-    /* Number of elements whose new values have been computed
-     */
-    double eltsComp;
-//#ifdef USE_CKDIRECT
-  /* 0: left, 1: right, 2: top, 3: bottom*/
-  infiDirectUserHandle shandles[2][NBRS];
-  infiDirectUserHandle rhandles[NBRS];
-
-//#endif
-#ifdef ARR_CHECK
-#ifdef USE_MESSAGES
-  float *recvBuf[NBRS];
-#else
-  CkVec<float> recvBuf[NBRS];
-#endif
-  CkVec<float> sendBuf[2][NBRS];
-  CkVec<float> localChunk[2];
-#else
-  /* send buffers */
-  float *sendBuf[2][NBRS]; /* rectangle */
-  /* receive buffers */
-  float *recvBuf[NBRS]; /* rectangle */
-  float *localChunk[2]; /* cuboid */
-#endif
-#ifdef USE_MESSAGES
-  StencilMsg *recvMsgs[NBRS];
-#endif
-
-  int remainingBufs;
-  int remainingChannels;
-  
-  public:
-  StencilPoint(CkMigrateMessage *){}
-  StencilPoint(){
-    remainingBufs = NBRS;
-    remainingChannels = NBRS;
-    whichLocal= 0;
-
-    iterations = 0;
-    eltsComp = 0;
-    
-    plane = thisIndex/(charesx*charesy);
-    row = (thisIndex/charesx)%charesy;
-    col = thisIndex%charesx;
-    
-    rows = dimy/charesy;
-    cols = dimx/charesx;
-    planes = dimz/charesz;
-
-    payload[0] = rows*planes; // x
-    payload[1] = cols*planes; // y
-    payload[2] = rows*cols;   // z
-    
-    if(thisIndex == 0){
-      CkPrintf("rows: %d cols: %d planes: %d\n", rows, cols, planes);
-    }
-    // allocate memory
-#ifdef ARR_CHECK
-    sendBuf[0][0].resize(payload[0]);
-    sendBuf[1][0].resize(payload[0]);
-    sendBuf[0][1].resize(payload[0]);
-    sendBuf[1][1].resize(payload[0]);
-
-    sendBuf[0][2].resize(payload[1]);
-    sendBuf[1][2].resize(payload[1]);
-    sendBuf[0][3].resize(payload[1]);
-    sendBuf[1][3].resize(payload[1]);
-    
-    sendBuf[0][4].resize(payload[2]);
-    sendBuf[1][4].resize(payload[2]);
-    sendBuf[0][5].resize(payload[2]);
-    sendBuf[1][5].resize(payload[2]);
-#else
-    sendBuf[0][0] = new float [payload[0]];
-    sendBuf[1][0] = new float [payload[0]];
-    sendBuf[0][1] = new float [payload[0]];
-    sendBuf[1][1] = new float [payload[0]];
-
-    sendBuf[0][2] = new float [payload[1]];
-    sendBuf[1][2] = new float [payload[1]];
-    sendBuf[0][3] = new float [payload[1]];
-    sendBuf[1][3] = new float [payload[1]];
-
-    sendBuf[0][4] = new float [payload[2]];
-    sendBuf[1][4] = new float [payload[2]];
-    sendBuf[0][5] = new float [payload[2]];
-    sendBuf[1][5] = new float [payload[2]];
-#endif
-
-#ifndef USE_MESSAGES
-#ifdef ARR_CHECK
-    recvBuf[0].resize(payload[0]);
-    recvBuf[1].resize(payload[0]);
-    recvBuf[2].resize(payload[1]);
-    recvBuf[3].resize(payload[1]);
-    recvBuf[4].resize(payload[2]);
-    recvBuf[5].resize(payload[2]);
-#else
-    recvBuf[0] = new float[payload[0]];
-    recvBuf[1] = new float[payload[0]];
-    recvBuf[2] = new float[payload[1]];
-    recvBuf[3] = new float[payload[1]];
-    recvBuf[4] = new float[payload[2]];
-    recvBuf[5] = new float[payload[2]];
-#endif
-#else
-    for(int i = 0; i < NBRS; i++)
-      recvBuf[i] = 0;
-#endif
-    
-#ifdef ARR_CHECK
-    localChunk[0].resize((rows-2)*(cols-2)*(planes-2));
-    localChunk[1].resize((rows-2)*(cols-2)*(planes-2));
-#else
-    localChunk[0] = new float[(rows-2)*(cols-2)*(planes-2)];
-    localChunk[1] = new float[(rows-2)*(cols-2)*(planes-2)];
-#endif
-    // initialize
-    if(plane == 0){// face is held at 0
-      for(int i = 0; i < rows*cols; i++)
-        // first index says which version of the double buffer is to be used
-        // second index, the face in question
-        sendBuf[0][5][i] = 1.0;
-    }
-    else{
-      for(int i = 0; i < rows*cols; i++)
-        // first index says which version of the double buffer is to be used
-        // second index, the face in question
-        sendBuf[0][5][i] = 0.0;
-    }
-
-    for(int i = 0; i < rows*cols; i++)
-      sendBuf[0][4][i] = 0.0;
-      
-    for(int i = 0; i < rows*planes; i++)
-      sendBuf[0][1][i] = 0.0;
-      
-    for(int i = 0; i < rows*planes; i++)
-      sendBuf[0][0][i] = 0.0;
-      
-    for(int i = 0; i < cols*planes; i++)
-      sendBuf[0][3][i] = 0.0;
-      
-    for(int i = 0; i < cols*planes; i++)
-      sendBuf[0][2][i] = 0.0;
-      
-    
-    // now the rest of the cube
-    for(int j = 0; j < (rows-2)*(cols-2)*(planes-2); j++){
-      localChunk[0][j] = 0.0; 
-    }
-  }
-  
-  ~StencilPoint(){
-    for(int i = 0; i < NBRS; i++){
-#ifdef ARR_CHECK
-      sendBuf[0][i].free();
-      sendBuf[1][i].free();
-#else
-      delete [] sendBuf[0][i];
-      delete [] sendBuf[1][i];
-#endif
-#ifndef USE_MESSAGES
-#ifdef ARR_CHECK
-      recvBuf[i].free();
-#else
-      delete [] recvBuf[i];
-#endif
-#endif
-    }
-
-#ifdef ARR_CHECK
-    localChunk[0].free();
-    localChunk[1].free();
-#else
-    delete [] localChunk[0];
-    delete [] localChunk[1];
-#endif
-  }
-
-#define lin(c,r,p) ((p)*charesx*charesy+(r)*charesx+(c))
-//#ifdef USE_CKDIRECT
-  void setupChannels(){
-    int node = CkMyNode();
-#ifdef STENCIL2D_VERBOSE
-    /*
-    CkPrintf("(%d,%d): notify (%d,%d):l, (%d,%d):r, (%d,%d):t, (%d,%d):b\n", 
-              row, col,
-              row, (col+1)%num_cols,
-              row, (col-1+num_cols)%num_cols,
-              (row+1)%num_rows, col,
-              (row-1+num_rows)%num_rows, col);
-    */
-#endif
-
-    thisProxy[lin((col+1)%charesx, (row), (plane))].notify(node,0,thisIndex);
-    thisProxy[lin((col-1+charesx)%charesx, (row), (plane))].notify(node,1,thisIndex);
-    thisProxy[lin((col), (row+1)%charesy, (plane))].notify(node,2,thisIndex);
-    thisProxy[lin((col), (row-1+charesy)%charesy, (plane))].notify(node,3,thisIndex);
-    thisProxy[lin((col), (row), (plane+1)%charesz)].notify(node,4,thisIndex);
-    thisProxy[lin((col), (row), (plane-1+charesz)%charesz)].notify(node,5,thisIndex);
-  }
-
-  void notify(int node, int which, CkIndex1D whoSent){
-    // make handle
-#ifdef STENCIL2D_VERBOSE
-    CkPrintf("(%d,%d,%d): (%d) is %d to me\n", row, col, plane, whoSent, which);
-#endif
-#if defined ARR_CHECK && !defined USE_MESSAGES
-    rhandles[which] = CkDirect_createHandle(node, recvBuf[which].getVec(), payload[which/2]*sizeof(float), StencilPoint::callbackWrapper, (void *)this, OOB);
-#else
-    rhandles[which] = CkDirect_createHandle(node, recvBuf[which], payload[which/2]*sizeof(float), StencilPoint::callbackWrapper, (void *)this, OOB);
-#endif
-    thisProxy[whoSent].recvHandle(rhandles[which], which);
-  }
-
-  void recvHandle(infiDirectUserHandle handle, int which){
-    // this causes an inversion of the buffer indices:
-    // left send buffer is 1, right 0, top 3, bottom 2
-    shandles[0][which] = handle;
-    shandles[1][which] = handle;
-    
-#ifdef ARR_CHECK
-    CkDirect_assocLocalBuffer(&shandles[0][which], sendBuf[0][which].getVec(), payload[which/2]*sizeof(float));
-    CkDirect_assocLocalBuffer(&shandles[1][which], sendBuf[1][which].getVec(), payload[which/2]*sizeof(float));
-#else
-    CkDirect_assocLocalBuffer(&shandles[0][which], sendBuf[0][which], payload[which/2]*sizeof(float));
-    CkDirect_assocLocalBuffer(&shandles[1][which], sendBuf[1][which], payload[which/2]*sizeof(float));
-#endif
-    remainingChannels--;
-    if(remainingChannels == 0){
-      // start 
-#ifdef STENCIL2D_VERBOSE
-      //CkPrintf("(%d,%d,%d): recvd all handles, start put\n", x,y,z);
-#endif
-      sendData();
-    }
-  }
-
-  void recvBuffer(){
-    remainingBufs--;
-#ifdef STENCIL2D_VERBOSE
-    //CkPrintf("(%d,%d,%d): remainingBufs: %d\n", x,y,z,remainingBufs);
-#endif
-    if(remainingBufs == 0){
-#ifdef STENCIL2D_VERBOSE
-      //CkPrintf("(%d,%d,%d): recvd all buffers, start compute(%d)\n", x,y,z, iterations);
-#endif
-      remainingBufs = NBRS;
-      thisProxy[thisIndex].compute();
-    }
-  }
-  
-  static void callbackWrapper(void *arg){
-    StencilPoint *obj = (StencilPoint *)arg;
-    obj->recvBuffer();
-  }
-//#else   // don't USE_CKDIRECT
-  void recvBuffer(float *array, int size, int whichBuf){
-    remainingBufs--;
-#if defined ARR_CHECK && !defined USE_MESSAGES
-    memcpy(recvBuf[whichBuf].getVec(), array, size*sizeof(float));
-#else
-    memcpy(recvBuf[whichBuf], array, size*sizeof(float));
-#endif
-    /*
-    for(int i = 0; i < size; i++){
-      recvBuf[whichBuf][i] = array[i];
-    }
-    */
-    
-    if(remainingBufs == 0){
-      remainingBufs = NBRS;
-      compute();
-    }
-  }
-
-  void recvBufferMsg(StencilMsg *msg){
-#ifdef USE_MESSAGES
-    remainingBufs--;
-    recvBuf[msg->which] = msg->arr;
-    recvMsgs[msg->which] = msg;
-    
-    if(remainingBufs == 0){
-      remainingBufs = NBRS;
-      compute();
-      for(int i = 0; i < NBRS; i++){
-        delete recvMsgs[i];
-        recvMsgs[i] = 0;
-      }
-    }
-#else
-      CkAbort("Messages not used, don't call\n");
-#endif
-      
-  }
-//#endif // end ifdef USE_CKDIRECT
-  
-
-// to access localChunk interior (i.e. interior of interior) 
-#define small(r,c,p) (p)*(rows-2)*(cols-2)+(r)*(cols-2)+(c)
-// to access sendBufs 
-#define facex(r,c) (r)*planes+(c)
-#define facey(r,c) (r)*planes+(c)
-#define facez(r,c) (r)*cols+(c)
-// indices into sendBufs, after inversion
-#define xp 1
-#define xn 0
-#define yp 3
-#define yn 2
-#define zp 5
-#define zn 4
-
-// indices into recvBufs, without inversion
-#define XP 0
-#define XN 1
-#define YP 2
-#define YN 3
-#define ZP 4
-#define ZN 5
-
-  void compute(){
-    // 1. do actual work
-    int newLocal= 1-whichLocal;
-    
-    // interior of interior: uses only localChunk
-    for(int k = 1; k < planes-3; k++){
-      for(int i = 1; i < rows-3; i++){
-        for(int j = 1; j < cols-3; j++){
-          localChunk[newLocal][small(i,j,k)] = (
-                              localChunk[whichLocal][small(i,j,k)]+
-                              localChunk[whichLocal][small(i+1,j,k)]+
-                              localChunk[whichLocal][small(i-1,j,k)]+
-                              localChunk[whichLocal][small(i,j+1,k)]+
-                              localChunk[whichLocal][small(i,j-1,k)]+
-                              localChunk[whichLocal][small(i,j,k+1)]+
-                              localChunk[whichLocal][small(i,j,k-1)]
-                            )/7;
-          eltsComp+=1;
-        }
-      }
-    }
-
-    // 8 corners of interior: uses localChunk and sendBuf's
-    //0.
-    localChunk[newLocal][small(0,0,0)] = (
-                            localChunk[whichLocal][small(0,0,0)]+
-                            localChunk[whichLocal][small(1,0,0)]+
-                            localChunk[whichLocal][small(0,1,0)]+
-                            localChunk[whichLocal][small(0,0,1)]+
-                            sendBuf[whichLocal][zp][facez(1,1)]+
-                            sendBuf[whichLocal][xp][facex(1,1)]+
-                            sendBuf[whichLocal][yp][facey(1,1)]
-                            )/7;
-
-    //1.
-    localChunk[newLocal][small(0,cols-3,0)] = (
-                            localChunk[whichLocal][small(0,cols-3,0)]+
-                            localChunk[whichLocal][small(0,cols-4,0)]+
-                            localChunk[whichLocal][small(1,cols-3,0)]+
-                            localChunk[whichLocal][small(0,cols-3,1)]+
-                            sendBuf[whichLocal][zp][facez(1,cols-2)]+
-                            sendBuf[whichLocal][yp][facey(cols-2,1)]+
-                            sendBuf[whichLocal][xn][facex(1,1)]
-                            )/7;
-
-    //2.
-    localChunk[newLocal][small(rows-3,0,0)] = (
-                            localChunk[whichLocal][small(rows-3,0,0)]+
-                            localChunk[whichLocal][small(rows-4,0,0)]+
-                            localChunk[whichLocal][small(rows-3,1,0)]+
-                            localChunk[whichLocal][small(rows-3,0,1)]+
-                            sendBuf[whichLocal][zp][facez(rows-2,1)]+
-                            sendBuf[whichLocal][yn][facey(1,1)]+
-                            sendBuf[whichLocal][xp][facex(rows-2,1)]
-                            )/7;
-
-    //3.
-    localChunk[newLocal][small(rows-3,cols-3,0)] = (
-                            localChunk[whichLocal][small(rows-3,cols-3,0)]+
-                            localChunk[whichLocal][small(rows-3,cols-4,0)]+
-                            localChunk[whichLocal][small(rows-4,cols-3,0)]+
-                            localChunk[whichLocal][small(rows-3,cols-3,1)]+
-                            sendBuf[whichLocal][zp][facez(rows-2,cols-2)]+
-                            sendBuf[whichLocal][yn][facey(cols-2,1)]+
-                            sendBuf[whichLocal][xn][facex(rows-2,1)]
-                            )/7;
-    
-    //4.
-    localChunk[newLocal][small(0,0,planes-3)] = (
-                            localChunk[whichLocal][small(0,0,planes-3)]+
-                            localChunk[whichLocal][small(0,0,planes-4)]+
-                            localChunk[whichLocal][small(0,1,planes-3)]+
-                            localChunk[whichLocal][small(1,0,planes-3)]+
-                            sendBuf[whichLocal][zn][facez(1,1)]+
-                            sendBuf[whichLocal][yp][facey(1,planes-2)]+
-                            sendBuf[whichLocal][xp][facex(1,planes-2)]
-                            )/7;
-
-    //5.
-    localChunk[newLocal][small(0,cols-3,planes-3)] = (
-                            localChunk[whichLocal][small(0,cols-3,planes-3)]+
-                            localChunk[whichLocal][small(0,cols-4,planes-3)]+
-                            localChunk[whichLocal][small(1,cols-3,planes-3)]+
-                            localChunk[whichLocal][small(0,cols-3,planes-4)]+
-                            sendBuf[whichLocal][zn][facez(1,cols-2)]+
-                            sendBuf[whichLocal][yp][facey(cols-2,planes-2)]+
-                            sendBuf[whichLocal][xn][facex(1,planes-2)]
-                            )/7;
-    
-    //6.
-    localChunk[newLocal][small(rows-3,0,planes-3)] = (
-                            localChunk[whichLocal][small(rows-3,0,planes-3)]+
-                            localChunk[whichLocal][small(rows-4,0,planes-3)]+
-                            localChunk[whichLocal][small(rows-3,1,planes-3)]+
-                            localChunk[whichLocal][small(rows-3,0,planes-4)]+
-                            sendBuf[whichLocal][zn][facez(rows-2,1)]+
-                            sendBuf[whichLocal][yn][facey(1,planes-2)]+
-                            sendBuf[whichLocal][xp][facex(rows-2,planes-2)]
-                            )/7;
-
-    //7.
-    localChunk[newLocal][small(rows-3,cols-3,planes-3)] = (
-                            localChunk[whichLocal][small(rows-3,cols-3,planes-3)]+
-                            localChunk[whichLocal][small(rows-4,cols-3,planes-3)]+
-                            localChunk[whichLocal][small(rows-3,cols-4,planes-3)]+
-                            localChunk[whichLocal][small(rows-3,cols-3,planes-4)]+
-                            sendBuf[whichLocal][zn][facez(rows-2,cols-2)]+
-                            sendBuf[whichLocal][yn][facey(cols-2,planes-2)]+
-                            sendBuf[whichLocal][xn][facex(rows-2,planes-2)]
-                            )/7;
-
-   eltsComp += 8;
-
-    // 12 edges
-    //zp,yp
-    for(int i = 1; i < cols-3; i++){
-      localChunk[newLocal][small(0,i,0)] = (
-                localChunk[whichLocal][small(0,i,0)]+
-                localChunk[whichLocal][small(0,i-1,0)]+
-                localChunk[whichLocal][small(0,i+1,0)]+
-                localChunk[whichLocal][small(1,i,0)]+
-                localChunk[whichLocal][small(0,i,1)]+
-                sendBuf[whichLocal][yp][facey(i+1,1)]+
-                sendBuf[whichLocal][zp][facez(1,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xn,yp
-    for(int i = 1; i < planes-3; i++){
-      localChunk[newLocal][small(0,cols-3,i)] = (
-                localChunk[whichLocal][small(0,cols-3,i)]+
-                localChunk[whichLocal][small(0,cols-3,i-1)]+
-                localChunk[whichLocal][small(0,cols-3,i+1)]+
-                localChunk[whichLocal][small(1,cols-3,i)]+
-                localChunk[whichLocal][small(0,cols-4,i)]+
-                sendBuf[whichLocal][xn][facex(1,i+1)]+
-                sendBuf[whichLocal][yp][facey(cols-2,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //zn,yp
-    for(int i = 1; i < cols-3; i++){
-      localChunk[newLocal][small(0,i,planes-3)] = (
-                localChunk[whichLocal][small(0,i,planes-3)]+
-                localChunk[whichLocal][small(0,i-1,planes-3)]+
-                localChunk[whichLocal][small(0,i+1,planes-3)]+
-                localChunk[whichLocal][small(0,i,planes-4)]+
-                localChunk[whichLocal][small(1,i,planes-3)]+
-                sendBuf[whichLocal][zn][facez(1,i+1)]+
-                sendBuf[whichLocal][yp][facey(i+1,planes-2)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //yp,xp
-    for(int i = 1; i < planes-3; i++){
-      localChunk[newLocal][small(0,0,i)] = (
-                localChunk[whichLocal][small(0,0,i)]+
-                localChunk[whichLocal][small(0,0,i-1)]+
-                localChunk[whichLocal][small(0,0,i+1)]+
-                localChunk[whichLocal][small(0,1,i)]+
-                localChunk[whichLocal][small(1,0,i)]+
-                sendBuf[whichLocal][yp][facey(1,i+1)]+
-                sendBuf[whichLocal][xp][facex(1,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //yn,zp
-    for(int i = 1; i < cols-3; i++){
-      localChunk[newLocal][small(rows-3,i,0)] = (
-                localChunk[whichLocal][small(rows-3,i,0)]+
-                localChunk[whichLocal][small(rows-3,i-1,0)]+
-                localChunk[whichLocal][small(rows-3,i+1,0)]+
-                localChunk[whichLocal][small(rows-4,i,0)]+
-                localChunk[whichLocal][small(rows-3,i,1)]+
-                sendBuf[whichLocal][yn][facey(i+1,1)]+
-                sendBuf[whichLocal][zp][facez(rows-2,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xn,yn
-    for(int i = 1; i < planes-3; i++){
-      localChunk[newLocal][small(rows-3,cols-3,i)] = (
-                localChunk[whichLocal][small(rows-3,cols-3,i)]+
-                localChunk[whichLocal][small(rows-3,cols-3,i-1)]+
-                localChunk[whichLocal][small(rows-3,cols-3,i+1)]+
-                localChunk[whichLocal][small(rows-4,cols-3,i)]+
-                localChunk[whichLocal][small(rows-3,cols-4,i)]+
-                sendBuf[whichLocal][xn][facex(rows-2,i+1)]+
-                sendBuf[whichLocal][yn][facey(cols-2,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //yn,zn
-    for(int i = 1; i < cols-3; i++){
-      localChunk[newLocal][small(rows-3,i,planes-3)] = (
-                localChunk[whichLocal][small(rows-3,i,planes-3)]+
-                localChunk[whichLocal][small(rows-3,i-1,planes-3)]+
-                localChunk[whichLocal][small(rows-3,i+1,planes-3)]+
-                localChunk[whichLocal][small(rows-4,i,planes-3)]+
-                localChunk[whichLocal][small(rows-3,i,planes-4)]+
-                sendBuf[whichLocal][yn][facey(i+1,planes-2)]+
-                sendBuf[whichLocal][zn][facez(rows-2,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xp,yn
-    for(int i = 1; i < planes-3; i++){
-      localChunk[newLocal][small(rows-3,0,i)] = (
-                localChunk[whichLocal][small(rows-3,0,i)]+
-                localChunk[whichLocal][small(rows-3,0,i-1)]+
-                localChunk[whichLocal][small(rows-3,0,i+1)]+
-                localChunk[whichLocal][small(rows-3,1,i)]+
-                localChunk[whichLocal][small(rows-4,0,i)]+
-                sendBuf[whichLocal][xp][facex(rows-2,i+1)]+
-                sendBuf[whichLocal][yn][facey(1,i+1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xp,zp
-    for(int i = 1; i < rows-3; i++){
-      localChunk[newLocal][small(i,0,0)] = (
-                localChunk[whichLocal][small(i,0,0)]+
-                localChunk[whichLocal][small(i-1,0,0)]+
-                localChunk[whichLocal][small(i+1,0,0)]+
-                localChunk[whichLocal][small(i,1,0)]+
-                localChunk[whichLocal][small(i,0,1)]+
-                sendBuf[whichLocal][xp][facex(i+1,1)]+
-                sendBuf[whichLocal][zp][facez(i+1,1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xn,zp
-    for(int i = 1; i < rows-3; i++){
-      localChunk[newLocal][small(i,cols-3,0)] = (
-                localChunk[whichLocal][small(i,cols-3,0)]+
-                localChunk[whichLocal][small(i-1,cols-3,0)]+
-                localChunk[whichLocal][small(i+1,cols-3,0)]+
-                localChunk[whichLocal][small(i,cols-4,0)]+
-                localChunk[whichLocal][small(i,cols-3,1)]+
-                sendBuf[whichLocal][xn][facex(i+1,1)]+
-                sendBuf[whichLocal][zp][facez(i+1,cols-2)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xp,zn
-    for(int i = 1; i < rows-3; i++){
-      localChunk[newLocal][small(i,0,planes-3)] = (
-                localChunk[whichLocal][small(i,0,planes-3)]+
-                localChunk[whichLocal][small(i-1,0,planes-3)]+
-                localChunk[whichLocal][small(i+1,0,planes-3)]+
-                localChunk[whichLocal][small(i,1,planes-3)]+
-                localChunk[whichLocal][small(i,0,planes-4)]+
-                sendBuf[whichLocal][xp][facex(i+1,planes-2)]+
-                sendBuf[whichLocal][zn][facez(i+1,1)]
-          )/7;
-       eltsComp+=1;
-    }
-
-    //xn,zn
-    for(int i = 1; i < rows-3; i++){
-      localChunk[newLocal][small(i,cols-3,planes-3)] = (
-                localChunk[whichLocal][small(i,cols-3,planes-3)]+
-                localChunk[whichLocal][small(i-1,cols-3,planes-3)]+
-                localChunk[whichLocal][small(i+1,cols-3,planes-3)]+
-                localChunk[whichLocal][small(i,cols-4,planes-3)]+
-                localChunk[whichLocal][small(i,cols-3,planes-4)]+
-                sendBuf[whichLocal][xn][facex(i+1,planes-2)]+
-                sendBuf[whichLocal][zn][facez(i+1,cols-2)]
-          )/7;
-       eltsComp+=1;
-
-    }
-
-    // 6 more faces - use 6 (including self) from localChunk and 1 from one of the sendBufs
-    for(int i = 1; i < rows-3; i++){
-      for(int j = 1; j < cols-3; j++){
-        localChunk[newLocal][small(i,j,0)] = (
-            localChunk[whichLocal][small(i,j,0)]+
-            localChunk[whichLocal][small(i-1,j,0)]+
-            localChunk[whichLocal][small(i+1,j,0)]+
-            localChunk[whichLocal][small(i,j-1,0)]+
-            localChunk[whichLocal][small(i,j+1,0)]+
-            localChunk[whichLocal][small(i,j,1)]+
-            sendBuf[whichLocal][zp][facez(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    for(int i = 1; i < rows-3; i++){
-      for(int j = 1; j < cols-3; j++){
-        localChunk[newLocal][small(i,j,planes-3)] = (
-            localChunk[whichLocal][small(i,j,planes-3)]+
-            localChunk[whichLocal][small(i-1,j,planes-3)]+
-            localChunk[whichLocal][small(i+1,j,planes-3)]+
-            localChunk[whichLocal][small(i,j-1,planes-3)]+
-            localChunk[whichLocal][small(i,j+1,planes-3)]+
-            localChunk[whichLocal][small(i,j,planes-4)]+
-            sendBuf[whichLocal][zn][facez(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    
-    for(int j = 1; j < planes-3; j++){
-      for(int i = 1; i < rows-3; i++){
-        localChunk[newLocal][small(i,0,j)] = (
-            localChunk[whichLocal][small(i,0,j)]+
-            localChunk[whichLocal][small(i-1,0,j)]+
-            localChunk[whichLocal][small(i+1,0,j)]+
-            localChunk[whichLocal][small(i,0,j-1)]+
-            localChunk[whichLocal][small(i,0,j+1)]+
-            localChunk[whichLocal][small(i,1,j)]+
-            sendBuf[whichLocal][xp][facex(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    for(int j = 1; j < planes-3; j++){
-      for(int i = 1; i < rows-3; i++){
-        localChunk[newLocal][small(i,cols-3,j)] = (
-            localChunk[whichLocal][small(i,cols-3,j)]+
-            localChunk[whichLocal][small(i-1,cols-3,j)]+
-            localChunk[whichLocal][small(i+1,cols-3,j)]+
-            localChunk[whichLocal][small(i,cols-3,j-1)]+
-            localChunk[whichLocal][small(i,cols-3,j+1)]+
-            localChunk[whichLocal][small(i,cols-4,j)]+
-            sendBuf[whichLocal][xn][facex(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    
-    for(int j = 1; j < planes-3; j++){
-      for(int i = 1; i < cols-3; i++){
-        localChunk[newLocal][small(0,i,j)] = (
-            localChunk[whichLocal][small(0,i,j)]+
-            localChunk[whichLocal][small(0,i-1,j)]+
-            localChunk[whichLocal][small(0,i+1,j)]+
-            localChunk[whichLocal][small(0,i,j-1)]+
-            localChunk[whichLocal][small(0,i,j+1)]+
-            localChunk[whichLocal][small(1,i,j)]+
-            sendBuf[whichLocal][yp][facey(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    for(int j = 1; j < planes-3; j++){
-      for(int i = 1; i < cols-3; i++){
-        localChunk[newLocal][small(rows-3,i,j)] = (
-            localChunk[whichLocal][small(rows-3,i,j)]+
-            localChunk[whichLocal][small(rows-3,i-1,j)]+
-            localChunk[whichLocal][small(rows-3,i+1,j)]+
-            localChunk[whichLocal][small(rows-3,i,j-1)]+
-            localChunk[whichLocal][small(rows-3,i,j+1)]+
-            localChunk[whichLocal][small(rows-4,i,j)]+
-            sendBuf[whichLocal][yn][facey(i+1,j+1)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-    /* Now we can compute the sendBufs. there are 6 faces.
-     Each element of sendbuf uses the ghost layer (recvbufs) in some way.
-     There are elements that :
-     a) form the interior of the faces. these use only one ghost element each
-     b) are the corners of the faces, that use 3 ghost elements each
-     c) are part of the edges, using 2 ghost elements each
-    */
-
-    // 1. zp face
-    // Interior points first
-    // enforce boundary conditions if plane = 0
-    if(plane == 0){
-      for(int i = 0; i < rows*cols; i++){
-        sendBuf[newLocal][zp][i] = 1.0;
-        eltsComp+=1;
-      }
-    }
-    else{
-    for(int i = 1; i < rows-1; i++){
-      for(int j = 1; j < cols-1; j++){
-        sendBuf[newLocal][zp][facez(i,j)] = (
-                sendBuf[whichLocal][zp][facez(i,j)]+
-                sendBuf[whichLocal][zp][facez(i-1,j)]+
-                sendBuf[whichLocal][zp][facez(i+1,j)]+
-                sendBuf[whichLocal][zp][facez(i,j-1)]+
-                sendBuf[whichLocal][zp][facez(i,j+1)]+
-                localChunk[whichLocal][small(i-1,j-1,0)]+
-                recvBuf[ZP][facez(i,j)]
-        )/7;
-       eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    // Each element uses 3 ghosts, 3 from its own sendBuf (including itself)  and 1 from a different sendBuf
-    sendBuf[newLocal][zp][facez(0,0)] = (
-          recvBuf[YP][facey(0,0)]+
-          recvBuf[XP][facex(0,0)]+
-          recvBuf[ZP][facez(0,0)]+
-          sendBuf[whichLocal][zp][facez(0,0)]+
-          sendBuf[whichLocal][zp][facez(0,1)]+
-          sendBuf[whichLocal][zp][facez(1,0)]+
-          sendBuf[whichLocal][xp][facex(0,1)]
-    )/7;
-
-    sendBuf[newLocal][zp][facez(0,cols-1)] = (
-          recvBuf[YP][facey(cols-1,0)]+
-          recvBuf[XN][facex(0,0)]+
-          recvBuf[ZP][facez(0,cols-1)]+
-          sendBuf[whichLocal][zp][facez(0,cols-1)]+
-          sendBuf[whichLocal][zp][facez(1,cols-1)]+
-          sendBuf[whichLocal][zp][facez(0,cols-2)]+
-          sendBuf[whichLocal][xn][facex(0,1)]
-    )/7;
-
-    sendBuf[newLocal][zp][facez(rows-1,0)] = (
-          recvBuf[ZP][facez(rows-1,0)]+
-          recvBuf[XP][facex(rows-1,0)]+
-          recvBuf[YN][facey(0,0)]+
-          sendBuf[whichLocal][zp][facez(rows-1,0)]+
-          sendBuf[whichLocal][zp][facez(rows-2,0)]+
-          sendBuf[whichLocal][zp][facez(rows-1,1)]+
-          sendBuf[whichLocal][xp][facex(rows-1,1)]
-    )/7;
-
-    sendBuf[newLocal][zp][facez(rows-1,cols-1)] = (
-          recvBuf[XN][facex(rows-1,0)]+
-          recvBuf[YN][facey(cols-1,0)]+
-          recvBuf[ZP][facez(rows-1,cols-1)]+
-          sendBuf[whichLocal][zp][facez(rows-1,cols-1)]+
-          sendBuf[whichLocal][zp][facez(rows-2,cols-1)]+
-          sendBuf[whichLocal][zp][facez(rows-1,cols-2)]+
-          sendBuf[whichLocal][xn][facex(rows-1,1)]
-    )/7;
-
-    eltsComp += 4;
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][zp][facez(0,i)] = (
-          recvBuf[ZP][facez(0,i)]+
-          recvBuf[YP][facey(i,0)]+
-          sendBuf[whichLocal][zp][facez(0,i)]+
-          sendBuf[whichLocal][zp][facez(0,i-1)]+
-          sendBuf[whichLocal][zp][facez(0,i+1)]+
-          sendBuf[whichLocal][zp][facez(1,i)]+
-          sendBuf[whichLocal][yp][facey(i,1)]
-      )/7; 
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][zp][facez(i,0)] = (
-          recvBuf[XP][facex(i,0)]+
-          recvBuf[ZP][facez(i,0)]+
-          sendBuf[whichLocal][zp][facez(i,0)]+
-          sendBuf[whichLocal][zp][facez(i-1,0)]+
-          sendBuf[whichLocal][zp][facez(i+1,0)]+
-          sendBuf[whichLocal][zp][facez(i,1)]+
-          sendBuf[whichLocal][xp][facex(i,1)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][zp][facez(rows-1,i)] = (
-          recvBuf[YN][facey(i,0)]+
-          recvBuf[ZP][facez(rows-1,i)]+
-          sendBuf[whichLocal][zp][facez(rows-1,i)]+
-          sendBuf[whichLocal][zp][facez(rows-1,i-1)]+
-          sendBuf[whichLocal][zp][facez(rows-1,i+1)]+
-          sendBuf[whichLocal][zp][facez(rows-2,i)]+
-          sendBuf[whichLocal][yn][facey(i,1)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][zp][facez(i,cols-1)] = (
-          recvBuf[XN][facex(i,0)]+
-          recvBuf[ZP][facez(i,cols-1)]+
-          sendBuf[whichLocal][zp][facez(i,cols-1)]+
-          sendBuf[whichLocal][zp][facez(i-1,cols-1)]+
-          sendBuf[whichLocal][zp][facez(i+1,cols-1)]+
-          sendBuf[whichLocal][zp][facez(i,cols-2)]+
-          sendBuf[whichLocal][xn][facex(i,1)]
-      )/7;
-      eltsComp+=1;
-    }
-    }
-    
-    // 2. zn face
-    // Interior points first
-    for(int i = 1; i < rows-1; i++){
-      for(int j = 1; j < cols-1; j++){
-        sendBuf[newLocal][zn][facez(i,j)] = (
-                sendBuf[whichLocal][zn][facez(i,j)]+
-                sendBuf[whichLocal][zn][facez(i-1,j)]+
-                sendBuf[whichLocal][zn][facez(i+1,j)]+
-                sendBuf[whichLocal][zn][facez(i,j-1)]+
-                sendBuf[whichLocal][zn][facez(i,j+1)]+
-                localChunk[whichLocal][small(i-1,j-1,planes-3)]+
-                recvBuf[ZN][facez(i,j)]
-        )/7;
-      eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    // Each element uses 3 ghosts, 3 from its own sendBuf (including itself)  and 1 from a different sendBuf
-    sendBuf[newLocal][zn][facez(0,0)] = (
-          recvBuf[YP][facey(0,planes-1)]+
-          recvBuf[XP][facex(0,planes-1)]+
-          recvBuf[ZN][facez(0,0)]+
-          sendBuf[whichLocal][zn][facez(0,0)]+
-          sendBuf[whichLocal][zn][facez(0,1)]+
-          sendBuf[whichLocal][zn][facez(1,0)]+
-          sendBuf[whichLocal][xp][facex(0,planes-2)]
-    )/7;
-
-    sendBuf[newLocal][zn][facez(0,cols-1)] = (
-          recvBuf[YP][facey(cols-1,planes-1)]+
-          recvBuf[XN][facex(0,planes-1)]+
-          recvBuf[ZN][facez(0,cols-1)]+
-          sendBuf[whichLocal][zn][facez(0,cols-1)]+
-          sendBuf[whichLocal][zn][facez(1,cols-1)]+
-          sendBuf[whichLocal][zn][facez(0,cols-2)]+
-          sendBuf[whichLocal][xn][facex(0,cols-2)]
-    )/7;
-
-    sendBuf[newLocal][zn][facez(rows-1,0)] = (
-          recvBuf[ZN][facez(rows-1,0)]+
-          recvBuf[XP][facex(rows-1,planes-1)]+
-          recvBuf[YN][facey(0,planes-1)]+
-          sendBuf[whichLocal][zn][facez(rows-1,0)]+
-          sendBuf[whichLocal][zn][facez(rows-2,0)]+
-          sendBuf[whichLocal][zn][facez(rows-1,1)]+
-          sendBuf[whichLocal][xp][facex(rows-1,planes-2)]
-    )/7;
-
-    sendBuf[newLocal][zn][facez(rows-1,cols-1)] = (
-          recvBuf[XN][facex(rows-1,planes-1)]+
-          recvBuf[YN][facey(cols-1,planes-1)]+
-          recvBuf[ZN][facez(rows-1,cols-1)]+
-          sendBuf[whichLocal][zn][facez(rows-1,cols-1)]+
-          sendBuf[whichLocal][zn][facez(rows-2,cols-1)]+
-          sendBuf[whichLocal][zn][facez(rows-1,cols-2)]+
-          sendBuf[whichLocal][xn][facex(rows-1,planes-2)]
-    )/7;
-
-      eltsComp+=4;
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][zn][facez(0,i)] = (
-          recvBuf[ZN][facez(0,i)]+
-          recvBuf[YP][facey(i,planes-1)]+
-          sendBuf[whichLocal][zn][facez(0,i)]+
-          sendBuf[whichLocal][zn][facez(0,i-1)]+
-          sendBuf[whichLocal][zn][facez(0,i+1)]+
-          sendBuf[whichLocal][zn][facez(1,i)]+
-          sendBuf[whichLocal][yp][facey(i,planes-2)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][zn][facez(i,0)] = (
-          recvBuf[XP][facex(i,planes-1)]+
-          recvBuf[ZN][facez(i,0)]+
-          sendBuf[whichLocal][zn][facez(i,0)]+
-          sendBuf[whichLocal][zn][facez(i-1,0)]+
-          sendBuf[whichLocal][zn][facez(i+1,0)]+
-          sendBuf[whichLocal][zn][facez(i,1)]+
-          sendBuf[whichLocal][xp][facex(i,planes-2)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][zn][facez(rows-1,i)] = (
-          recvBuf[YN][facey(i,planes-1)]+
-          recvBuf[ZN][facez(rows-1,i)]+
-          sendBuf[whichLocal][zn][facez(rows-1,i)]+
-          sendBuf[whichLocal][zn][facez(rows-1,i-1)]+
-          sendBuf[whichLocal][zn][facez(rows-1,i+1)]+
-          sendBuf[whichLocal][zn][facez(rows-2,i)]+
-          sendBuf[whichLocal][yn][facey(i,planes-2)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][zn][facez(i,cols-1)] = (
-          recvBuf[XN][facex(i,planes-1)]+
-          recvBuf[ZN][facez(i,cols-1)]+
-          sendBuf[whichLocal][zn][facez(i,cols-1)]+
-          sendBuf[whichLocal][zn][facez(i-1,cols-1)]+
-          sendBuf[whichLocal][zn][facez(i+1,cols-1)]+
-          sendBuf[whichLocal][zn][facez(i,cols-2)]+
-          sendBuf[whichLocal][xn][facex(i,planes-2)]
-      )/7;
-      eltsComp+=1;
-    }
-    
-    // 3. xp face
-    // Interior points first
-    for(int j = 1; j < planes-1; j++){
-      for(int i = 1; i < rows-1; i++){
-        sendBuf[newLocal][xp][facex(i,j)] = (
-                sendBuf[whichLocal][xp][facex(i,j)]+
-                sendBuf[whichLocal][xp][facex(i-1,j)]+
-                sendBuf[whichLocal][xp][facex(i+1,j)]+
-                sendBuf[whichLocal][xp][facex(i,j-1)]+
-                sendBuf[whichLocal][xp][facex(i,j+1)]+
-                localChunk[whichLocal][small(i-1,0,j-1)]+
-                recvBuf[XP][facex(i,j)]
-        )/7;
-      eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    sendBuf[newLocal][xp][facex(0,0)] = sendBuf[newLocal][zp][facez(0,0)];
-    sendBuf[newLocal][xp][facex(0,planes-1)] = sendBuf[newLocal][zn][facez(0,0)];
-    sendBuf[newLocal][xp][facex(rows-1,0)] = sendBuf[newLocal][zp][facez(rows-1,0)];
-    sendBuf[newLocal][xp][facex(rows-1,planes-1)] = sendBuf[newLocal][zn][facez(rows-1,0)];
-
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][xp][facex(0,i)] = (
-          recvBuf[XP][facex(0,i)]+
-          recvBuf[YP][facey(0,i)]+
-          sendBuf[whichLocal][xp][facex(0,i)]+
-          sendBuf[whichLocal][xp][facex(0,i-1)]+
-          sendBuf[whichLocal][xp][facex(0,i+1)]+
-          sendBuf[whichLocal][xp][facex(1,i)]+
-          sendBuf[whichLocal][yp][facey(1,i)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][xp][facex(i,0)] = sendBuf[newLocal][zp][facez(i,0)];
-    }
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][xp][facex(rows-1,i)] = (
-          recvBuf[YN][facey(0,i)]+
-          recvBuf[XP][facex(rows-1,i)]+
-          sendBuf[whichLocal][xp][facex(rows-1,i)]+
-          sendBuf[whichLocal][xp][facex(rows-1,i-1)]+
-          sendBuf[whichLocal][xp][facex(rows-1,i+1)]+
-          sendBuf[whichLocal][xp][facex(rows-2,i)]+
-          sendBuf[whichLocal][yn][facey(1,i)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][xp][facex(i,planes-1)] = sendBuf[newLocal][zn][facez(i,0)]; 
-    }
-   
-    // 4. xn face
-    // Interior points first
-    for(int j = 1; j < planes-1; j++){
-      for(int i = 1; i < rows-1; i++){
-        sendBuf[newLocal][xn][facex(i,j)] = (
-                sendBuf[whichLocal][xn][facex(i,j)]+
-                sendBuf[whichLocal][xn][facex(i-1,j)]+
-                sendBuf[whichLocal][xn][facex(i+1,j)]+
-                sendBuf[whichLocal][xn][facex(i,j-1)]+
-                sendBuf[whichLocal][xn][facex(i,j+1)]+
-                localChunk[whichLocal][small(i-1,cols-3,j-1)]+
-                recvBuf[XN][facex(i,j)]
-        )/7;
-      eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    sendBuf[newLocal][xn][facex(0,0)] = sendBuf[newLocal][zp][facez(0,cols-1)];
-    sendBuf[newLocal][xn][facex(0,planes-1)] = sendBuf[newLocal][zn][facez(0,cols-1)];
-    sendBuf[newLocal][xn][facex(rows-1,0)] = sendBuf[newLocal][zp][facez(rows-1,cols-1)];
-    sendBuf[newLocal][xn][facex(rows-1,planes-1)] = sendBuf[newLocal][zn][facez(rows-1,cols-1)];
-    
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][xn][facex(0,i)] = (
-          recvBuf[XN][facex(0,i)]+
-          recvBuf[YP][facey(cols-1,i)]+
-          sendBuf[whichLocal][xn][facex(0,i)]+
-          sendBuf[whichLocal][xn][facex(0,i-1)]+
-          sendBuf[whichLocal][xn][facex(0,i+1)]+
-          sendBuf[whichLocal][xn][facex(1,i)]+
-          sendBuf[whichLocal][yp][facey(cols-2,i)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][xn][facex(i,0)] = sendBuf[newLocal][zp][facez(i,cols-1)]; 
-    }
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][xn][facex(rows-1,i)] = (
-          recvBuf[YN][facey(cols-1,i)]+
-          recvBuf[XN][facex(rows-1,i)]+
-          sendBuf[whichLocal][xn][facex(rows-1,i)]+
-          sendBuf[whichLocal][xn][facex(rows-1,i-1)]+
-          sendBuf[whichLocal][xn][facex(rows-1,i+1)]+
-          sendBuf[whichLocal][xn][facex(rows-2,i)]+
-          sendBuf[whichLocal][yn][facey(cols-2,i)]
-      )/7;
-      eltsComp+=1;
-    }
-    for(int i = 1; i < rows-1; i++){
-      sendBuf[newLocal][xn][facex(i,planes-1)] = sendBuf[newLocal][zn][facez(i,cols-1)];
-    }
-    
-    // 5. yp face
-    // Interior points first
-    for(int j = 1; j < planes-1; j++){
-      for(int i = 1; i < cols-1; i++){
-        sendBuf[newLocal][yp][facey(i,j)] = (
-                sendBuf[whichLocal][yp][facey(i,j)]+
-                sendBuf[whichLocal][yp][facey(i-1,j)]+
-                sendBuf[whichLocal][yp][facey(i+1,j)]+
-                sendBuf[whichLocal][yp][facey(i,j-1)]+
-                sendBuf[whichLocal][yp][facey(i,j+1)]+
-                localChunk[whichLocal][small(0,i-1,j-1)]+
-                recvBuf[YP][facey(i,j)]
-        )/7;
-      eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    // Each element uses 3 ghosts, 3 from its own sendBuf (including itself)  and 1 from a different sendBuf
-    sendBuf[newLocal][yp][facey(0,0)] = sendBuf[newLocal][zp][facez(0,0)];
-    sendBuf[newLocal][yp][facey(0,planes-1)] = sendBuf[newLocal][zn][facez(0,0)];
-    sendBuf[newLocal][yp][facey(cols-1,0)] = sendBuf[newLocal][xn][facex(0,0)];
-    sendBuf[newLocal][yp][facey(cols-1,planes-1)] = sendBuf[newLocal][zn][facez(0,cols-1)]; 
-    
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][yp][facey(0,i)] = sendBuf[newLocal][xp][facex(0,i)]; 
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][yp][facey(i,0)] = sendBuf[newLocal][zp][facez(0,i)];
-    }
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][yp][facey(cols-1,i)] = sendBuf[newLocal][xn][facex(0,i)];
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][yp][facey(i,planes-1)] = sendBuf[newLocal][zn][facez(0,i)];
-    }
-
-    // 6. yn face
-    // Interior points first
-    for(int j = 1; j < planes-1; j++){
-      for(int i = 1; i < cols-1; i++){
-        sendBuf[newLocal][yn][facey(i,j)] = (
-                sendBuf[whichLocal][yn][facey(i,j)]+
-                sendBuf[whichLocal][yn][facey(i-1,j)]+
-                sendBuf[whichLocal][yn][facey(i+1,j)]+
-                sendBuf[whichLocal][yn][facey(i,j-1)]+
-                sendBuf[whichLocal][yn][facey(i,j+1)]+
-                localChunk[whichLocal][small(rows-3,i-1,j-1)]+
-                recvBuf[YN][facey(i,j)]
-        )/7;
-      eltsComp+=1;
-      }
-    }
-
-    // Corners next
-    // Each element uses 3 ghosts, 3 from its own sendBuf (including itself)  and 1 from a different sendBuf
-    sendBuf[newLocal][yn][facey(0,0)] = sendBuf[newLocal][zp][facez(rows-1,0)];
-    sendBuf[newLocal][yn][facey(0,planes-1)] = sendBuf[newLocal][zn][facez(rows-1,0)];
-    sendBuf[newLocal][yn][facey(cols-1,0)] = sendBuf[newLocal][zp][facez(rows-1,cols-1)];
-    sendBuf[newLocal][yn][facey(cols-1,planes-1)] = sendBuf[newLocal][zn][facez(rows-1,cols-1)]; 
-    
-    // Finally, the edges: 
-    // Each element here uses 2 ghosts, 4 elts from its own sendBuf, 1 elt from a different sendBuf
-    
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][yn][facey(0,i)] = sendBuf[newLocal][xp][facex(rows-1,i)]; 
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][yn][facey(i,0)] = sendBuf[newLocal][zp][facez(rows-1,i)];
-    }
-    for(int i = 1; i < planes-1; i++){
-      sendBuf[newLocal][yn][facey(cols-1,i)] = sendBuf[newLocal][xn][facex(rows-1,i)];
-    }
-    for(int i = 1; i < cols-1; i++){
-      sendBuf[newLocal][yn][facey(i,planes-1)] = sendBuf[newLocal][zn][facez(rows-1,i)];
-    }
-
-
-    
-    
-    // exclude the time for setup and the first iteration
-    iterations++;
-    if(iterations == 1){
-      contribute(0,0,CkReduction::concat, CkCallback(CkIndex_Main::doneSetup(), mainProxy));
-    }
-    
-    // toggle between localChunks
-    whichLocal = newLocal;
-    if(iterations == num_iterations){
-#ifdef STENCIL2D_VERBOSE
-      CkPrintf("(%d,%d): contributing to exit\n", row, col);
-#endif
-      contribute(sizeof(double), &eltsComp, CkReduction::sum_double, CkCallback(CkIndex_Main::done(NULL), mainProxy));
-    }
-    else{
-#ifdef USE_CKDIRECT
-      // 2. signal readiness to recv next round of data
-      for(int i = 0; i < NBRS; i++){
-        CkDirect_ready(&rhandles[i]);
-      }
-#endif
-      // contribute to barrier
-#ifdef STENCIL2D_VERBOSE
-      CkPrintf("(%d,%d): contributing to allReady\n", row, col);
-#endif
-      contribute(0, 0, CkReduction::concat, CkCallback(CkIndex_StencilPoint::allReadyCallback(NULL), thisProxy));
-    }
-    
-    if(iterations > num_iterations){
-      CkPrintf("******\n(%d,%d,%d):\n******\n", row, col,plane);
-      CkAbort("death is inevitable; bugs needn't be.\n");
-    }
-  }
-
-  void allReadyCallback(CkReductionMsg *msg){
-    delete msg;
-#ifdef STENCIL2D_VERBOSE
-    CkPrintf("(%d,%d,%d): all ready, send data\n", row, col,plane);
-#endif
-    sendData();
-  }
-
-  inline void sendMsg(int col, int row, int plane, int which, float *buf){
-    StencilMsg *msg = new (payload[which/2]) StencilMsg;
-    memcpy(msg->arr,buf,payload[which/2]*sizeof(float));
-    msg->which = which;
-    msg->size = payload[which/2];
-    thisProxy[lin(col,row,plane)].recvBufferMsg(msg);
-  }
-
-  void sendData(){
-    // 1. copy data into buffers from local chunk
-    // top and bottom boundaries
-#ifdef STENCIL2D_VERBOSE
-    //CkPrintf("(%d,%d,%d): sendData() called\n", x,y,z);
-#endif
-#ifdef USE_CKDIRECT
-    // 2. put buffers
-    for(int i = 0; i < NBRS; i++){
-      CkDirect_put(&shandles[whichLocal][i]);
-    }
-#else
-#ifdef USE_MESSAGES
-#ifdef ARR_CHECK
-    sendMsg((col+1)%charesx,row,plane,XP,sendBuf[whichLocal][xn].getVec());
-    sendMsg((col-1+charesx)%charesx,row,plane,XN,sendBuf[whichLocal][xp].getVec());
-    sendMsg(col,(row+1)%charesy,plane,YP,sendBuf[whichLocal][yn].getVec());
-    sendMsg(col,(row-1+charesy)%charesy,plane,YN,sendBuf[whichLocal][yp].getVec());
-    sendMsg(col,row,(plane+1)%charesz,ZP,sendBuf[whichLocal][zn].getVec());
-    sendMsg(col,row,(plane-1+charesz)%charesz,ZN,sendBuf[whichLocal][zp].getVec());
-#else
-    sendMsg((col+1)%charesx,row,plane,XP,sendBuf[whichLocal][xn]);
-    sendMsg((col-1+charesx)%charesx,row,plane,XN,sendBuf[whichLocal][xp]);
-    sendMsg(col,(row+1)%charesy,plane,YP,sendBuf[whichLocal][yn]);
-    sendMsg(col,(row-1+charesy)%charesy,plane,YN,sendBuf[whichLocal][yp]);
-    sendMsg(col,row,(plane+1)%charesz,ZP,sendBuf[whichLocal][zn]);
-    sendMsg(col,row,(plane-1+charesz)%charesz,ZN,sendBuf[whichLocal][zp]);
-#endif
-#else
-#ifdef ARR_CHECK
-    // 2. send messages
-    thisProxy[lin((col+1)%charesx,row,plane)].recvBuffer(sendBuf[whichLocal][xn].getVec(),payload[0],XP);
-    thisProxy[lin((col-1+charesx)%charesx,row,plane)].recvBuffer(sendBuf[whichLocal][xp].getVec(),payload[0],XN);
-    thisProxy[lin(col,(row+1)%charesy,plane)].recvBuffer(sendBuf[whichLocal][yn].getVec(),payload[1],YP);
-    thisProxy[lin(col,(row-1+charesy)%charesy,plane)].recvBuffer(sendBuf[whichLocal][yp].getVec(),payload[1],YN);
-    thisProxy[lin(col,row,(plane+1)%charesz)].recvBuffer(sendBuf[whichLocal][zn].getVec(),payload[2],ZP);
-    thisProxy[lin(col,row,(plane-1+charesz)%charesz)].recvBuffer(sendBuf[whichLocal][zp].getVec(),payload[2],ZN);
-#else
-    thisProxy[lin((col+1)%charesx,row,plane)].recvBuffer(sendBuf[whichLocal][xn],payload[0],XP);
-    thisProxy[lin((col-1+charesx)%charesx,row,plane)].recvBuffer(sendBuf[whichLocal][xp],payload[0],XN);
-    thisProxy[lin(col,(row+1)%charesy,plane)].recvBuffer(sendBuf[whichLocal][yn],payload[1],YP);
-    thisProxy[lin(col,(row-1+charesy)%charesy,plane)].recvBuffer(sendBuf[whichLocal][yp],payload[1],YN);
-    thisProxy[lin(col,row,(plane+1)%charesz)].recvBuffer(sendBuf[whichLocal][zn],payload[2],ZP);
-    thisProxy[lin(col,row,(plane-1+charesz)%charesz)].recvBuffer(sendBuf[whichLocal][zp],payload[2],ZN);
-#endif
-#endif // end USE_MESSAGES
-#endif
-  }
-
-};
-
-#include "stencil3d.def.h"
diff --git a/examples/charm++/ckdirect/stencil3d/stencil3d.ci b/examples/charm++/ckdirect/stencil3d/stencil3d.ci
deleted file mode 100644 (file)
index 9397944..0000000
+++ /dev/null
@@ -1,39 +0,0 @@
-mainmodule stencil3d{
-  readonly CProxy_Main mainProxy;
-  readonly int dimx;
-  readonly int dimy;
-  readonly int dimz;
-  readonly int charesx;
-  readonly int charesy;
-  readonly int charesz;
-  readonly int num_iterations;
-
-  mainchare Main{
-    entry Main(CkArgMsg *m);
-    //entry void done();
-    entry void done(CkReductionMsg *msg);
-    entry void doneSetup();
-  };
-
-  message StencilMsg{
-    float arr[];
-  };
-
-  array [1D] StencilPoint {
-    entry StencilPoint(void);
-    // with CkDirect
-    entry void setupChannels(void);
-    entry void notify(int node, int whichBoundary, CkIndex1D whoSent);
-    entry void recvHandle(infiDirectUserHandle handle, int whichBoundary);
-    entry void allReadyCallback(CkReductionMsg *msg);
-    
-    entry void compute();   
-
-
-    // when not using CkDirect
-    entry void sendData();
-    entry void recvBuffer(float arr[size], int size, int which);
-    entry void recvBufferMsg(StencilMsg *msg);
-    
-  };
-};