Adding example program for interoperability
authorNikhil Jain <nikhil@illinois.edu>
Tue, 7 Aug 2012 05:58:33 +0000 (00:58 -0500)
committerNikhil Jain <nikhil@illinois.edu>
Tue, 7 Aug 2012 05:58:33 +0000 (00:58 -0500)
16 files changed:
examples/charm++/mpi-coexist/Makefile [new file with mode: 0644]
examples/charm++/mpi-coexist/Makefile.in [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/Makefile [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hello/Makefile [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hello/hello.C [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hello/hello.ci [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hello/hello.h [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hi/Makefile [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hi/hi.C [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hi/hi.ci [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/hi/hi.h [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/kNeighbor/Makefile [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.C [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.ci [new file with mode: 0644]
examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.h [new file with mode: 0644]
examples/charm++/mpi-coexist/multirun.cpp [new file with mode: 0644]

diff --git a/examples/charm++/mpi-coexist/Makefile b/examples/charm++/mpi-coexist/Makefile
new file mode 100644 (file)
index 0000000..fbb011c
--- /dev/null
@@ -0,0 +1,24 @@
+include Makefile.in
+all: multirun
+
+#Charm++ libraries to be used for Interoperation
+LIBS=libmodulehi.a libmodulehello.a libmodulekNeighbor.a
+
+multirun: multirun.cpp $(LIBS)
+       $(CXX) -c multirun.cpp -o multirun.o -I$(CHARMDIR)/include
+       $(CHARMC) -mpi -o multirun multirun.o -L./ -module hello -module hi -module kNeighbor -module CommonLBs 
+
+$(LIBS):  
+       cd libs;make;
+
+clean: clear
+       cd libs;make clean;cd ..;
+       rm -f multirun *.o *.a charmrun
+
+clear:
+       rm -f PI*
+
+test: all
+       mpirun -np 16 ./multirun
+       mpirun -np 16 ./multirun +balancer GreedyLB +LBDebug 1
+
diff --git a/examples/charm++/mpi-coexist/Makefile.in b/examples/charm++/mpi-coexist/Makefile.in
new file mode 100644 (file)
index 0000000..b923f60
--- /dev/null
@@ -0,0 +1,4 @@
+OPTS = -O3
+CHARMDIR=../../..
+CHARMC=$(CHARMDIR)/bin/charmc $(OPTS)
+CXX=mpicxx $(OPTS)
diff --git a/examples/charm++/mpi-coexist/libs/Makefile b/examples/charm++/mpi-coexist/libs/Makefile
new file mode 100644 (file)
index 0000000..819f2e5
--- /dev/null
@@ -0,0 +1,14 @@
+all: libs
+
+DIRS=hi hello kNeighbor
+
+libs:
+       for d in $(DIRS); do \
+               (cd $$d && $(MAKE) all || exit 1) || exit 1; \
+       done
+       for d in $(DIRS); do \
+               cp $$d/"libmodule$$d.a" ../; \
+       done
+
+clean:
+       for d in $(DIRS); do (cd $$d; $(MAKE) clean); done
diff --git a/examples/charm++/mpi-coexist/libs/hello/Makefile b/examples/charm++/mpi-coexist/libs/hello/Makefile
new file mode 100644 (file)
index 0000000..d983b5d
--- /dev/null
@@ -0,0 +1,16 @@
+include ../../Makefile.in
+CHARMC:=../../${CHARMC}
+
+all: libmodulehello.a
+
+libmodulehello.a: hello.o
+       $(CHARMC) -o libmodulehello.a hello.o
+
+hello.decl.h: hello.ci
+       $(CHARMC) hello.ci
+
+hello.o: hello.C hello.decl.h
+       $(CHARMC) -c hello.C
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.a *.o charmrun *~ 
diff --git a/examples/charm++/mpi-coexist/libs/hello/hello.C b/examples/charm++/mpi-coexist/libs/hello/hello.C
new file mode 100644 (file)
index 0000000..9b4b6c9
--- /dev/null
@@ -0,0 +1,69 @@
+#include <stdio.h>
+#include "hello.decl.h"
+//header from Charm to enable Interoperation
+#include "mpi-interoperate.h"
+
+/*mainchare*/
+class MainHello : public CBase_MainHello
+{
+public:
+  MainHello(int elems)
+  {
+    //Process command-line arguments
+    int nElements=elems;
+    //Start the computation
+    CkPrintf("Running Hello on %d processors for %d elements\n",
+            CkNumPes(),nElements);
+    CProxy_MainHello mainHelloProxy = thisProxy;
+
+    CProxy_Hello arr = CProxy_Hello::ckNew(mainHelloProxy, nElements, nElements);
+
+    arr[0].SayHi(0);
+  };
+
+  MainHello(CkMigrateMessage *m) {}
+
+  void done(void)
+  {
+    CkExit();
+  };
+};
+
+/*array [1D]*/
+class Hello : public CBase_Hello 
+{
+private:
+  CProxy_MainHello mainProxy;
+  int nElements;
+public:
+  Hello(CProxy_MainHello mp, int nElems)
+  {
+    mainProxy = mp;
+    nElements = nElems;
+  }
+
+  Hello(CkMigrateMessage *m) {}
+  
+  void SayHi(int hiNo)
+  {
+    CkPrintf("Hello[%d] from element %d\n",hiNo,thisIndex);
+    if (thisIndex < nElements-1)
+      //Pass the hello on:
+      thisProxy[thisIndex+1].SayHi(hiNo+1);
+    else 
+      //We've been around once-- we're done.
+      mainProxy.done();
+  }
+};
+
+//C++ function invoked from MPI, marks the begining of Charm++
+void HelloStart(int elems)
+{
+  if(CkMyPe() == 0) {
+    CkPrintf("HelloStart - Starting lib by calling constructor of MainHello\n");
+    CProxy_MainHello mainhello = CProxy_MainHello::ckNew(elems);
+  }
+  CsdScheduler(-1);
+}
+
+#include "hello.def.h"
diff --git a/examples/charm++/mpi-coexist/libs/hello/hello.ci b/examples/charm++/mpi-coexist/libs/hello/hello.ci
new file mode 100644 (file)
index 0000000..cb8277d
--- /dev/null
@@ -0,0 +1,12 @@
+module hello {
+
+  chare MainHello {
+    entry MainHello(int elems);
+    entry void done(void);
+  };
+
+  array [1D] Hello {
+    entry Hello(CProxy_MainHello mp, int nElems);
+    entry void SayHi(int hiNo);
+  };           
+};
diff --git a/examples/charm++/mpi-coexist/libs/hello/hello.h b/examples/charm++/mpi-coexist/libs/hello/hello.h
new file mode 100644 (file)
index 0000000..06a7f90
--- /dev/null
@@ -0,0 +1,3 @@
+//interface for invocation from MPI
+void HelloStart(int elems);
+
diff --git a/examples/charm++/mpi-coexist/libs/hi/Makefile b/examples/charm++/mpi-coexist/libs/hi/Makefile
new file mode 100644 (file)
index 0000000..29bed73
--- /dev/null
@@ -0,0 +1,16 @@
+include ../../Makefile.in
+CHARMC:=../../${CHARMC}
+
+all: libmodulehi.a
+
+libmodulehi.a: hi.o
+       $(CHARMC) -o libmodulehi.a hi.o
+
+hi.decl.h: hi.ci
+       $(CHARMC) hi.ci
+
+hi.o: hi.C hi.decl.h
+       $(CHARMC) -c hi.C
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.a *.o charmrun *~ 
diff --git a/examples/charm++/mpi-coexist/libs/hi/hi.C b/examples/charm++/mpi-coexist/libs/hi/hi.C
new file mode 100644 (file)
index 0000000..c8f4da7
--- /dev/null
@@ -0,0 +1,73 @@
+#include <stdio.h>
+#include "hi.decl.h"
+//header from Charm to enable Interoperation
+#include "mpi-interoperate.h"
+
+CProxy_MainHi mainHi;
+/*mainchare*/
+class MainHi : public CBase_MainHi
+{
+public:
+  MainHi(CkArgMsg *m) {
+    mainHi = thisProxy;
+    delete m;
+  }
+
+  void StartHi(int elems)
+  {
+    //Process command-line arguments
+    int nElements=elems;
+    //Start the computation
+
+    CkPrintf("Running Hi on %d processors for %d elements\n",
+            CkNumPes(),nElements);
+    CProxy_Hi arr = CProxy_Hi::ckNew(mainHi, nElements, nElements);
+
+    arr[0].SayHi(1);
+  };
+
+  MainHi(CkMigrateMessage *m) {}
+
+  void done(void)
+  {
+    CkExit();
+  };
+};
+
+/*array [1D]*/
+class Hi : public CBase_Hi 
+{
+private:
+  CProxy_MainHi mainProxy;
+  int nElements;
+public:
+  Hi(CProxy_MainHi mp, int nElems)
+  {
+    mainProxy = mp;
+    nElements = nElems;
+  }
+
+  Hi(CkMigrateMessage *m) {}
+  
+  void SayHi(int hiNo)
+  {
+    CkPrintf("Hi[%d] from element %d\n",hiNo,thisIndex);
+    if (thisIndex < nElements-1)
+      //Pass the hello on:
+      thisProxy[thisIndex+1].SayHi(hiNo+1);
+    else 
+      //We've been around once-- we're done.
+      mainProxy.done();
+  }
+};
+
+//C++ function invoked from MPI, marks the begining of Charm++
+void HiStart(int elems)
+{
+  if(CkMyPe() == 0) {
+    mainHi.StartHi(elems);
+  }
+  CsdScheduler(-1);
+}
+
+#include "hi.def.h"
diff --git a/examples/charm++/mpi-coexist/libs/hi/hi.ci b/examples/charm++/mpi-coexist/libs/hi/hi.ci
new file mode 100644 (file)
index 0000000..f14e39d
--- /dev/null
@@ -0,0 +1,15 @@
+module hi {
+
+  readonly CProxy_MainHi mainHi;
+
+  mainchare MainHi {
+    entry MainHi(CkArgMsg *m);
+    entry void StartHi(int elems);
+    entry void done(void);
+  };
+
+  array [1D] Hi {
+    entry Hi(CProxy_MainHi mp, int nElems);
+    entry void SayHi(int hiNo);
+  };           
+};
diff --git a/examples/charm++/mpi-coexist/libs/hi/hi.h b/examples/charm++/mpi-coexist/libs/hi/hi.h
new file mode 100644 (file)
index 0000000..4b5f481
--- /dev/null
@@ -0,0 +1,2 @@
+//interface for invocation from MPI
+void HiStart(int elems);
diff --git a/examples/charm++/mpi-coexist/libs/kNeighbor/Makefile b/examples/charm++/mpi-coexist/libs/kNeighbor/Makefile
new file mode 100644 (file)
index 0000000..8bf642e
--- /dev/null
@@ -0,0 +1,16 @@
+include ../../Makefile.in
+CHARMC:=../../${CHARMC}
+
+all: libmodulekNeighbor.a
+
+libmodulekNeighbor.a: kNeighbor.o
+       $(CHARMC) -o libmodulekNeighbor.a kNeighbor.o
+
+kNeighbor.decl.h: kNeighbor.ci
+       $(CHARMC) kNeighbor.ci
+
+kNeighbor.o: kNeighbor.C kNeighbor.decl.h
+       $(CHARMC) -c kNeighbor.C
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.a *.o charmrun *~ 
diff --git a/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.C b/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.C
new file mode 100644 (file)
index 0000000..05ff120
--- /dev/null
@@ -0,0 +1,356 @@
+#include "kNeighbor.decl.h"
+#include <stdio.h>
+#include <stdlib.h>
+//header from Charm to enable Interoperation
+#include "mpi-interoperate.h"
+
+#define STRIDEK                1
+#define CALCPERSTEP    10
+
+#define DEBUG          0
+
+
+/* readonly */ CProxy_Main mainProxy;
+/* readonly */ int num_chares;
+/* readonly */ int gMsgSize;
+/* readonly */ int gLBFreq;
+
+int cmpFunc(const void *a, const void *b) {
+  if(*(double *)a < *(double *)b) return -1;
+  if(*(double *)a > *(double *)b) return 1;
+  return 0;
+}
+
+class toNeighborMsg: public CMessage_toNeighborMsg {
+  public:
+    int *data;
+    int size;
+    int fromX;
+    int nID;
+
+  public:
+    toNeighborMsg() {};
+    toNeighborMsg(int s): size(s) {  
+    }
+
+    void setMsgSrc(int X, int id) {
+      fromX = X;
+      nID = id;
+    }
+
+};
+
+class Main: public CBase_Main {
+  public:
+    CProxy_Block array;
+
+    int numSteps;
+    int currentStep;
+    int currentMsgSize;
+
+    int numElemsRcvd;
+    double totalTime;
+    double maxTime;
+    double minTime;
+    double *timeRec;
+
+    double gStarttime;
+
+  public:
+    Main(CkArgMsg *m) {
+      mainProxy = thisProxy;
+    }
+
+               void beginWork(int numsteps, int msgsize, int lbfreq)
+               {
+       numSteps = numsteps;;
+       currentMsgSize = msgsize;
+
+                       gLBFreq = lbfreq;
+
+       CkPrintf("\nStarting kNeighbor ...\n");
+       gMsgSize = currentMsgSize;
+       currentStep = -1;
+
+       timeRec = new double[numSteps];
+                       array = CProxy_Block::ckNew(num_chares);
+       CkCallback *cb = new CkCallback(CkIndex_Main::nextStep(NULL), thisProxy);
+       array.ckSetReductionClient(cb);
+
+       beginIteration();
+               }
+
+    void beginIteration() {
+      currentStep++;
+      if (currentStep == numSteps) {
+       CkPrintf("kNeighbor program finished!\n\n");
+       //CkCallback *cb = new CkCallback(CkIndex_Main::terminate(NULL), thisProxy);
+       //array.ckSetReductionClient(cb);
+       terminate(NULL);
+       return;
+      }
+
+      numElemsRcvd = 0;
+      totalTime = 0.0;
+      maxTime = 0.0;
+      minTime = 3600.0;
+
+      //currentMsgSize = msgSize;
+      if(currentStep!=0 && (currentStep % gLBFreq == 0)) {
+       array.pauseForLB();
+       return;
+      }
+
+      gStarttime = CmiWallTimer();
+      for (int i=0; i<num_chares; i++)
+       array[i].commWithNeighbors();
+    }
+
+    void resumeIter() {
+#if DEBUG
+      CkPrintf("Resume iteration at step %d\n", currentStep);
+#endif
+      gStarttime = CmiWallTimer();
+      for (int i=0; i<num_chares; i++)
+       array[i].commWithNeighbors();
+    }
+
+    void terminate(CkReductionMsg *msg) {
+      delete msg;
+      double total = 0.0;
+
+      for (int i=0; i<numSteps; i++)
+       timeRec[i] = timeRec[i]*1e6;
+
+      qsort(timeRec, numSteps, sizeof(double), cmpFunc);
+      printf("Time stats: lowest: %f, median: %f, highest: %f\n", timeRec[0], timeRec[numSteps/2], timeRec[numSteps-1]);
+
+      int samples = 100;
+      if(numSteps<=samples) samples = numSteps-1;
+      for (int i=0; i<samples; i++)
+       total += timeRec[i];
+      total /= samples;
+
+      CkPrintf("Average time for each %d-Neighbor iteration with msg size %d is %f (us)\n", STRIDEK, currentMsgSize, total);
+      CkExit();
+    }
+
+    void nextStep(CkReductionMsg  *msg) {
+      maxTime = *((double *)msg->getData());
+      delete msg;
+      double wholeStepTime = CmiWallTimer() - gStarttime;
+      timeRec[currentStep] = wholeStepTime/CALCPERSTEP;
+      if(currentStep % 10 == 0)
+       CkPrintf("Step %d with msg size %d finished: max=%f, total=%f\n", currentStep, currentMsgSize, maxTime/CALCPERSTEP, wholeStepTime/CALCPERSTEP);
+      beginIteration();
+    }
+
+};
+
+
+//no wrap around for sending messages to neighbors
+class Block: public CBase_Block {
+  public:
+    int numNeighbors;
+    int numNborsRcvd;
+    int *neighbors;
+    double *recvTimes;
+    double startTime;
+
+    int random;
+    int curIterMsgSize;
+    int internalStepCnt;
+    int sum;
+
+    toNeighborMsg **iterMsg;
+
+  public:
+    Block() {
+      //srand(thisIndex.x+thisIndex.y);
+      usesAtSync = CmiTrue;
+
+      numNeighbors = 2*STRIDEK;
+      neighbors = new int[numNeighbors];
+      recvTimes = new double[numNeighbors];
+      int nidx=0;
+      //setting left neighbors
+      for (int i=thisIndex-STRIDEK; i<thisIndex; i++, nidx++) {
+       int tmpnei = i;
+       while (tmpnei<0) tmpnei += num_chares;
+       neighbors[nidx] = tmpnei;
+      }
+      //setting right neighbors
+      for (int i=thisIndex+1; i<=thisIndex+STRIDEK; i++, nidx++) {
+       int tmpnei = i;
+       while (tmpnei>=num_chares) tmpnei -= num_chares;
+       neighbors[nidx] = tmpnei;
+      }
+
+      for (int i=0; i<numNeighbors; i++)
+       recvTimes[i] = 0.0;
+
+      iterMsg = new toNeighborMsg *[numNeighbors];
+      for (int i=0; i<numNeighbors; i++)
+       iterMsg[i] = NULL;
+
+#if DEBUG
+      CkPrintf("Neighbors of %d: ", thisIndex);
+      for (int i=0; i<numNeighbors; i++)
+       CkPrintf("%d ", neighbors[i]);
+      CkPrintf("\n");
+#endif
+
+      random = thisIndex*31+73;
+    }
+
+    ~Block() {
+      delete [] neighbors;
+      delete [] recvTimes;
+      delete [] iterMsg;
+    }
+
+    void pup(PUP::er &p){
+      CBase_Block::pup(p);
+      p(numNeighbors);
+      p(numNborsRcvd);
+
+      if(p.isUnpacking()) {
+       neighbors = new int[numNeighbors];
+       recvTimes = new double[numNeighbors];
+      }
+      PUParray(p, neighbors, numNeighbors);
+      PUParray(p, recvTimes, numNeighbors);
+      p(startTime);
+      p(random);
+      p(curIterMsgSize);
+      p(internalStepCnt);
+      p(sum);
+      if(p.isUnpacking()) iterMsg = new toNeighborMsg *[numNeighbors];
+      for(int i=0; i<numNeighbors; i++){
+       CkPupMessage(p, (void **)&iterMsg[i]);
+      }
+    }
+
+    Block(CkMigrateMessage *m) {}
+
+    void pauseForLB(){
+#if DEBUG
+      CkPrintf("Element %d pause for LB on PE %d\n", thisIndex, CkMyPe());
+#endif
+      AtSync();
+    }
+
+    void ResumeFromSync(){ //Called by load-balancing framework
+      CkCallback cb(CkIndex_Main::resumeIter(), mainProxy);
+      contribute(0, NULL, CkReduction::sum_int, cb);
+    }
+
+    void startInternalIteration() {
+#if DEBUG
+      CkPrintf("[%d]: Start internal iteration \n", thisIndex);
+#endif
+
+      numNborsRcvd = 0;
+      /* 1: pick a work size and do some computation */
+      int N = (thisIndex * thisIndex / num_chares) * 100;
+      for (int i=0; i<N; i++)
+       for (int j=0; j<N; j++) {
+         sum += (thisIndex * i + j);
+       }
+
+      /* 2. send msg to K neighbors */
+      int msgSize = curIterMsgSize;
+
+      // Send msgs to neighbors
+      for (int i=0; i<numNeighbors; i++) {
+       //double memtimer = CmiWallTimer();
+
+       toNeighborMsg *msg = iterMsg[i];
+
+#if DEBUG
+       CkPrintf("[%d]: send msg to neighbor[%d]=%d\n", thisIndex, i, neighbors[i]);
+#endif
+       msg->setMsgSrc(thisIndex, i);
+       //double entrytimer = CmiWallTimer();
+       thisProxy(neighbors[i]).recvMsgs(msg);
+       //double entrylasttimer = CmiWallTimer();
+       //if(thisIndex==0){
+       //      CkPrintf("At current step %d to neighbor %d, msg creation time: %f, entrymethod fire time: %f\n", internalStepCnt, neighbors[i], entrytimer-memtimer, entrylasttimer-entrytimer);
+       //}
+      }
+    }
+
+    void commWithNeighbors() {
+      internalStepCnt = 0;
+      curIterMsgSize = gMsgSize;
+      //currently the work size is only changed every big steps (which
+      //are initiated by the main proxy
+      random++;
+
+      if(iterMsg[0]==NULL) { //indicating the messages have not been created
+       for(int i=0; i<numNeighbors; i++)
+         iterMsg[i] = new(curIterMsgSize/4, 0) toNeighborMsg(curIterMsgSize/4);
+      }
+
+      startTime = CmiWallTimer();
+      startInternalIteration();
+    }
+
+    void recvReplies(toNeighborMsg *m) {
+      int fromNID = m->nID;
+
+#if DEBUG
+      CkPrintf("[%d]: receive ack from neighbor[%d]=%d\n", thisIndex, fromNID, neighbors[fromNID]);
+#endif
+
+      iterMsg[fromNID] = m;
+      //recvTimes[fromNID] += (CmiWallTimer() - startTime);
+
+      //get one step time and send it back to mainProxy
+      numNborsRcvd++;
+      if (numNborsRcvd == numNeighbors) {
+       internalStepCnt++;
+       if (internalStepCnt==CALCPERSTEP) {
+         double iterCommTime = CmiWallTimer() - startTime;
+         contribute(sizeof(double), &iterCommTime, CkReduction::max_double);
+         /*if(thisIndex==0){
+           for(int i=0; i<numNeighbors; i++){
+           CkPrintf("RTT time from neighbor %d (actual elem id %d): %lf\n", i, neighbors[i], recvTimes[i]);
+           }
+           }*/
+       } else {
+         startInternalIteration();
+       }
+      }
+    }
+
+    void recvMsgs(toNeighborMsg *m) {
+#if DEBUG
+      CkPrintf("[%d]: recv msg from %d as its %dth neighbor\n", thisIndex, m->fromX, m->nID);
+#endif
+
+      thisProxy(m->fromX).recvReplies(m);
+    }
+
+    inline int MAX(int a, int b) {
+      return (a>b)?a:b;
+    }
+    inline int MIN(int a, int b) {
+      return (a<b)?a:b;
+    }
+};
+
+//C++ function invoked from MPI, marks the begining of Charm++
+void kNeighbor(int numchares, int numsteps, int msgsize, int lbfreq)
+{
+       num_chares = num_chares;
+  gMsgSize = msgsize;
+  if(num_chares < CkNumPes()) {
+               num_chares = CkNumPes();
+  }
+       if(CkMyPe() == 0) {
+               mainProxy.beginWork(numsteps,msgsize,lbfreq);
+       }
+  CsdScheduler(-1);
+}
+#include "kNeighbor.def.h"
diff --git a/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.ci b/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.ci
new file mode 100644 (file)
index 0000000..b444df6
--- /dev/null
@@ -0,0 +1,27 @@
+module kNeighbor {
+  readonly CProxy_Main mainProxy;
+  readonly int num_chares;
+  readonly int gMsgSize;
+  readonly int gLBFreq;
+
+  message toNeighborMsg {
+    int data[];
+  };
+
+  mainchare Main {
+    entry Main(CkArgMsg *m);
+               entry void beginWork(int numsteps,int msgsize,int lbfreq);
+    entry void nextStep(CkReductionMsg *);
+    entry void terminate(CkReductionMsg *);
+    entry void resumeIter();
+  };
+
+  array [1D] Block {
+    entry Block();
+    entry void commWithNeighbors();
+    entry void recvReplies(toNeighborMsg *);
+    entry void recvMsgs(toNeighborMsg *);
+    entry void pauseForLB();
+  };
+
+}
diff --git a/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.h b/examples/charm++/mpi-coexist/libs/kNeighbor/kNeighbor.h
new file mode 100644 (file)
index 0000000..7a70b55
--- /dev/null
@@ -0,0 +1,3 @@
+//interface for invocation from MPI
+void kNeighbor(int numchares, int numsteps, int msgsize, int lbfreq);
+
diff --git a/examples/charm++/mpi-coexist/multirun.cpp b/examples/charm++/mpi-coexist/multirun.cpp
new file mode 100644 (file)
index 0000000..c424103
--- /dev/null
@@ -0,0 +1,84 @@
+/*  Example code to demonstrate Interoperability between MPI and Charm
+    Author - Nikhil Jain
+    Contact - nikhil@illinois.edu
+*/
+
+//standard header files
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+//header files for libraries in Charm I wish to use with MPI
+#include "libs/hi/hi.h"
+#include "libs/hello/hello.h"
+#include "libs/kNeighbor/kNeighbor.h"
+//header file from Charm needed for Interoperation
+#include "mpi-interoperate.h"
+
+int main(int argc, char **argv){
+  int peid, numpes;
+  MPI_Comm newComm;
+
+  //basic MPI initilization
+  MPI_Init(&argc, &argv);
+  MPI_Comm_rank(MPI_COMM_WORLD, &peid);
+  MPI_Comm_size(MPI_COMM_WORLD, &numpes);
+
+  if(numpes % 4 != 0){
+    if(peid==0){
+      printf("This test program must be run with number of procs = 4x\n");
+    }
+    MPI_Finalize();
+    return 1;
+  }
+
+  //splitting WORLD into 2 sets
+  MPI_Comm_split(MPI_COMM_WORLD, peid%2, peid, &newComm);
+
+  //initialize Charm for each set
+  CharmLibInit(newComm, argc, argv);
+  MPI_Barrier(MPI_COMM_WORLD);
+
+  //do some MPI work
+  for(int i=0; i<5; i++) {
+    if(peid % 2 == 0) {    
+      MPI_Send(&peid, 1, MPI_INT, peid+1, 808, MPI_COMM_WORLD);
+    } else {
+      int recvid = 0;
+      MPI_Status sts;
+      MPI_Recv(&recvid, 1, MPI_INT, peid-1, 808, MPI_COMM_WORLD, &sts);
+    }
+  }
+
+  //on the first set of processors, invoke Hello
+  if(peid%2) {
+    HelloStart(5);
+    MPI_Barrier(newComm);
+  } else {
+  //on the other set of processors, invoke kNeighbor and clean up Charm
+    kNeighbor(10,10,128,2);
+    MPI_Barrier(newComm);
+    CharmLibExit();
+  }
+    
+  //on the first set, do more MPI work, invoke Hi and clean up Charm
+  if(peid%2) {
+    for(int i=0; i<5; i++) {
+      if(peid % 4 == 1) {    
+        MPI_Send(&peid, 1, MPI_INT, peid+2, 808, MPI_COMM_WORLD);
+      }  else {
+        int recvid = 0;
+        MPI_Status sts;
+        MPI_Recv(&recvid, 1, MPI_INT, peid-2, 808, MPI_COMM_WORLD, &sts);
+      }
+    }
+    HiStart(16);
+    MPI_Barrier(newComm);
+    CharmLibExit();
+  }
+
+  //final synchronization
+  MPI_Barrier(MPI_COMM_WORLD);
+
+  MPI_Finalize();
+  return 0;  
+}