mpi charmcollide library and example
authorEhsan Totoni <totoni2@illinois.edu>
Sun, 28 Oct 2012 21:49:31 +0000 (16:49 -0500)
committerEhsan Totoni <totoni2@illinois.edu>
Sun, 28 Oct 2012 21:49:31 +0000 (16:49 -0500)
examples/charm++/mpi-charmcollide/Makefile [new file with mode: 0644]
examples/charm++/mpi-charmcollide/Makefile.in [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/Makefile [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/collideClient.h [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/mpiCollisionClient.h [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/mpicollide.C [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/mpicollide.ci [new file with mode: 0644]
examples/charm++/mpi-charmcollide/collidelib/mpicollide.h [new file with mode: 0644]
examples/charm++/mpi-charmcollide/multirun.cpp [new file with mode: 0644]

diff --git a/examples/charm++/mpi-charmcollide/Makefile b/examples/charm++/mpi-charmcollide/Makefile
new file mode 100644 (file)
index 0000000..c667d01
--- /dev/null
@@ -0,0 +1,24 @@
+include Makefile.in
+all: multirun
+
+#Charm++ libraries to be used for Interoperation
+LIBS=libmodulempicollide.a
+
+multirun: multirun.cpp $(LIBS)
+       $(CXX) -c multirun.cpp -o multirun.o -I$(CHARMDIR)/include
+       $(CHARMC) -mpi -o multirun multirun.o -L./collidelib -module mpicollide -module CommonLBs -module collidecharm
+
+$(LIBS):  
+       cd collidelib;make;
+
+clean: clear
+       cd collidelib;make clean;cd ..;
+       rm -f multirun *.o *.a charmrun
+
+clear:
+       rm -f PI*
+
+test: all
+       mpirun -np 16 ./multirun
+       mpirun -np 16 ./multirun +balancer GreedyLB +LBDebug 1
+
diff --git a/examples/charm++/mpi-charmcollide/Makefile.in b/examples/charm++/mpi-charmcollide/Makefile.in
new file mode 100644 (file)
index 0000000..b923f60
--- /dev/null
@@ -0,0 +1,4 @@
+OPTS = -O3
+CHARMDIR=../../..
+CHARMC=$(CHARMDIR)/bin/charmc $(OPTS)
+CXX=mpicxx $(OPTS)
diff --git a/examples/charm++/mpi-charmcollide/collidelib/Makefile b/examples/charm++/mpi-charmcollide/collidelib/Makefile
new file mode 100644 (file)
index 0000000..d222bc3
--- /dev/null
@@ -0,0 +1,16 @@
+include ../Makefile.in
+CHARMC:=../${CHARMC}
+
+all: libmodulempicollide.a
+
+libmodulempicollide.a: mpicollide.o
+       $(CHARMC) -o libmodulempicollide.a mpicollide.o
+
+mpicollide.decl.h: mpicollide.ci
+       $(CHARMC) mpicollide.ci
+
+mpicollide.o: mpicollide.C mpicollide.decl.h
+       $(CHARMC) -module collidecharm -c mpicollide.C
+
+clean:
+       rm -f *.decl.h *.def.h conv-host *.a *.o charmrun *~ 
diff --git a/examples/charm++/mpi-charmcollide/collidelib/collideClient.h b/examples/charm++/mpi-charmcollide/collidelib/collideClient.h
new file mode 100644 (file)
index 0000000..d2bf79b
--- /dev/null
@@ -0,0 +1,218 @@
+/**
+ * @file collideClient.h
+ * @brief charm declarations (Chare group) required for defining collision detection libraries client,
+ * the library needs a cleaner fix
+ * @author Ehsan Totoni
+ * @version 
+ * @date 2012-09-05
+ */
+#ifndef _CLIENT_DECL_H_
+#define _CLIENT_DECL_H_
+
+/* DECLS: group collideClient: IrrGroup;
+ */
+ class collideClient;
+ class CkIndex_collideClient;
+ class CProxy_collideClient;
+ class CProxyElement_collideClient;
+ class CProxySection_collideClient;
+/* --------------- index object ------------------ */
+class CkIndex_collideClient:public CProxyElement_IrrGroup{
+  public:
+    typedef collideClient local_t;
+    typedef CkIndex_collideClient index_t;
+    typedef CProxy_collideClient proxy_t;
+    typedef CProxyElement_collideClient element_t;
+    typedef CProxySection_collideClient section_t;
+
+    static int __idx;
+    static void __register(const char *s, size_t size);
+};
+/* --------------- element proxy ------------------ */
+class CProxyElement_collideClient: public CProxyElement_IrrGroup{
+  public:
+    typedef collideClient local_t;
+    typedef CkIndex_collideClient index_t;
+    typedef CProxy_collideClient proxy_t;
+    typedef CProxyElement_collideClient element_t;
+    typedef CProxySection_collideClient section_t;
+
+    CProxyElement_collideClient(void) {}
+    CProxyElement_collideClient(const IrrGroup *g) : CProxyElement_IrrGroup(g){  }
+    CProxyElement_collideClient(CkGroupID _gid,int _onPE,CK_DELCTOR_PARAM) : CProxyElement_IrrGroup(_gid,_onPE,CK_DELCTOR_ARGS){  }
+    CProxyElement_collideClient(CkGroupID _gid,int _onPE) : CProxyElement_IrrGroup(_gid,_onPE){  }
+
+    int ckIsDelegated(void) const
+    { return CProxyElement_IrrGroup::ckIsDelegated(); }
+    inline CkDelegateMgr *ckDelegatedTo(void) const
+    { return CProxyElement_IrrGroup::ckDelegatedTo(); }
+    inline CkDelegateData *ckDelegatedPtr(void) const
+    { return CProxyElement_IrrGroup::ckDelegatedPtr(); }
+    CkGroupID ckDelegatedIdx(void) const
+    { return CProxyElement_IrrGroup::ckDelegatedIdx(); }
+inline void ckCheck(void) const {CProxyElement_IrrGroup::ckCheck();}
+CkChareID ckGetChareID(void) const
+   {return CProxyElement_IrrGroup::ckGetChareID();}
+CkGroupID ckGetGroupID(void) const
+   {return CProxyElement_IrrGroup::ckGetGroupID();}
+operator CkGroupID () const { return ckGetGroupID(); }
+
+    inline void setReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxyElement_IrrGroup::setReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxyElement_IrrGroup::ckSetReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkCallback *cb) const
+    { CProxyElement_IrrGroup::ckSetReductionClient(cb); }
+int ckGetGroupPe(void) const
+{return CProxyElement_IrrGroup::ckGetGroupPe();}
+
+    void ckDelegate(CkDelegateMgr *dTo,CkDelegateData *dPtr=NULL)
+    {       CProxyElement_IrrGroup::ckDelegate(dTo,dPtr); }
+    void ckUndelegate(void)
+    {       CProxyElement_IrrGroup::ckUndelegate(); }
+    void pup(PUP::er &p)
+    {       CProxyElement_IrrGroup::pup(p); }
+    void ckSetGroupID(CkGroupID g) {
+      CProxyElement_IrrGroup::ckSetGroupID(g);
+    }
+    collideClient* ckLocalBranch(void) const {
+      return ckLocalBranch(ckGetGroupID());
+    }
+    static collideClient* ckLocalBranch(CkGroupID gID) {
+      return (collideClient*)CkLocalBranch(gID);
+    }
+};
+PUPmarshall(CProxyElement_collideClient)
+/* ---------------- collective proxy -------------- */
+class CProxy_collideClient: public CProxy_IrrGroup{
+  public:
+    typedef collideClient local_t;
+    typedef CkIndex_collideClient index_t;
+    typedef CProxy_collideClient proxy_t;
+    typedef CProxyElement_collideClient element_t;
+    typedef CProxySection_collideClient section_t;
+
+    CProxy_collideClient(void) {}
+    CProxy_collideClient(const IrrGroup *g) : CProxy_IrrGroup(g){  }
+    CProxy_collideClient(CkGroupID _gid,CK_DELCTOR_PARAM) : CProxy_IrrGroup(_gid,CK_DELCTOR_ARGS){  }
+    CProxy_collideClient(CkGroupID _gid) : CProxy_IrrGroup(_gid){  }
+    CProxyElement_collideClient operator[](int onPE) const
+      {return CProxyElement_collideClient(ckGetGroupID(),onPE,CK_DELCTOR_CALL);}
+
+    int ckIsDelegated(void) const
+    { return CProxy_IrrGroup::ckIsDelegated(); }
+    inline CkDelegateMgr *ckDelegatedTo(void) const
+    { return CProxy_IrrGroup::ckDelegatedTo(); }
+    inline CkDelegateData *ckDelegatedPtr(void) const
+    { return CProxy_IrrGroup::ckDelegatedPtr(); }
+    CkGroupID ckDelegatedIdx(void) const
+    { return CProxy_IrrGroup::ckDelegatedIdx(); }
+inline void ckCheck(void) const {CProxy_IrrGroup::ckCheck();}
+CkChareID ckGetChareID(void) const
+   {return CProxy_IrrGroup::ckGetChareID();}
+CkGroupID ckGetGroupID(void) const
+   {return CProxy_IrrGroup::ckGetGroupID();}
+operator CkGroupID () const { return ckGetGroupID(); }
+
+    inline void setReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxy_IrrGroup::setReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxy_IrrGroup::ckSetReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkCallback *cb) const
+    { CProxy_IrrGroup::ckSetReductionClient(cb); }
+
+    void ckDelegate(CkDelegateMgr *dTo,CkDelegateData *dPtr=NULL)
+    {       CProxy_IrrGroup::ckDelegate(dTo,dPtr); }
+    void ckUndelegate(void)
+    {       CProxy_IrrGroup::ckUndelegate(); }
+    void pup(PUP::er &p)
+    {       CProxy_IrrGroup::pup(p); }
+    void ckSetGroupID(CkGroupID g) {
+      CProxy_IrrGroup::ckSetGroupID(g);
+    }
+    collideClient* ckLocalBranch(void) const {
+      return ckLocalBranch(ckGetGroupID());
+    }
+    static collideClient* ckLocalBranch(CkGroupID gID) {
+      return (collideClient*)CkLocalBranch(gID);
+    }
+};
+PUPmarshall(CProxy_collideClient)
+/* ---------------- section proxy -------------- */
+class CProxySection_collideClient: public CProxySection_IrrGroup{
+  public:
+    typedef collideClient local_t;
+    typedef CkIndex_collideClient index_t;
+    typedef CProxy_collideClient proxy_t;
+    typedef CProxyElement_collideClient element_t;
+    typedef CProxySection_collideClient section_t;
+
+    CProxySection_collideClient(void) {}
+    CProxySection_collideClient(const IrrGroup *g) : CProxySection_IrrGroup(g){  }
+    CProxySection_collideClient(const CkGroupID &_gid,const int *_pelist,int _npes,CK_DELCTOR_PARAM) : CProxySection_IrrGroup(_gid,_pelist,_npes,CK_DELCTOR_ARGS){  }
+    CProxySection_collideClient(const CkGroupID &_gid,const int *_pelist,int _npes) : CProxySection_IrrGroup(_gid,_pelist,_npes){  }
+    CProxySection_collideClient(int n,const CkGroupID *_gid, int const * const *_pelist,const int *_npes) : CProxySection_IrrGroup(n,_gid,_pelist,_npes){  }
+    CProxySection_collideClient(int n,const CkGroupID *_gid, int const * const *_pelist,const int *_npes,CK_DELCTOR_PARAM) : CProxySection_IrrGroup(n,_gid,_pelist,_npes,CK_DELCTOR_ARGS){  }
+
+    int ckIsDelegated(void) const
+    { return CProxySection_IrrGroup::ckIsDelegated(); }
+    inline CkDelegateMgr *ckDelegatedTo(void) const
+    { return CProxySection_IrrGroup::ckDelegatedTo(); }
+    inline CkDelegateData *ckDelegatedPtr(void) const
+    { return CProxySection_IrrGroup::ckDelegatedPtr(); }
+    CkGroupID ckDelegatedIdx(void) const
+    { return CProxySection_IrrGroup::ckDelegatedIdx(); }
+inline void ckCheck(void) const {CProxySection_IrrGroup::ckCheck();}
+CkChareID ckGetChareID(void) const
+   {return CProxySection_IrrGroup::ckGetChareID();}
+CkGroupID ckGetGroupID(void) const
+   {return CProxySection_IrrGroup::ckGetGroupID();}
+operator CkGroupID () const { return ckGetGroupID(); }
+
+    inline void setReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxySection_IrrGroup::setReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkReductionClientFn fn,void *param=NULL) const
+    { CProxySection_IrrGroup::ckSetReductionClient(fn,param); }
+    inline void ckSetReductionClient(CkCallback *cb) const
+    { CProxySection_IrrGroup::ckSetReductionClient(cb); }
+inline int ckGetNumSections() const
+{ return CProxySection_IrrGroup::ckGetNumSections(); }
+inline CkSectionInfo &ckGetSectionInfo()
+{ return CProxySection_IrrGroup::ckGetSectionInfo(); }
+inline CkSectionID *ckGetSectionIDs()
+{ return CProxySection_IrrGroup::ckGetSectionIDs(); }
+inline CkSectionID &ckGetSectionID()
+{ return CProxySection_IrrGroup::ckGetSectionID(); }
+inline CkSectionID &ckGetSectionID(int i)
+{ return CProxySection_IrrGroup::ckGetSectionID(i); }
+inline CkGroupID ckGetGroupIDn(int i) const
+{ return CProxySection_IrrGroup::ckGetGroupIDn(i); }
+inline int *ckGetElements() const
+{ return CProxySection_IrrGroup::ckGetElements(); }
+inline int *ckGetElements(int i) const
+{ return CProxySection_IrrGroup::ckGetElements(i); }
+inline int ckGetNumElements() const
+{ return CProxySection_IrrGroup::ckGetNumElements(); } 
+inline int ckGetNumElements(int i) const
+{ return CProxySection_IrrGroup::ckGetNumElements(i); }
+
+    void ckDelegate(CkDelegateMgr *dTo,CkDelegateData *dPtr=NULL)
+    {       CProxySection_IrrGroup::ckDelegate(dTo,dPtr); }
+    void ckUndelegate(void)
+    {       CProxySection_IrrGroup::ckUndelegate(); }
+    void pup(PUP::er &p)
+    {       CProxySection_IrrGroup::pup(p); }
+    void ckSetGroupID(CkGroupID g) {
+      CProxySection_IrrGroup::ckSetGroupID(g);
+    }
+    collideClient* ckLocalBranch(void) const {
+      return ckLocalBranch(ckGetGroupID());
+    }
+    static collideClient* ckLocalBranch(CkGroupID gID) {
+      return (collideClient*)CkLocalBranch(gID);
+    }
+};
+PUPmarshall(CProxySection_collideClient)
+typedef CBaseT1<Group, CProxy_collideClient> CBase_collideClient;
+
+#endif
diff --git a/examples/charm++/mpi-charmcollide/collidelib/mpiCollisionClient.h b/examples/charm++/mpi-charmcollide/collidelib/mpiCollisionClient.h
new file mode 100644 (file)
index 0000000..43fb306
--- /dev/null
@@ -0,0 +1,41 @@
+/**
+ * @file mpiCollisionClient.h
+ * @brief mpi collision library's Chare group client, 
+ * collisions are delivered to it and sets the return pointer
+ * @author Ehsan Totoni
+ * @version 
+ * @date 2012-10-10
+ */
+
+/*readonly*/ extern CProxy_MainCollide mainProxy;
+
+
+class MpiCollisionClient: public collideClient {
+
+       CollisionList* returnColls;
+       public:
+               MpiCollisionClient() {
+               }
+               virtual ~MpiCollisionClient(){}
+               /* --------------------------------------------------------------------------*/
+               /**
+                * @brief charm library delivers potential collisions here to client, it sets the pointer
+                *
+                * @param src
+                * @param step
+                * @param colls list of potential collisions
+                */
+               /* ----------------------------------------------------------------------------*/
+               virtual void collisions(ArrayElement *src, int step, CollisionList &colls){
+                       int size = colls.length();
+                       for (int c=0;c<size;c++) {
+                               returnColls->add(colls[c].A,colls[c].B);
+                       }
+                       CkCallback cb(CkReductionTarget(MainCollide, done), mainProxy);
+                       src->contribute(sizeof(int), &size, CkReduction::sum_int, cb);
+               }
+               void setResultPointer(CollisionList *&colls) {
+                       returnColls = colls =new CollisionList();
+               }
+
+};
diff --git a/examples/charm++/mpi-charmcollide/collidelib/mpicollide.C b/examples/charm++/mpi-charmcollide/collidelib/mpicollide.C
new file mode 100644 (file)
index 0000000..ee2c9c0
--- /dev/null
@@ -0,0 +1,55 @@
+#include <stdio.h>
+#include "collidecharm.h"
+#include "collideClient.h"
+#include "mpicollide.decl.h"
+#include "mpiCollisionClient.h"
+//header from Charm to enable Interoperation
+#include "mpi-interoperate.h"
+
+/* readonly */ CProxy_MpiCollisionClient myClient;
+/* readonly */ CollideHandle collision_detector;
+/* readonly */ CProxy_MainCollide mainProxy;
+
+/*mainchare*/
+class MainCollide : public CBase_MainCollide
+{
+public:
+  MainCollide(CkArgMsg *m)
+  {
+       mainProxy = thisProxy;   
+       double grid_dim_size = 1.0;
+       if(m->argc >1 ) grid_dim_size=atof(m->argv[1]);
+       // create CProxy_collideMgr
+       myClient = CProxy_MpiCollisionClient::ckNew();
+       // origin of grid and grid cell size
+       //vector3d origin(0,0,0), grid_size(grid_dim_size, grid_dim_size, grid_dim_size);
+       vector3d origin(0,0,0), grid_size(2,100,2);
+       CollideGrid3d grid(origin, grid_size);
+       collision_detector = CollideCreate(grid, myClient);
+        //Start the computation
+        CkPrintf("Running collide on %d processors\n", CkNumPes());
+
+  };
+
+  MainCollide(CkMigrateMessage *m) {}
+
+  void done(int numColls)
+  {
+         CkPrintf("number of collisions:%d \n", numColls);
+    CkExit();
+  };
+};
+
+//C++ function invoked from MPI, marks the begining of Charm++
+void detectCollision(CollisionList *&colls,int nBoxes, bbox3d *boxes, int *prio)
+{
+       CollideRegister(collision_detector, CkMyPe());
+       myClient.ckLocalBranch()->setResultPointer(colls);
+       CollideBoxesPrio(collision_detector, CkMyPe(), nBoxes, boxes, prio);
+  if(CkMyPe() == 0) {
+    CkPrintf("calling collision detection\n");
+  }
+  CsdScheduler(-1);
+}
+
+#include "mpicollide.def.h"
diff --git a/examples/charm++/mpi-charmcollide/collidelib/mpicollide.ci b/examples/charm++/mpi-charmcollide/collidelib/mpicollide.ci
new file mode 100644 (file)
index 0000000..82b8801
--- /dev/null
@@ -0,0 +1,16 @@
+module mpicollide {
+
+  readonly CProxy_MpiCollisionClient myClient;
+  readonly CollideHandle collision_detector;
+  readonly CProxy_MainCollide mainProxy;
+
+  mainchare MainCollide {
+    entry MainCollide(CkArgMsg *m);
+    entry [reductiontarget] void done(int numColls);
+  };
+
+  group MpiCollisionClient : collideClient{
+          entry MpiCollisionClient();
+  }
+
+};
diff --git a/examples/charm++/mpi-charmcollide/collidelib/mpicollide.h b/examples/charm++/mpi-charmcollide/collidelib/mpicollide.h
new file mode 100644 (file)
index 0000000..b74063a
--- /dev/null
@@ -0,0 +1,4 @@
+#include "collidecharm.h"
+//interface for invocation from MPI
+void detectCollision(CollisionList *&colls,int nBoxes, bbox3d *boxes, int *prio=NULL);
+
diff --git a/examples/charm++/mpi-charmcollide/multirun.cpp b/examples/charm++/mpi-charmcollide/multirun.cpp
new file mode 100644 (file)
index 0000000..8fa8921
--- /dev/null
@@ -0,0 +1,58 @@
+/*  Example code to demonstrate Interoperability between MPI and Charm
+    Author - Nikhil Jain
+    Contact - nikhil@illinois.edu
+*/
+
+//standard header files
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+//header files for libraries in Charm I wish to use with MPI
+#include "collidelib/mpicollide.h"
+//header file from Charm needed for Interoperation
+#include "mpi-interoperate.h"
+
+int main(int argc, char **argv){
+  int peid, numpes;
+  MPI_Comm newComm;
+
+  //basic MPI initilization
+  MPI_Init(&argc, &argv);
+  MPI_Comm_rank(MPI_COMM_WORLD, &peid);
+  MPI_Comm_size(MPI_COMM_WORLD, &numpes);
+
+
+  //initialize Charm for each set
+  CharmLibInit(MPI_COMM_WORLD, argc, argv);
+  MPI_Barrier(MPI_COMM_WORLD);
+  CollisionList *colls;
+  CkVector3d o(-6.8,7.9,8.0), x(4.0,0,0), y(0,0.3,0);
+  CkVector3d boxSize(0.2,0.2,0.2);
+  int nBoxes=1000;
+  bbox3d *box=new bbox3d[nBoxes];
+  for (int i=0;i<nBoxes;i++) {
+         CkVector3d c(o+x*peid+y*i);
+         CkVector3d c2(c+boxSize);
+         box[i].empty();
+         box[i].add(c); box[i].add(c2);
+  }
+  // first box stretches over into next object:
+  box[0].add(o+x*(peid+1.5)+y*2);
+  detectCollision(colls,nBoxes, box, NULL);
+  int numColls=colls->length();
+  for (int c=0;c<numColls;c++) {
+         printf("%d:%d hits %d:%d\n",
+                         (*colls)[c].A.chunk,(*colls)[c].A.number,
+                         (*colls)[c].B.chunk,(*colls)[c].B.number);
+  }
+
+  delete box;
+  MPI_Barrier(MPI_COMM_WORLD);
+  CharmLibExit();
+
+  //final synchronization
+  MPI_Barrier(MPI_COMM_WORLD);
+
+  MPI_Finalize();
+  return 0;  
+}