Zcpy Bcast Sender-side API: Add examples for each registration mode 80/4680/18
authorNitin Bhat <nbhat4@illinois.edu>
Wed, 10 Oct 2018 15:12:40 +0000 (11:12 -0400)
committerEvan Ramos <evan@hpccharm.com>
Wed, 10 Apr 2019 18:04:50 +0000 (13:04 -0500)
Change-Id: I9cc4ec2366bc5f270302cae286e43c0fa194e31a

15 files changed:
examples/charm++/zerocopy/Makefile
examples/charm++/zerocopy/entry_method_api/Makefile
examples/charm++/zerocopy/entry_method_bcast_api/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/prereg/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/Makefile [moved from examples/charm++/zerocopy/entry_method_api/simpleBcast/Makefile with 83% similarity]
examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/simpleBcast.C [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/simpleBcast.ci [moved from examples/charm++/zerocopy/entry_method_api/simpleBcast/simpleBcast.ci with 100% similarity]
examples/charm++/zerocopy/entry_method_bcast_api/reg/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/simpleBcast.C [moved from examples/charm++/zerocopy/entry_method_api/simpleBcast/simpleBcast.C with 98% similarity]
examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/simpleBcast.ci [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/unreg/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/Makefile [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.C [new file with mode: 0644]
examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.ci [new file with mode: 0644]

index 2ccbb0aaa3fa92f323473dec2bc0bd06fb546c35..83d44b3179b8e6f874cf08ad349b1cc9884f8dad 100644 (file)
@@ -1,6 +1,7 @@
 DIRS = \
   direct_api \
   entry_method_api \
+  entry_method_bcast_api \
 
 TESTDIRS = $(DIRS)
 
index 2430a23cda9189abf56732640d59a0b630220e5c..4cc65d57535128690450183b0f338401a1a58c4e 100644 (file)
@@ -3,7 +3,6 @@ DIRS = \
   prereg \
   reg \
   misc \
-  simpleBcast \
 
 TESTDIRS = $(DIRS)
 
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/Makefile
new file mode 100644 (file)
index 0000000..dc0558e
--- /dev/null
@@ -0,0 +1,23 @@
+DIRS = \
+  unreg \
+  prereg \
+  reg \
+
+TESTDIRS = $(DIRS)
+
+all: $(foreach i,$(DIRS),build-$i)
+
+test: $(foreach i,$(TESTDIRS),test-$i)
+
+clean: $(foreach i,$(DIRS),clean-$i)
+       rm -f TAGS #*#
+       rm -f core *~
+
+$(foreach i,$(DIRS),build-$i):
+       $(MAKE) -C $(subst build-,,$@) all OPTS='$(OPTS)'
+
+$(foreach i,$(DIRS),test-$i):
+       $(MAKE) -C $(subst test-,,$@) test OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)'
+
+$(foreach i,$(DIRS),clean-$i):
+       $(MAKE) -C $(subst clean-,,$@) clean OPTS='$(OPTS)'
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/prereg/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/prereg/Makefile
new file mode 100644 (file)
index 0000000..f0471d3
--- /dev/null
@@ -0,0 +1,21 @@
+DIRS = \
+  simpleBcast \
+
+TESTDIRS = $(DIRS)
+
+all: $(foreach i,$(DIRS),build-$i)
+
+test: $(foreach i,$(TESTDIRS),test-$i)
+
+clean: $(foreach i,$(DIRS),clean-$i)
+       rm -f TAGS #*#
+       rm -f core *~
+
+$(foreach i,$(DIRS),build-$i):
+       $(MAKE) -C $(subst build-,,$@) all OPTS='$(OPTS)'
+
+$(foreach i,$(DIRS),test-$i):
+       $(MAKE) -C $(subst test-,,$@) test OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)'
+
+$(foreach i,$(DIRS),clean-$i):
+       $(MAKE) -C $(subst clean-,,$@) clean OPTS='$(OPTS)'
similarity index 83%
rename from examples/charm++/zerocopy/entry_method_api/simpleBcast/Makefile
rename to examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/Makefile
index 4703933b9eb37be9e75338c38e734b6243b99b24..d0730167f3f802ce23fb54e9301bd95171ec7d78 100644 (file)
@@ -1,5 +1,5 @@
--include ../../../../common.mk
-CHARMC=../../../../../bin/charmc $(OPTS)
+-include ../../../../../common.mk
+CHARMC=../../../../../../bin/charmc $(OPTS)
 
 all: simpleBcast
 
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/simpleBcast.C b/examples/charm++/zerocopy/entry_method_bcast_api/prereg/simpleBcast/simpleBcast.C
new file mode 100644 (file)
index 0000000..635880e
--- /dev/null
@@ -0,0 +1,118 @@
+#include "simpleBcast.decl.h"
+
+#define DEBUG(x) //x
+
+CProxy_Main mainProxy;
+
+template<class T>
+void assignValues(T *&arr, int size){
+  for(int i=0; i<size; i++)
+     arr[i] = i;
+}
+
+template<class T>
+void checkArrValues(T *&arr, int size){
+  for(int i=0; i<size; i++)
+     CkAssert(arr[i] == i);
+}
+
+class Main : public CBase_Main{
+  int size;
+  int counter;
+  public:
+  Main(CkArgMsg *m) {
+    // Create an array of size received in arguments
+    if(m->argc > 2) {
+      // print error message
+      CkAbort("Usage: ./simpleBcast <array-size>");
+    } else if(m->argc == 2 ) {
+      size = atoi(m->argv[1]);
+    } else {
+      size = CkNumPes() * 10; // default with 10 chare array elements per pe
+    }
+
+    counter = 0;
+    mainProxy = thisProxy;
+
+    // Create a chare array
+    CProxy_zcArray arrProxy = CProxy_zcArray::ckNew(size);
+
+    // Create a group
+    CProxy_zcGroup grpProxy = CProxy_zcGroup::ckNew();
+
+    // Create a nodegroup
+    CProxy_zcNodegroup ngrpProxy = CProxy_zcNodegroup::ckNew();
+
+    // allocate a large array
+    int bufferSize = 2000000;
+    int *buffer = (int *)CkRdmaAlloc(sizeof(int) * bufferSize);
+    assignValues(buffer, bufferSize);
+
+    // create a callback method
+    int idx_zerocopySent = CkIndex_Main::zerocopySent(NULL);
+    CkCallback cb = CkCallback(idx_zerocopySent, thisProxy);
+
+    CkCallback doneCb = CkCallback(CkReductionTarget(Main, done), thisProxy);
+
+    // invoking bcast on chare array
+    arrProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_PREREG), bufferSize, doneCb);
+
+    // invoking bcast on group
+    grpProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_PREREG), bufferSize, doneCb);
+
+    // invoking bcast on nodegroup
+    ngrpProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_PREREG), bufferSize, doneCb);
+  }
+
+  void zerocopySent(CkDataMsg *m) {
+    CkPrintf("[%d][%d][%d] Source callback invoked\n", CkMyPe(), CkMyNode(), CmiMyRank());
+    done();
+  }
+
+  void done() {
+    // Wait for 3 reductions to complete: Chare Array, Group, Nodegroup and
+    // 3 more calls from zerocopySent callback method on completion of sending the buffer
+    if(++counter == 6) {
+      CkPrintf("[%d][%d][%d] All operations completed\n", CkMyPe(), CkMyNode(), CmiMyRank());
+      CkExit();
+    }
+  }
+};
+
+class zcArray : public CBase_zcArray {
+  public:
+  zcArray() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Chare array element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+class zcGroup : public CBase_zcGroup {
+  public:
+  zcGroup() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Group element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+class zcNodegroup : public CBase_zcNodegroup {
+  public:
+  zcNodegroup() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Nodegroup element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+#include "simpleBcast.def.h"
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/reg/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/reg/Makefile
new file mode 100644 (file)
index 0000000..f0471d3
--- /dev/null
@@ -0,0 +1,21 @@
+DIRS = \
+  simpleBcast \
+
+TESTDIRS = $(DIRS)
+
+all: $(foreach i,$(DIRS),build-$i)
+
+test: $(foreach i,$(TESTDIRS),test-$i)
+
+clean: $(foreach i,$(DIRS),clean-$i)
+       rm -f TAGS #*#
+       rm -f core *~
+
+$(foreach i,$(DIRS),build-$i):
+       $(MAKE) -C $(subst build-,,$@) all OPTS='$(OPTS)'
+
+$(foreach i,$(DIRS),test-$i):
+       $(MAKE) -C $(subst test-,,$@) test OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)'
+
+$(foreach i,$(DIRS),clean-$i):
+       $(MAKE) -C $(subst clean-,,$@) clean OPTS='$(OPTS)'
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/Makefile
new file mode 100644 (file)
index 0000000..d073016
--- /dev/null
@@ -0,0 +1,21 @@
+-include ../../../../../common.mk
+CHARMC=../../../../../../bin/charmc $(OPTS)
+
+all: simpleBcast
+
+simpleBcast:  simpleBcast.o
+       $(CHARMC) simpleBcast.o -o simpleBcast -language charm++ -module CommonLBs
+
+cifiles: simpleBcast.ci
+       $(CHARMC) -c simpleBcast.ci
+       touch cifiles
+
+simpleBcast.o : simpleBcast.C cifiles
+       $(CHARMC) -c simpleBcast.C
+
+test: all
+       $(call run, +p4 ./simpleBcast)
+       $(call run, +p6 ./simpleBcast)
+
+clean:
+       rm -f *.def.h *.decl.h *.o *~ *.exe cifiles charmrun simpleBcast
similarity index 98%
rename from examples/charm++/zerocopy/entry_method_api/simpleBcast/simpleBcast.C
rename to examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/simpleBcast.C
index 2809cec7becdd6e414c5a6d8dd85a09343f15e56..b28b0aa4774b9eef58a2314e46a79135162382ed 100644 (file)
@@ -6,7 +6,6 @@ CProxy_Main mainProxy;
 
 template<class T>
 void assignValues(T *&arr, int size){
-  arr = new T[size];
   for(int i=0; i<size; i++)
      arr[i] = i;
 }
@@ -47,7 +46,7 @@ class Main : public CBase_Main{
     CProxy_zcNodegroup ngrpProxy = CProxy_zcNodegroup::ckNew();
 
     // allocate a large array
-    int bufferSize = 200000;
+    int bufferSize = 2000000;
     int *buffer = new int[bufferSize];
     assignValues(buffer, bufferSize);
 
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/simpleBcast.ci b/examples/charm++/zerocopy/entry_method_bcast_api/reg/simpleBcast/simpleBcast.ci
new file mode 100644 (file)
index 0000000..6c1e158
--- /dev/null
@@ -0,0 +1,25 @@
+mainmodule simpleBcast {
+
+  readonly CProxy_Main mainProxy;
+
+  mainchare Main {
+    entry Main(CkArgMsg *m);
+    entry void zerocopySent(CkDataMsg *m);
+    entry [reductiontarget] void done();
+  };
+
+  array [1D] zcArray {
+    entry zcArray();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+
+  group zcGroup {
+    entry zcGroup();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+
+  nodegroup zcNodegroup {
+    entry zcNodegroup();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+}
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/unreg/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/unreg/Makefile
new file mode 100644 (file)
index 0000000..f0471d3
--- /dev/null
@@ -0,0 +1,21 @@
+DIRS = \
+  simpleBcast \
+
+TESTDIRS = $(DIRS)
+
+all: $(foreach i,$(DIRS),build-$i)
+
+test: $(foreach i,$(TESTDIRS),test-$i)
+
+clean: $(foreach i,$(DIRS),clean-$i)
+       rm -f TAGS #*#
+       rm -f core *~
+
+$(foreach i,$(DIRS),build-$i):
+       $(MAKE) -C $(subst build-,,$@) all OPTS='$(OPTS)'
+
+$(foreach i,$(DIRS),test-$i):
+       $(MAKE) -C $(subst test-,,$@) test OPTS='$(OPTS)' TESTOPTS='$(TESTOPTS)'
+
+$(foreach i,$(DIRS),clean-$i):
+       $(MAKE) -C $(subst clean-,,$@) clean OPTS='$(OPTS)'
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/Makefile b/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/Makefile
new file mode 100644 (file)
index 0000000..d073016
--- /dev/null
@@ -0,0 +1,21 @@
+-include ../../../../../common.mk
+CHARMC=../../../../../../bin/charmc $(OPTS)
+
+all: simpleBcast
+
+simpleBcast:  simpleBcast.o
+       $(CHARMC) simpleBcast.o -o simpleBcast -language charm++ -module CommonLBs
+
+cifiles: simpleBcast.ci
+       $(CHARMC) -c simpleBcast.ci
+       touch cifiles
+
+simpleBcast.o : simpleBcast.C cifiles
+       $(CHARMC) -c simpleBcast.C
+
+test: all
+       $(call run, +p4 ./simpleBcast)
+       $(call run, +p6 ./simpleBcast)
+
+clean:
+       rm -f *.def.h *.decl.h *.o *~ *.exe cifiles charmrun simpleBcast
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.C b/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.C
new file mode 100644 (file)
index 0000000..efa95af
--- /dev/null
@@ -0,0 +1,118 @@
+#include "simpleBcast.decl.h"
+
+#define DEBUG(x) //x
+
+CProxy_Main mainProxy;
+
+template<class T>
+void assignValues(T *&arr, int size){
+  for(int i=0; i<size; i++)
+     arr[i] = i;
+}
+
+template<class T>
+void checkArrValues(T *&arr, int size){
+  for(int i=0; i<size; i++)
+     CkAssert(arr[i] == i);
+}
+
+class Main : public CBase_Main{
+  int size;
+  int counter;
+  public:
+  Main(CkArgMsg *m) {
+    // Create an array of size received in arguments
+    if(m->argc > 2) {
+      // print error message
+      CkAbort("Usage: ./simpleBcast <array-size>");
+    } else if(m->argc == 2 ) {
+      size = atoi(m->argv[1]);
+    } else {
+      size = CkNumPes() * 10; // default with 10 chare array elements per pe
+    }
+
+    counter = 0;
+    mainProxy = thisProxy;
+
+    // Create a chare array
+    CProxy_zcArray arrProxy = CProxy_zcArray::ckNew(size);
+
+    // Create a group
+    CProxy_zcGroup grpProxy = CProxy_zcGroup::ckNew();
+
+    // Create a nodegroup
+    CProxy_zcNodegroup ngrpProxy = CProxy_zcNodegroup::ckNew();
+
+    // allocate a large array
+    int bufferSize = 2000000;
+    int *buffer = new int[bufferSize];
+    assignValues(buffer, bufferSize);
+
+    // create a callback method
+    int idx_zerocopySent = CkIndex_Main::zerocopySent(NULL);
+    CkCallback cb = CkCallback(idx_zerocopySent, thisProxy);
+
+    CkCallback doneCb = CkCallback(CkReductionTarget(Main, done), thisProxy);
+
+    // invoking bcast on chare array
+    arrProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_UNREG), bufferSize, doneCb);
+
+    // invoking bcast on group
+    grpProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_UNREG), bufferSize, doneCb);
+
+    // invoking bcast on nodegroup
+    ngrpProxy.recvLargeArray(CkSendBuffer(buffer, cb, CK_BUFFER_UNREG), bufferSize, doneCb);
+  }
+
+  void zerocopySent(CkDataMsg *m) {
+    CkPrintf("[%d][%d][%d] Source callback invoked\n", CkMyPe(), CkMyNode(), CmiMyRank());
+    done();
+  }
+
+  void done() {
+    // Wait for 3 reductions to complete: Chare Array, Group, Nodegroup and
+    // 3 more calls from zerocopySent callback method on completion of sending the buffer
+    if(++counter == 6) {
+      CkPrintf("[%d][%d][%d] All operations completed\n", CkMyPe(), CkMyNode(), CmiMyRank());
+      CkExit();
+    }
+  }
+};
+
+class zcArray : public CBase_zcArray {
+  public:
+  zcArray() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Chare array element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+class zcGroup : public CBase_zcGroup {
+  public:
+  zcGroup() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Group element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+class zcNodegroup : public CBase_zcNodegroup {
+  public:
+  zcNodegroup() {}
+
+  void recvLargeArray(int *ptr1, int n1, CkCallback doneCb) {
+    DEBUG(CkPrintf("[%d][%d] APP Nodegroup element received large array %p\n", thisIndex, CkMyPe(), ptr1);)
+    checkArrValues(ptr1, n1);
+
+    contribute(doneCb);
+  }
+};
+
+#include "simpleBcast.def.h"
diff --git a/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.ci b/examples/charm++/zerocopy/entry_method_bcast_api/unreg/simpleBcast/simpleBcast.ci
new file mode 100644 (file)
index 0000000..6c1e158
--- /dev/null
@@ -0,0 +1,25 @@
+mainmodule simpleBcast {
+
+  readonly CProxy_Main mainProxy;
+
+  mainchare Main {
+    entry Main(CkArgMsg *m);
+    entry void zerocopySent(CkDataMsg *m);
+    entry [reductiontarget] void done();
+  };
+
+  array [1D] zcArray {
+    entry zcArray();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+
+  group zcGroup {
+    entry zcGroup();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+
+  nodegroup zcNodegroup {
+    entry zcNodegroup();
+    entry void recvLargeArray(nocopy int ptr1[n1], int n1, CkCallback doneCb);
+  };
+}